4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2018, Joyent, Inc.
25 * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
29 * DTrace - Dynamic Tracing for Solaris
31 * This is the implementation of the Solaris Dynamic Tracing framework
32 * (DTrace). The user-visible interface to DTrace is described at length in
33 * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace
34 * library, the in-kernel DTrace framework, and the DTrace providers are
35 * described in the block comments in the <sys/dtrace.h> header file. The
36 * internal architecture of DTrace is described in the block comments in the
37 * <sys/dtrace_impl.h> header file. The comments contained within the DTrace
38 * implementation very much assume mastery of all of these sources; if one has
39 * an unanswered question about the implementation, one should consult them
42 * The functions here are ordered roughly as follows:
44 * - Probe context functions
45 * - Probe hashing functions
46 * - Non-probe context utility functions
47 * - Matching functions
48 * - Provider-to-Framework API functions
49 * - Probe management functions
50 * - DIF object functions
52 * - Predicate functions
55 * - Enabling functions
57 * - Anonymous enabling functions
58 * - Consumer state functions
61 * - Driver cookbook functions
63 * Each group of functions begins with a block comment labelled the "DTrace
64 * [Group] Functions", allowing one to find each block by searching forward
65 * on capital-f functions.
67 #include <sys/errno.h>
69 #include <sys/modctl.h>
71 #include <sys/systm.h>
73 #include <sys/sunddi.h>
74 #include <sys/cpuvar.h>
76 #include <sys/strsubr.h>
77 #include <sys/sysmacros.h>
78 #include <sys/dtrace_impl.h>
79 #include <sys/atomic.h>
80 #include <sys/cmn_err.h>
81 #include <sys/mutex_impl.h>
82 #include <sys/rwlock_impl.h>
83 #include <sys/ctf_api.h>
84 #include <sys/panic.h>
85 #include <sys/priv_impl.h>
86 #include <sys/policy.h>
87 #include <sys/cred_impl.h>
88 #include <sys/procfs_isa.h>
89 #include <sys/taskq.h>
90 #include <sys/mkdev.h>
93 #include <sys/socket.h>
94 #include <netinet/in.h>
95 #include "strtolctype.h"
98 * DTrace Tunable Variables
100 * The following variables may be tuned by adding a line to /etc/system that
101 * includes both the name of the DTrace module ("dtrace") and the name of the
102 * variable. For example:
104 * set dtrace:dtrace_destructive_disallow = 1
106 * In general, the only variables that one should be tuning this way are those
107 * that affect system-wide DTrace behavior, and for which the default behavior
108 * is undesirable. Most of these variables are tunable on a per-consumer
109 * basis using DTrace options, and need not be tuned on a system-wide basis.
110 * When tuning these variables, avoid pathological values; while some attempt
111 * is made to verify the integrity of these variables, they are not considered
112 * part of the supported interface to DTrace, and they are therefore not
113 * checked comprehensively. Further, these variables should not be tuned
114 * dynamically via "mdb -kw" or other means; they should only be tuned via
117 int dtrace_destructive_disallow
= 0;
118 dtrace_optval_t dtrace_nonroot_maxsize
= (16 * 1024 * 1024);
119 size_t dtrace_difo_maxsize
= (256 * 1024);
120 dtrace_optval_t dtrace_dof_maxsize
= (8 * 1024 * 1024);
121 size_t dtrace_statvar_maxsize
= (16 * 1024);
122 size_t dtrace_actions_max
= (16 * 1024);
123 size_t dtrace_retain_max
= 1024;
124 dtrace_optval_t dtrace_helper_actions_max
= 1024;
125 dtrace_optval_t dtrace_helper_providers_max
= 32;
126 dtrace_optval_t dtrace_dstate_defsize
= (1 * 1024 * 1024);
127 size_t dtrace_strsize_default
= 256;
128 dtrace_optval_t dtrace_cleanrate_default
= 9900990; /* 101 hz */
129 dtrace_optval_t dtrace_cleanrate_min
= 200000; /* 5000 hz */
130 dtrace_optval_t dtrace_cleanrate_max
= (uint64_t)60 * NANOSEC
; /* 1/minute */
131 dtrace_optval_t dtrace_aggrate_default
= NANOSEC
; /* 1 hz */
132 dtrace_optval_t dtrace_statusrate_default
= NANOSEC
; /* 1 hz */
133 dtrace_optval_t dtrace_statusrate_max
= (hrtime_t
)10 * NANOSEC
; /* 6/minute */
134 dtrace_optval_t dtrace_switchrate_default
= NANOSEC
; /* 1 hz */
135 dtrace_optval_t dtrace_nspec_default
= 1;
136 dtrace_optval_t dtrace_specsize_default
= 32 * 1024;
137 dtrace_optval_t dtrace_stackframes_default
= 20;
138 dtrace_optval_t dtrace_ustackframes_default
= 20;
139 dtrace_optval_t dtrace_jstackframes_default
= 50;
140 dtrace_optval_t dtrace_jstackstrsize_default
= 512;
141 int dtrace_msgdsize_max
= 128;
142 hrtime_t dtrace_chill_max
= MSEC2NSEC(500); /* 500 ms */
143 hrtime_t dtrace_chill_interval
= NANOSEC
; /* 1000 ms */
144 int dtrace_devdepth_max
= 32;
145 int dtrace_err_verbose
;
146 hrtime_t dtrace_deadman_interval
= NANOSEC
;
147 hrtime_t dtrace_deadman_timeout
= (hrtime_t
)10 * NANOSEC
;
148 hrtime_t dtrace_deadman_user
= (hrtime_t
)30 * NANOSEC
;
149 hrtime_t dtrace_unregister_defunct_reap
= (hrtime_t
)60 * NANOSEC
;
152 * DTrace External Variables
154 * As dtrace(7D) is a kernel module, any DTrace variables are obviously
155 * available to DTrace consumers via the backtick (`) syntax. One of these,
156 * dtrace_zero, is made deliberately so: it is provided as a source of
157 * well-known, zero-filled memory. While this variable is not documented,
158 * it is used by some translators as an implementation detail.
160 const char dtrace_zero
[256] = { 0 }; /* zero-filled memory */
163 * DTrace Internal Variables
165 static dev_info_t
*dtrace_devi
; /* device info */
166 static vmem_t
*dtrace_arena
; /* probe ID arena */
167 static vmem_t
*dtrace_minor
; /* minor number arena */
168 static taskq_t
*dtrace_taskq
; /* task queue */
169 static dtrace_probe_t
**dtrace_probes
; /* array of all probes */
170 static int dtrace_nprobes
; /* number of probes */
171 static dtrace_provider_t
*dtrace_provider
; /* provider list */
172 static dtrace_meta_t
*dtrace_meta_pid
; /* user-land meta provider */
173 static int dtrace_opens
; /* number of opens */
174 static int dtrace_helpers
; /* number of helpers */
175 static int dtrace_getf
; /* number of unpriv getf()s */
176 static void *dtrace_softstate
; /* softstate pointer */
177 static dtrace_hash_t
*dtrace_bymod
; /* probes hashed by module */
178 static dtrace_hash_t
*dtrace_byfunc
; /* probes hashed by function */
179 static dtrace_hash_t
*dtrace_byname
; /* probes hashed by name */
180 static dtrace_toxrange_t
*dtrace_toxrange
; /* toxic range array */
181 static int dtrace_toxranges
; /* number of toxic ranges */
182 static int dtrace_toxranges_max
; /* size of toxic range array */
183 static dtrace_anon_t dtrace_anon
; /* anonymous enabling */
184 static kmem_cache_t
*dtrace_state_cache
; /* cache for dynamic state */
185 static uint64_t dtrace_vtime_references
; /* number of vtimestamp refs */
186 static kthread_t
*dtrace_panicked
; /* panicking thread */
187 static dtrace_ecb_t
*dtrace_ecb_create_cache
; /* cached created ECB */
188 static dtrace_genid_t dtrace_probegen
; /* current probe generation */
189 static dtrace_helpers_t
*dtrace_deferred_pid
; /* deferred helper list */
190 static dtrace_enabling_t
*dtrace_retained
; /* list of retained enablings */
191 static dtrace_genid_t dtrace_retained_gen
; /* current retained enab gen */
192 static dtrace_dynvar_t dtrace_dynhash_sink
; /* end of dynamic hash chains */
193 static int dtrace_dynvar_failclean
; /* dynvars failed to clean */
197 * DTrace is protected by three (relatively coarse-grained) locks:
199 * (1) dtrace_lock is required to manipulate essentially any DTrace state,
200 * including enabling state, probes, ECBs, consumer state, helper state,
201 * etc. Importantly, dtrace_lock is _not_ required when in probe context;
202 * probe context is lock-free -- synchronization is handled via the
203 * dtrace_sync() cross call mechanism.
205 * (2) dtrace_provider_lock is required when manipulating provider state, or
206 * when provider state must be held constant.
208 * (3) dtrace_meta_lock is required when manipulating meta provider state, or
209 * when meta provider state must be held constant.
211 * The lock ordering between these three locks is dtrace_meta_lock before
212 * dtrace_provider_lock before dtrace_lock. (In particular, there are
213 * several places where dtrace_provider_lock is held by the framework as it
214 * calls into the providers -- which then call back into the framework,
215 * grabbing dtrace_lock.)
217 * There are two other locks in the mix: mod_lock and cpu_lock. With respect
218 * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
219 * role as a coarse-grained lock; it is acquired before both of these locks.
220 * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must
221 * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
222 * mod_lock is similar with respect to dtrace_provider_lock in that it must be
223 * acquired _between_ dtrace_provider_lock and dtrace_lock.
225 static kmutex_t dtrace_lock
; /* probe state lock */
226 static kmutex_t dtrace_provider_lock
; /* provider state lock */
227 static kmutex_t dtrace_meta_lock
; /* meta-provider state lock */
230 * DTrace Provider Variables
232 * These are the variables relating to DTrace as a provider (that is, the
233 * provider of the BEGIN, END, and ERROR probes).
235 static dtrace_pattr_t dtrace_provider_attr
= {
236 { DTRACE_STABILITY_STABLE
, DTRACE_STABILITY_STABLE
, DTRACE_CLASS_COMMON
},
237 { DTRACE_STABILITY_PRIVATE
, DTRACE_STABILITY_PRIVATE
, DTRACE_CLASS_UNKNOWN
},
238 { DTRACE_STABILITY_PRIVATE
, DTRACE_STABILITY_PRIVATE
, DTRACE_CLASS_UNKNOWN
},
239 { DTRACE_STABILITY_STABLE
, DTRACE_STABILITY_STABLE
, DTRACE_CLASS_COMMON
},
240 { DTRACE_STABILITY_STABLE
, DTRACE_STABILITY_STABLE
, DTRACE_CLASS_COMMON
},
248 dtrace_enable_nullop(void)
253 static dtrace_pops_t dtrace_provider_ops
= {
254 (void (*)(void *, const dtrace_probedesc_t
*))dtrace_nullop
,
255 (void (*)(void *, struct modctl
*))dtrace_nullop
,
256 (int (*)(void *, dtrace_id_t
, void *))dtrace_enable_nullop
,
257 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
,
258 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
,
259 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
,
263 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
266 static dtrace_id_t dtrace_probeid_begin
; /* special BEGIN probe */
267 static dtrace_id_t dtrace_probeid_end
; /* special END probe */
268 dtrace_id_t dtrace_probeid_error
; /* special ERROR probe */
271 * DTrace Helper Tracing Variables
273 * These variables should be set dynamically to enable helper tracing. The
274 * only variables that should be set are dtrace_helptrace_enable (which should
275 * be set to a non-zero value to allocate helper tracing buffers on the next
276 * open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a
277 * non-zero value to deallocate helper tracing buffers on the next close of
278 * /dev/dtrace). When (and only when) helper tracing is disabled, the
279 * buffer size may also be set via dtrace_helptrace_bufsize.
281 int dtrace_helptrace_enable
= 0;
282 int dtrace_helptrace_disable
= 0;
283 int dtrace_helptrace_bufsize
= 16 * 1024 * 1024;
284 uint32_t dtrace_helptrace_nlocals
;
285 static dtrace_helptrace_t
*dtrace_helptrace_buffer
;
286 static uint32_t dtrace_helptrace_next
= 0;
287 static int dtrace_helptrace_wrapped
= 0;
290 * DTrace Error Hashing
292 * On DEBUG kernels, DTrace will track the errors that has seen in a hash
293 * table. This is very useful for checking coverage of tests that are
294 * expected to induce DIF or DOF processing errors, and may be useful for
295 * debugging problems in the DIF code generator or in DOF generation . The
296 * error hash may be examined with the ::dtrace_errhash MDB dcmd.
299 static dtrace_errhash_t dtrace_errhash
[DTRACE_ERRHASHSZ
];
300 static const char *dtrace_errlast
;
301 static kthread_t
*dtrace_errthread
;
302 static kmutex_t dtrace_errlock
;
306 * DTrace Macros and Constants
308 * These are various macros that are useful in various spots in the
309 * implementation, along with a few random constants that have no meaning
310 * outside of the implementation. There is no real structure to this cpp
311 * mishmash -- but is there ever?
313 #define DTRACE_HASHSTR(hash, probe) \
314 dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
316 #define DTRACE_HASHNEXT(hash, probe) \
317 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
319 #define DTRACE_HASHPREV(hash, probe) \
320 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
322 #define DTRACE_HASHEQ(hash, lhs, rhs) \
323 (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
324 *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
326 #define DTRACE_AGGHASHSIZE_SLEW 17
328 #define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)
331 * The key for a thread-local variable consists of the lower 61 bits of the
332 * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
333 * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
334 * equal to a variable identifier. This is necessary (but not sufficient) to
335 * assure that global associative arrays never collide with thread-local
336 * variables. To guarantee that they cannot collide, we must also define the
337 * order for keying dynamic variables. That order is:
339 * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
341 * Because the variable-key and the tls-key are in orthogonal spaces, there is
342 * no way for a global variable key signature to match a thread-local key
345 #define DTRACE_TLS_THRKEY(where) { \
347 uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
348 for (; actv; actv >>= 1) \
350 ASSERT(intr < (1 << 3)); \
351 (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
352 (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
355 #define DT_BSWAP_8(x) ((x) & 0xff)
356 #define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
357 #define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
358 #define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
360 #define DT_MASK_LO 0x00000000FFFFFFFFULL
362 #define DTRACE_STORE(type, tomax, offset, what) \
363 *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
366 #define DTRACE_ALIGNCHECK(addr, size, flags) \
367 if (addr & (size - 1)) { \
368 *flags |= CPU_DTRACE_BADALIGN; \
369 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
373 #define DTRACE_ALIGNCHECK(addr, size, flags)
377 * Test whether a range of memory starting at testaddr of size testsz falls
378 * within the range of memory described by addr, sz. We take care to avoid
379 * problems with overflow and underflow of the unsigned quantities, and
380 * disallow all negative sizes. Ranges of size 0 are allowed.
382 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
383 ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
384 (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
385 (testaddr) + (testsz) >= (testaddr))
387 #define DTRACE_RANGE_REMAIN(remp, addr, baseaddr, basesz) \
389 if ((remp) != NULL) { \
390 *(remp) = (uintptr_t)(baseaddr) + (basesz) - (addr); \
392 _NOTE(CONSTCOND) } while (0)
396 * Test whether alloc_sz bytes will fit in the scratch region. We isolate
397 * alloc_sz on the righthand side of the comparison in order to avoid overflow
398 * or underflow in the comparison with it. This is simpler than the INRANGE
399 * check above, because we know that the dtms_scratch_ptr is valid in the
400 * range. Allocations of size zero are allowed.
402 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
403 ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
404 (mstate)->dtms_scratch_ptr >= (alloc_sz))
406 #define DTRACE_LOADFUNC(bits) \
409 dtrace_load##bits(uintptr_t addr) \
411 size_t size = bits / NBBY; \
413 uint##bits##_t rval; \
415 volatile uint16_t *flags = (volatile uint16_t *) \
416 &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; \
418 DTRACE_ALIGNCHECK(addr, size, flags); \
420 for (i = 0; i < dtrace_toxranges; i++) { \
421 if (addr >= dtrace_toxrange[i].dtt_limit) \
424 if (addr + size <= dtrace_toxrange[i].dtt_base) \
428 * This address falls within a toxic region; return 0. \
430 *flags |= CPU_DTRACE_BADADDR; \
431 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
435 *flags |= CPU_DTRACE_NOFAULT; \
437 rval = *((volatile uint##bits##_t *)addr); \
438 *flags &= ~CPU_DTRACE_NOFAULT; \
440 return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \
444 #define dtrace_loadptr dtrace_load64
446 #define dtrace_loadptr dtrace_load32
449 #define DTRACE_DYNHASH_FREE 0
450 #define DTRACE_DYNHASH_SINK 1
451 #define DTRACE_DYNHASH_VALID 2
453 #define DTRACE_MATCH_FAIL -1
454 #define DTRACE_MATCH_NEXT 0
455 #define DTRACE_MATCH_DONE 1
456 #define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')
457 #define DTRACE_STATE_ALIGN 64
459 #define DTRACE_FLAGS2FLT(flags) \
460 (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \
461 ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \
462 ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \
463 ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \
464 ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \
465 ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \
466 ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \
467 ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \
468 ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \
471 #define DTRACEACT_ISSTRING(act) \
472 ((act)->dta_kind == DTRACEACT_DIFEXPR && \
473 (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
475 static size_t dtrace_strlen(const char *, size_t);
476 static dtrace_probe_t
*dtrace_probe_lookup_id(dtrace_id_t id
);
477 static void dtrace_enabling_provide(dtrace_provider_t
*);
478 static int dtrace_enabling_match(dtrace_enabling_t
*, int *);
479 static void dtrace_enabling_matchall(void);
480 static void dtrace_enabling_reap(void);
481 static dtrace_state_t
*dtrace_anon_grab(void);
482 static uint64_t dtrace_helper(int, dtrace_mstate_t
*,
483 dtrace_state_t
*, uint64_t, uint64_t);
484 static dtrace_helpers_t
*dtrace_helpers_create(proc_t
*);
485 static void dtrace_buffer_drop(dtrace_buffer_t
*);
486 static int dtrace_buffer_consumed(dtrace_buffer_t
*, hrtime_t when
);
487 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t
*, size_t, size_t,
488 dtrace_state_t
*, dtrace_mstate_t
*);
489 static int dtrace_state_option(dtrace_state_t
*, dtrace_optid_t
,
491 static int dtrace_ecb_create_enable(dtrace_probe_t
*, void *);
492 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t
*);
493 static int dtrace_priv_proc(dtrace_state_t
*, dtrace_mstate_t
*);
494 static void dtrace_getf_barrier(void);
495 static int dtrace_canload_remains(uint64_t, size_t, size_t *,
496 dtrace_mstate_t
*, dtrace_vstate_t
*);
497 static int dtrace_canstore_remains(uint64_t, size_t, size_t *,
498 dtrace_mstate_t
*, dtrace_vstate_t
*);
501 * DTrace Probe Context Functions
503 * These functions are called from probe context. Because probe context is
504 * any context in which C may be called, arbitrarily locks may be held,
505 * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
506 * As a result, functions called from probe context may only call other DTrace
507 * support functions -- they may not interact at all with the system at large.
508 * (Note that the ASSERT macro is made probe-context safe by redefining it in
509 * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
510 * loads are to be performed from probe context, they _must_ be in terms of
511 * the safe dtrace_load*() variants.
513 * Some functions in this block are not actually called from probe context;
514 * for these functions, there will be a comment above the function reading
515 * "Note: not called from probe context."
518 dtrace_panic(const char *format
, ...)
522 va_start(alist
, format
);
523 dtrace_vpanic(format
, alist
);
528 dtrace_assfail(const char *a
, const char *f
, int l
)
530 dtrace_panic("assertion failed: %s, file: %s, line: %d", a
, f
, l
);
533 * We just need something here that even the most clever compiler
534 * cannot optimize away.
536 return (a
[(uintptr_t)f
]);
540 * Atomically increment a specified error counter from probe context.
543 dtrace_error(uint32_t *counter
)
546 * Most counters stored to in probe context are per-CPU counters.
547 * However, there are some error conditions that are sufficiently
548 * arcane that they don't merit per-CPU storage. If these counters
549 * are incremented concurrently on different CPUs, scalability will be
550 * adversely affected -- but we don't expect them to be white-hot in a
551 * correctly constructed enabling...
558 if ((nval
= oval
+ 1) == 0) {
560 * If the counter would wrap, set it to 1 -- assuring
561 * that the counter is never zero when we have seen
562 * errors. (The counter must be 32-bits because we
563 * aren't guaranteed a 64-bit compare&swap operation.)
564 * To save this code both the infamy of being fingered
565 * by a priggish news story and the indignity of being
566 * the target of a neo-puritan witch trial, we're
567 * carefully avoiding any colorful description of the
568 * likelihood of this condition -- but suffice it to
569 * say that it is only slightly more likely than the
570 * overflow of predicate cache IDs, as discussed in
571 * dtrace_predicate_create().
575 } while (dtrace_cas32(counter
, oval
, nval
) != oval
);
579 * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
580 * uint8_t, a uint16_t, a uint32_t and a uint64_t.
590 dtrace_inscratch(uintptr_t dest
, size_t size
, dtrace_mstate_t
*mstate
)
592 if (dest
< mstate
->dtms_scratch_base
)
595 if (dest
+ size
< dest
)
598 if (dest
+ size
> mstate
->dtms_scratch_ptr
)
605 dtrace_canstore_statvar(uint64_t addr
, size_t sz
, size_t *remain
,
606 dtrace_statvar_t
**svars
, int nsvars
)
609 size_t maxglobalsize
, maxlocalsize
;
614 maxglobalsize
= dtrace_statvar_maxsize
+ sizeof (uint64_t);
615 maxlocalsize
= maxglobalsize
* NCPU
;
617 for (i
= 0; i
< nsvars
; i
++) {
618 dtrace_statvar_t
*svar
= svars
[i
];
622 if (svar
== NULL
|| (size
= svar
->dtsv_size
) == 0)
625 scope
= svar
->dtsv_var
.dtdv_scope
;
628 * We verify that our size is valid in the spirit of providing
629 * defense in depth: we want to prevent attackers from using
630 * DTrace to escalate an orthogonal kernel heap corruption bug
631 * into the ability to store to arbitrary locations in memory.
633 VERIFY((scope
== DIFV_SCOPE_GLOBAL
&& size
<= maxglobalsize
) ||
634 (scope
== DIFV_SCOPE_LOCAL
&& size
<= maxlocalsize
));
636 if (DTRACE_INRANGE(addr
, sz
, svar
->dtsv_data
,
638 DTRACE_RANGE_REMAIN(remain
, addr
, svar
->dtsv_data
,
648 * Check to see if the address is within a memory region to which a store may
649 * be issued. This includes the DTrace scratch areas, and any DTrace variable
650 * region. The caller of dtrace_canstore() is responsible for performing any
651 * alignment checks that are needed before stores are actually executed.
654 dtrace_canstore(uint64_t addr
, size_t sz
, dtrace_mstate_t
*mstate
,
655 dtrace_vstate_t
*vstate
)
657 return (dtrace_canstore_remains(addr
, sz
, NULL
, mstate
, vstate
));
661 * Implementation of dtrace_canstore which communicates the upper bound of the
662 * allowed memory region.
665 dtrace_canstore_remains(uint64_t addr
, size_t sz
, size_t *remain
,
666 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
)
669 * First, check to see if the address is in scratch space...
671 if (DTRACE_INRANGE(addr
, sz
, mstate
->dtms_scratch_base
,
672 mstate
->dtms_scratch_size
)) {
673 DTRACE_RANGE_REMAIN(remain
, addr
, mstate
->dtms_scratch_base
,
674 mstate
->dtms_scratch_size
);
679 * Now check to see if it's a dynamic variable. This check will pick
680 * up both thread-local variables and any global dynamically-allocated
683 if (DTRACE_INRANGE(addr
, sz
, vstate
->dtvs_dynvars
.dtds_base
,
684 vstate
->dtvs_dynvars
.dtds_size
)) {
685 dtrace_dstate_t
*dstate
= &vstate
->dtvs_dynvars
;
686 uintptr_t base
= (uintptr_t)dstate
->dtds_base
+
687 (dstate
->dtds_hashsize
* sizeof (dtrace_dynhash_t
));
689 dtrace_dynvar_t
*dvar
;
692 * Before we assume that we can store here, we need to make
693 * sure that it isn't in our metadata -- storing to our
694 * dynamic variable metadata would corrupt our state. For
695 * the range to not include any dynamic variable metadata,
698 * (1) Start above the hash table that is at the base of
699 * the dynamic variable space
701 * (2) Have a starting chunk offset that is beyond the
702 * dtrace_dynvar_t that is at the base of every chunk
704 * (3) Not span a chunk boundary
706 * (4) Not be in the tuple space of a dynamic variable
712 chunkoffs
= (addr
- base
) % dstate
->dtds_chunksize
;
714 if (chunkoffs
< sizeof (dtrace_dynvar_t
))
717 if (chunkoffs
+ sz
> dstate
->dtds_chunksize
)
720 dvar
= (dtrace_dynvar_t
*)((uintptr_t)addr
- chunkoffs
);
722 if (dvar
->dtdv_hashval
== DTRACE_DYNHASH_FREE
)
725 if (chunkoffs
< sizeof (dtrace_dynvar_t
) +
726 ((dvar
->dtdv_tuple
.dtt_nkeys
- 1) * sizeof (dtrace_key_t
)))
729 DTRACE_RANGE_REMAIN(remain
, addr
, dvar
, dstate
->dtds_chunksize
);
734 * Finally, check the static local and global variables. These checks
735 * take the longest, so we perform them last.
737 if (dtrace_canstore_statvar(addr
, sz
, remain
,
738 vstate
->dtvs_locals
, vstate
->dtvs_nlocals
))
741 if (dtrace_canstore_statvar(addr
, sz
, remain
,
742 vstate
->dtvs_globals
, vstate
->dtvs_nglobals
))
750 * Convenience routine to check to see if the address is within a memory
751 * region in which a load may be issued given the user's privilege level;
752 * if not, it sets the appropriate error flags and loads 'addr' into the
753 * illegal value slot.
755 * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
756 * appropriate memory access protection.
759 dtrace_canload(uint64_t addr
, size_t sz
, dtrace_mstate_t
*mstate
,
760 dtrace_vstate_t
*vstate
)
762 return (dtrace_canload_remains(addr
, sz
, NULL
, mstate
, vstate
));
766 * Implementation of dtrace_canload which communicates the upper bound of the
767 * allowed memory region.
770 dtrace_canload_remains(uint64_t addr
, size_t sz
, size_t *remain
,
771 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
)
773 volatile uintptr_t *illval
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
;
777 * If we hold the privilege to read from kernel memory, then
778 * everything is readable.
780 if ((mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) != 0) {
781 DTRACE_RANGE_REMAIN(remain
, addr
, addr
, sz
);
786 * You can obviously read that which you can store.
788 if (dtrace_canstore_remains(addr
, sz
, remain
, mstate
, vstate
))
792 * We're allowed to read from our own string table.
794 if (DTRACE_INRANGE(addr
, sz
, mstate
->dtms_difo
->dtdo_strtab
,
795 mstate
->dtms_difo
->dtdo_strlen
)) {
796 DTRACE_RANGE_REMAIN(remain
, addr
,
797 mstate
->dtms_difo
->dtdo_strtab
,
798 mstate
->dtms_difo
->dtdo_strlen
);
802 if (vstate
->dtvs_state
!= NULL
&&
803 dtrace_priv_proc(vstate
->dtvs_state
, mstate
)) {
807 * When we have privileges to the current process, there are
808 * several context-related kernel structures that are safe to
809 * read, even absent the privilege to read from kernel memory.
810 * These reads are safe because these structures contain only
811 * state that (1) we're permitted to read, (2) is harmless or
812 * (3) contains pointers to additional kernel state that we're
813 * not permitted to read (and as such, do not present an
814 * opportunity for privilege escalation). Finally (and
815 * critically), because of the nature of their relation with
816 * the current thread context, the memory associated with these
817 * structures cannot change over the duration of probe context,
818 * and it is therefore impossible for this memory to be
819 * deallocated and reallocated as something else while it's
820 * being operated upon.
822 if (DTRACE_INRANGE(addr
, sz
, curthread
, sizeof (kthread_t
))) {
823 DTRACE_RANGE_REMAIN(remain
, addr
, curthread
,
828 if ((p
= curthread
->t_procp
) != NULL
&& DTRACE_INRANGE(addr
,
829 sz
, curthread
->t_procp
, sizeof (proc_t
))) {
830 DTRACE_RANGE_REMAIN(remain
, addr
, curthread
->t_procp
,
835 if (curthread
->t_cred
!= NULL
&& DTRACE_INRANGE(addr
, sz
,
836 curthread
->t_cred
, sizeof (cred_t
))) {
837 DTRACE_RANGE_REMAIN(remain
, addr
, curthread
->t_cred
,
842 if (p
!= NULL
&& p
->p_pidp
!= NULL
&& DTRACE_INRANGE(addr
, sz
,
843 &(p
->p_pidp
->pid_id
), sizeof (pid_t
))) {
844 DTRACE_RANGE_REMAIN(remain
, addr
, &(p
->p_pidp
->pid_id
),
849 if (curthread
->t_cpu
!= NULL
&& DTRACE_INRANGE(addr
, sz
,
850 curthread
->t_cpu
, offsetof(cpu_t
, cpu_pause_thread
))) {
851 DTRACE_RANGE_REMAIN(remain
, addr
, curthread
->t_cpu
,
852 offsetof(cpu_t
, cpu_pause_thread
));
857 if ((fp
= mstate
->dtms_getf
) != NULL
) {
858 uintptr_t psz
= sizeof (void *);
863 * When getf() returns a file_t, the enabling is implicitly
864 * granted the (transient) right to read the returned file_t
865 * as well as the v_path and v_op->vnop_name of the underlying
866 * vnode. These accesses are allowed after a successful
867 * getf() because the members that they refer to cannot change
868 * once set -- and the barrier logic in the kernel's closef()
869 * path assures that the file_t and its referenced vode_t
870 * cannot themselves be stale (that is, it impossible for
871 * either dtms_getf itself or its f_vnode member to reference
874 if (DTRACE_INRANGE(addr
, sz
, fp
, sizeof (file_t
))) {
875 DTRACE_RANGE_REMAIN(remain
, addr
, fp
, sizeof (file_t
));
879 if ((vp
= fp
->f_vnode
) != NULL
) {
882 if (DTRACE_INRANGE(addr
, sz
, &vp
->v_path
, psz
)) {
883 DTRACE_RANGE_REMAIN(remain
, addr
, &vp
->v_path
,
888 slen
= strlen(vp
->v_path
) + 1;
889 if (DTRACE_INRANGE(addr
, sz
, vp
->v_path
, slen
)) {
890 DTRACE_RANGE_REMAIN(remain
, addr
, vp
->v_path
,
895 if (DTRACE_INRANGE(addr
, sz
, &vp
->v_op
, psz
)) {
896 DTRACE_RANGE_REMAIN(remain
, addr
, &vp
->v_op
,
901 if ((op
= vp
->v_op
) != NULL
&&
902 DTRACE_INRANGE(addr
, sz
, &op
->vnop_name
, psz
)) {
903 DTRACE_RANGE_REMAIN(remain
, addr
,
904 &op
->vnop_name
, psz
);
908 if (op
!= NULL
&& op
->vnop_name
!= NULL
&&
909 DTRACE_INRANGE(addr
, sz
, op
->vnop_name
,
910 (slen
= strlen(op
->vnop_name
) + 1))) {
911 DTRACE_RANGE_REMAIN(remain
, addr
,
912 op
->vnop_name
, slen
);
918 DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV
);
924 * Convenience routine to check to see if a given string is within a memory
925 * region in which a load may be issued given the user's privilege level;
926 * this exists so that we don't need to issue unnecessary dtrace_strlen()
927 * calls in the event that the user has all privileges.
930 dtrace_strcanload(uint64_t addr
, size_t sz
, size_t *remain
,
931 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
)
936 * If we hold the privilege to read from kernel memory, then
937 * everything is readable.
939 if ((mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) != 0) {
940 DTRACE_RANGE_REMAIN(remain
, addr
, addr
, sz
);
945 * Even if the caller is uninterested in querying the remaining valid
946 * range, it is required to ensure that the access is allowed.
948 if (remain
== NULL
) {
951 if (dtrace_canload_remains(addr
, 0, remain
, mstate
, vstate
)) {
954 * Perform the strlen after determining the length of the
955 * memory region which is accessible. This prevents timing
956 * information from being used to find NULs in memory which is
957 * not accessible to the caller.
959 strsz
= 1 + dtrace_strlen((char *)(uintptr_t)addr
,
961 if (strsz
<= *remain
) {
970 * Convenience routine to check to see if a given variable is within a memory
971 * region in which a load may be issued given the user's privilege level.
974 dtrace_vcanload(void *src
, dtrace_diftype_t
*type
, size_t *remain
,
975 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
)
978 ASSERT(type
->dtdt_flags
& DIF_TF_BYREF
);
981 * Calculate the max size before performing any checks since even
982 * DTRACE_ACCESS_KERNEL-credentialed callers expect that this function
983 * return the max length via 'remain'.
985 if (type
->dtdt_kind
== DIF_TYPE_STRING
) {
986 dtrace_state_t
*state
= vstate
->dtvs_state
;
989 sz
= state
->dts_options
[DTRACEOPT_STRSIZE
];
992 * In helper context, we have a NULL state; fall back
993 * to using the system-wide default for the string size
996 sz
= dtrace_strsize_default
;
999 sz
= type
->dtdt_size
;
1003 * If we hold the privilege to read from kernel memory, then
1004 * everything is readable.
1006 if ((mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) != 0) {
1007 DTRACE_RANGE_REMAIN(remain
, (uintptr_t)src
, src
, sz
);
1011 if (type
->dtdt_kind
== DIF_TYPE_STRING
) {
1012 return (dtrace_strcanload((uintptr_t)src
, sz
, remain
, mstate
,
1015 return (dtrace_canload_remains((uintptr_t)src
, sz
, remain
, mstate
,
1020 * Convert a string to a signed integer using safe loads.
1022 * NOTE: This function uses various macros from strtolctype.h to manipulate
1023 * digit values, etc -- these have all been checked to ensure they make
1024 * no additional function calls.
1027 dtrace_strtoll(char *input
, int base
, size_t limit
)
1029 uintptr_t pos
= (uintptr_t)input
;
1032 boolean_t neg
= B_FALSE
;
1034 uintptr_t end
= pos
+ limit
;
1037 * Consume any whitespace preceding digits.
1039 while ((c
= dtrace_load8(pos
)) == ' ' || c
== '\t')
1043 * Handle an explicit sign if one is present.
1045 if (c
== '-' || c
== '+') {
1048 c
= dtrace_load8(++pos
);
1052 * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it
1055 if (base
== 16 && c
== '0' && ((cc
= dtrace_load8(pos
+ 1)) == 'x' ||
1056 cc
== 'X') && isxdigit(ccc
= dtrace_load8(pos
+ 2))) {
1062 * Read in contiguous digits until the first non-digit character.
1064 for (; pos
< end
&& c
!= '\0' && lisalnum(c
) && (x
= DIGIT(c
)) < base
;
1065 c
= dtrace_load8(++pos
))
1066 val
= val
* base
+ x
;
1068 return (neg
? -val
: val
);
1072 * Compare two strings using safe loads.
1075 dtrace_strncmp(char *s1
, char *s2
, size_t limit
)
1078 volatile uint16_t *flags
;
1080 if (s1
== s2
|| limit
== 0)
1083 flags
= (volatile uint16_t *)&cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
1089 c1
= dtrace_load8((uintptr_t)s1
++);
1095 c2
= dtrace_load8((uintptr_t)s2
++);
1100 } while (--limit
&& c1
!= '\0' && !(*flags
& CPU_DTRACE_FAULT
));
1106 * Compute strlen(s) for a string using safe memory accesses. The additional
1107 * len parameter is used to specify a maximum length to ensure completion.
1110 dtrace_strlen(const char *s
, size_t lim
)
1114 for (len
= 0; len
!= lim
; len
++) {
1115 if (dtrace_load8((uintptr_t)s
++) == '\0')
1123 * Check if an address falls within a toxic region.
1126 dtrace_istoxic(uintptr_t kaddr
, size_t size
)
1128 uintptr_t taddr
, tsize
;
1131 for (i
= 0; i
< dtrace_toxranges
; i
++) {
1132 taddr
= dtrace_toxrange
[i
].dtt_base
;
1133 tsize
= dtrace_toxrange
[i
].dtt_limit
- taddr
;
1135 if (kaddr
- taddr
< tsize
) {
1136 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR
);
1137 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
= kaddr
;
1141 if (taddr
- kaddr
< size
) {
1142 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR
);
1143 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
= taddr
;
1152 * Copy src to dst using safe memory accesses. The src is assumed to be unsafe
1153 * memory specified by the DIF program. The dst is assumed to be safe memory
1154 * that we can store to directly because it is managed by DTrace. As with
1155 * standard bcopy, overlapping copies are handled properly.
1158 dtrace_bcopy(const void *src
, void *dst
, size_t len
)
1162 const uint8_t *s2
= src
;
1166 *s1
++ = dtrace_load8((uintptr_t)s2
++);
1167 } while (--len
!= 0);
1173 *--s1
= dtrace_load8((uintptr_t)--s2
);
1174 } while (--len
!= 0);
1180 * Copy src to dst using safe memory accesses, up to either the specified
1181 * length, or the point that a nul byte is encountered. The src is assumed to
1182 * be unsafe memory specified by the DIF program. The dst is assumed to be
1183 * safe memory that we can store to directly because it is managed by DTrace.
1184 * Unlike dtrace_bcopy(), overlapping regions are not handled.
1187 dtrace_strcpy(const void *src
, void *dst
, size_t len
)
1190 uint8_t *s1
= dst
, c
;
1191 const uint8_t *s2
= src
;
1194 *s1
++ = c
= dtrace_load8((uintptr_t)s2
++);
1195 } while (--len
!= 0 && c
!= '\0');
1200 * Copy src to dst, deriving the size and type from the specified (BYREF)
1201 * variable type. The src is assumed to be unsafe memory specified by the DIF
1202 * program. The dst is assumed to be DTrace variable memory that is of the
1203 * specified type; we assume that we can store to directly.
1206 dtrace_vcopy(void *src
, void *dst
, dtrace_diftype_t
*type
, size_t limit
)
1208 ASSERT(type
->dtdt_flags
& DIF_TF_BYREF
);
1210 if (type
->dtdt_kind
== DIF_TYPE_STRING
) {
1211 dtrace_strcpy(src
, dst
, MIN(type
->dtdt_size
, limit
));
1213 dtrace_bcopy(src
, dst
, MIN(type
->dtdt_size
, limit
));
1218 * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be
1219 * unsafe memory specified by the DIF program. The s2 data is assumed to be
1220 * safe memory that we can access directly because it is managed by DTrace.
1223 dtrace_bcmp(const void *s1
, const void *s2
, size_t len
)
1225 volatile uint16_t *flags
;
1227 flags
= (volatile uint16_t *)&cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
1232 if (s1
== NULL
|| s2
== NULL
)
1235 if (s1
!= s2
&& len
!= 0) {
1236 const uint8_t *ps1
= s1
;
1237 const uint8_t *ps2
= s2
;
1240 if (dtrace_load8((uintptr_t)ps1
++) != *ps2
++)
1242 } while (--len
!= 0 && !(*flags
& CPU_DTRACE_FAULT
));
1248 * Zero the specified region using a simple byte-by-byte loop. Note that this
1249 * is for safe DTrace-managed memory only.
1252 dtrace_bzero(void *dst
, size_t len
)
1256 for (cp
= dst
; len
!= 0; len
--)
1261 dtrace_add_128(uint64_t *addend1
, uint64_t *addend2
, uint64_t *sum
)
1265 result
[0] = addend1
[0] + addend2
[0];
1266 result
[1] = addend1
[1] + addend2
[1] +
1267 (result
[0] < addend1
[0] || result
[0] < addend2
[0] ? 1 : 0);
1274 * Shift the 128-bit value in a by b. If b is positive, shift left.
1275 * If b is negative, shift right.
1278 dtrace_shift_128(uint64_t *a
, int b
)
1288 a
[0] = a
[1] >> (b
- 64);
1292 mask
= 1LL << (64 - b
);
1294 a
[0] |= ((a
[1] & mask
) << (64 - b
));
1299 a
[1] = a
[0] << (b
- 64);
1303 mask
= a
[0] >> (64 - b
);
1311 * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1312 * use native multiplication on those, and then re-combine into the
1313 * resulting 128-bit value.
1315 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1322 dtrace_multiply_128(uint64_t factor1
, uint64_t factor2
, uint64_t *product
)
1324 uint64_t hi1
, hi2
, lo1
, lo2
;
1327 hi1
= factor1
>> 32;
1328 hi2
= factor2
>> 32;
1330 lo1
= factor1
& DT_MASK_LO
;
1331 lo2
= factor2
& DT_MASK_LO
;
1333 product
[0] = lo1
* lo2
;
1334 product
[1] = hi1
* hi2
;
1338 dtrace_shift_128(tmp
, 32);
1339 dtrace_add_128(product
, tmp
, product
);
1343 dtrace_shift_128(tmp
, 32);
1344 dtrace_add_128(product
, tmp
, product
);
1348 * This privilege check should be used by actions and subroutines to
1349 * verify that the user credentials of the process that enabled the
1350 * invoking ECB match the target credentials
1353 dtrace_priv_proc_common_user(dtrace_state_t
*state
)
1355 cred_t
*cr
, *s_cr
= state
->dts_cred
.dcr_cred
;
1358 * We should always have a non-NULL state cred here, since if cred
1359 * is null (anonymous tracing), we fast-path bypass this routine.
1361 ASSERT(s_cr
!= NULL
);
1363 if ((cr
= CRED()) != NULL
&&
1364 s_cr
->cr_uid
== cr
->cr_uid
&&
1365 s_cr
->cr_uid
== cr
->cr_ruid
&&
1366 s_cr
->cr_uid
== cr
->cr_suid
&&
1367 s_cr
->cr_gid
== cr
->cr_gid
&&
1368 s_cr
->cr_gid
== cr
->cr_rgid
&&
1369 s_cr
->cr_gid
== cr
->cr_sgid
)
1376 * This privilege check should be used by actions and subroutines to
1377 * verify that the zone of the process that enabled the invoking ECB
1378 * matches the target credentials
1381 dtrace_priv_proc_common_zone(dtrace_state_t
*state
)
1383 cred_t
*cr
, *s_cr
= state
->dts_cred
.dcr_cred
;
1386 * We should always have a non-NULL state cred here, since if cred
1387 * is null (anonymous tracing), we fast-path bypass this routine.
1389 ASSERT(s_cr
!= NULL
);
1391 if ((cr
= CRED()) != NULL
&& s_cr
->cr_zone
== cr
->cr_zone
)
1398 * This privilege check should be used by actions and subroutines to
1399 * verify that the process has not setuid or changed credentials.
1402 dtrace_priv_proc_common_nocd()
1406 if ((proc
= ttoproc(curthread
)) != NULL
&&
1407 !(proc
->p_flag
& SNOCD
))
1414 dtrace_priv_proc_destructive(dtrace_state_t
*state
, dtrace_mstate_t
*mstate
)
1416 int action
= state
->dts_cred
.dcr_action
;
1418 if (!(mstate
->dtms_access
& DTRACE_ACCESS_PROC
))
1421 if (((action
& DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE
) == 0) &&
1422 dtrace_priv_proc_common_zone(state
) == 0)
1425 if (((action
& DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER
) == 0) &&
1426 dtrace_priv_proc_common_user(state
) == 0)
1429 if (((action
& DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG
) == 0) &&
1430 dtrace_priv_proc_common_nocd() == 0)
1436 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|= CPU_DTRACE_UPRIV
;
1442 dtrace_priv_proc_control(dtrace_state_t
*state
, dtrace_mstate_t
*mstate
)
1444 if (mstate
->dtms_access
& DTRACE_ACCESS_PROC
) {
1445 if (state
->dts_cred
.dcr_action
& DTRACE_CRA_PROC_CONTROL
)
1448 if (dtrace_priv_proc_common_zone(state
) &&
1449 dtrace_priv_proc_common_user(state
) &&
1450 dtrace_priv_proc_common_nocd())
1454 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|= CPU_DTRACE_UPRIV
;
1460 dtrace_priv_proc(dtrace_state_t
*state
, dtrace_mstate_t
*mstate
)
1462 if ((mstate
->dtms_access
& DTRACE_ACCESS_PROC
) &&
1463 (state
->dts_cred
.dcr_action
& DTRACE_CRA_PROC
))
1466 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|= CPU_DTRACE_UPRIV
;
1472 dtrace_priv_kernel(dtrace_state_t
*state
)
1474 if (state
->dts_cred
.dcr_action
& DTRACE_CRA_KERNEL
)
1477 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|= CPU_DTRACE_KPRIV
;
1483 dtrace_priv_kernel_destructive(dtrace_state_t
*state
)
1485 if (state
->dts_cred
.dcr_action
& DTRACE_CRA_KERNEL_DESTRUCTIVE
)
1488 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|= CPU_DTRACE_KPRIV
;
1494 * Determine if the dte_cond of the specified ECB allows for processing of
1495 * the current probe to continue. Note that this routine may allow continued
1496 * processing, but with access(es) stripped from the mstate's dtms_access
1500 dtrace_priv_probe(dtrace_state_t
*state
, dtrace_mstate_t
*mstate
,
1503 dtrace_probe_t
*probe
= ecb
->dte_probe
;
1504 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
1505 dtrace_pops_t
*pops
= &prov
->dtpv_pops
;
1506 int mode
= DTRACE_MODE_NOPRIV_DROP
;
1508 ASSERT(ecb
->dte_cond
);
1510 if (pops
->dtps_mode
!= NULL
) {
1511 mode
= pops
->dtps_mode(prov
->dtpv_arg
,
1512 probe
->dtpr_id
, probe
->dtpr_arg
);
1514 ASSERT(mode
& (DTRACE_MODE_USER
| DTRACE_MODE_KERNEL
));
1515 ASSERT(mode
& (DTRACE_MODE_NOPRIV_RESTRICT
|
1516 DTRACE_MODE_NOPRIV_DROP
));
1520 * If the dte_cond bits indicate that this consumer is only allowed to
1521 * see user-mode firings of this probe, check that the probe was fired
1522 * while in a user context. If that's not the case, use the policy
1523 * specified by the provider to determine if we drop the probe or
1524 * merely restrict operation.
1526 if (ecb
->dte_cond
& DTRACE_COND_USERMODE
) {
1527 ASSERT(mode
!= DTRACE_MODE_NOPRIV_DROP
);
1529 if (!(mode
& DTRACE_MODE_USER
)) {
1530 if (mode
& DTRACE_MODE_NOPRIV_DROP
)
1533 mstate
->dtms_access
&= ~DTRACE_ACCESS_ARGS
;
1538 * This is more subtle than it looks. We have to be absolutely certain
1539 * that CRED() isn't going to change out from under us so it's only
1540 * legit to examine that structure if we're in constrained situations.
1541 * Currently, the only times we'll this check is if a non-super-user
1542 * has enabled the profile or syscall providers -- providers that
1543 * allow visibility of all processes. For the profile case, the check
1544 * above will ensure that we're examining a user context.
1546 if (ecb
->dte_cond
& DTRACE_COND_OWNER
) {
1548 cred_t
*s_cr
= state
->dts_cred
.dcr_cred
;
1551 ASSERT(s_cr
!= NULL
);
1553 if ((cr
= CRED()) == NULL
||
1554 s_cr
->cr_uid
!= cr
->cr_uid
||
1555 s_cr
->cr_uid
!= cr
->cr_ruid
||
1556 s_cr
->cr_uid
!= cr
->cr_suid
||
1557 s_cr
->cr_gid
!= cr
->cr_gid
||
1558 s_cr
->cr_gid
!= cr
->cr_rgid
||
1559 s_cr
->cr_gid
!= cr
->cr_sgid
||
1560 (proc
= ttoproc(curthread
)) == NULL
||
1561 (proc
->p_flag
& SNOCD
)) {
1562 if (mode
& DTRACE_MODE_NOPRIV_DROP
)
1565 mstate
->dtms_access
&= ~DTRACE_ACCESS_PROC
;
1570 * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1571 * in our zone, check to see if our mode policy is to restrict rather
1572 * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1573 * and DTRACE_ACCESS_ARGS
1575 if (ecb
->dte_cond
& DTRACE_COND_ZONEOWNER
) {
1577 cred_t
*s_cr
= state
->dts_cred
.dcr_cred
;
1579 ASSERT(s_cr
!= NULL
);
1581 if ((cr
= CRED()) == NULL
||
1582 s_cr
->cr_zone
->zone_id
!= cr
->cr_zone
->zone_id
) {
1583 if (mode
& DTRACE_MODE_NOPRIV_DROP
)
1586 mstate
->dtms_access
&=
1587 ~(DTRACE_ACCESS_PROC
| DTRACE_ACCESS_ARGS
);
1592 * By merits of being in this code path at all, we have limited
1593 * privileges. If the provider has indicated that limited privileges
1594 * are to denote restricted operation, strip off the ability to access
1597 if (mode
& DTRACE_MODE_LIMITEDPRIV_RESTRICT
)
1598 mstate
->dtms_access
&= ~DTRACE_ACCESS_ARGS
;
1604 * Note: not called from probe context. This function is called
1605 * asynchronously (and at a regular interval) from outside of probe context to
1606 * clean the dirty dynamic variable lists on all CPUs. Dynamic variable
1607 * cleaning is explained in detail in <sys/dtrace_impl.h>.
1610 dtrace_dynvar_clean(dtrace_dstate_t
*dstate
)
1612 dtrace_dynvar_t
*dirty
;
1613 dtrace_dstate_percpu_t
*dcpu
;
1614 dtrace_dynvar_t
**rinsep
;
1617 for (i
= 0; i
< NCPU
; i
++) {
1618 dcpu
= &dstate
->dtds_percpu
[i
];
1619 rinsep
= &dcpu
->dtdsc_rinsing
;
1622 * If the dirty list is NULL, there is no dirty work to do.
1624 if (dcpu
->dtdsc_dirty
== NULL
)
1627 if (dcpu
->dtdsc_rinsing
!= NULL
) {
1629 * If the rinsing list is non-NULL, then it is because
1630 * this CPU was selected to accept another CPU's
1631 * dirty list -- and since that time, dirty buffers
1632 * have accumulated. This is a highly unlikely
1633 * condition, but we choose to ignore the dirty
1634 * buffers -- they'll be picked up a future cleanse.
1639 if (dcpu
->dtdsc_clean
!= NULL
) {
1641 * If the clean list is non-NULL, then we're in a
1642 * situation where a CPU has done deallocations (we
1643 * have a non-NULL dirty list) but no allocations (we
1644 * also have a non-NULL clean list). We can't simply
1645 * move the dirty list into the clean list on this
1646 * CPU, yet we also don't want to allow this condition
1647 * to persist, lest a short clean list prevent a
1648 * massive dirty list from being cleaned (which in
1649 * turn could lead to otherwise avoidable dynamic
1650 * drops). To deal with this, we look for some CPU
1651 * with a NULL clean list, NULL dirty list, and NULL
1652 * rinsing list -- and then we borrow this CPU to
1653 * rinse our dirty list.
1655 for (j
= 0; j
< NCPU
; j
++) {
1656 dtrace_dstate_percpu_t
*rinser
;
1658 rinser
= &dstate
->dtds_percpu
[j
];
1660 if (rinser
->dtdsc_rinsing
!= NULL
)
1663 if (rinser
->dtdsc_dirty
!= NULL
)
1666 if (rinser
->dtdsc_clean
!= NULL
)
1669 rinsep
= &rinser
->dtdsc_rinsing
;
1675 * We were unable to find another CPU that
1676 * could accept this dirty list -- we are
1677 * therefore unable to clean it now.
1679 dtrace_dynvar_failclean
++;
1687 * Atomically move the dirty list aside.
1690 dirty
= dcpu
->dtdsc_dirty
;
1693 * Before we zap the dirty list, set the rinsing list.
1694 * (This allows for a potential assertion in
1695 * dtrace_dynvar(): if a free dynamic variable appears
1696 * on a hash chain, either the dirty list or the
1697 * rinsing list for some CPU must be non-NULL.)
1700 dtrace_membar_producer();
1701 } while (dtrace_casptr(&dcpu
->dtdsc_dirty
,
1702 dirty
, NULL
) != dirty
);
1707 * We have no work to do; we can simply return.
1714 for (i
= 0; i
< NCPU
; i
++) {
1715 dcpu
= &dstate
->dtds_percpu
[i
];
1717 if (dcpu
->dtdsc_rinsing
== NULL
)
1721 * We are now guaranteed that no hash chain contains a pointer
1722 * into this dirty list; we can make it clean.
1724 ASSERT(dcpu
->dtdsc_clean
== NULL
);
1725 dcpu
->dtdsc_clean
= dcpu
->dtdsc_rinsing
;
1726 dcpu
->dtdsc_rinsing
= NULL
;
1730 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1731 * sure that all CPUs have seen all of the dtdsc_clean pointers.
1732 * This prevents a race whereby a CPU incorrectly decides that
1733 * the state should be something other than DTRACE_DSTATE_CLEAN
1734 * after dtrace_dynvar_clean() has completed.
1738 dstate
->dtds_state
= DTRACE_DSTATE_CLEAN
;
1742 * Depending on the value of the op parameter, this function looks-up,
1743 * allocates or deallocates an arbitrarily-keyed dynamic variable. If an
1744 * allocation is requested, this function will return a pointer to a
1745 * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1746 * variable can be allocated. If NULL is returned, the appropriate counter
1747 * will be incremented.
1750 dtrace_dynvar(dtrace_dstate_t
*dstate
, uint_t nkeys
,
1751 dtrace_key_t
*key
, size_t dsize
, dtrace_dynvar_op_t op
,
1752 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
)
1754 uint64_t hashval
= DTRACE_DYNHASH_VALID
;
1755 dtrace_dynhash_t
*hash
= dstate
->dtds_hash
;
1756 dtrace_dynvar_t
*free
, *new_free
, *next
, *dvar
, *start
, *prev
= NULL
;
1757 processorid_t me
= CPU
->cpu_id
, cpu
= me
;
1758 dtrace_dstate_percpu_t
*dcpu
= &dstate
->dtds_percpu
[me
];
1759 size_t bucket
, ksize
;
1760 size_t chunksize
= dstate
->dtds_chunksize
;
1761 uintptr_t kdata
, lock
, nstate
;
1767 * Hash the key. As with aggregations, we use Jenkins' "One-at-a-time"
1768 * algorithm. For the by-value portions, we perform the algorithm in
1769 * 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a
1770 * bit, and seems to have only a minute effect on distribution. For
1771 * the by-reference data, we perform "One-at-a-time" iterating (safely)
1772 * over each referenced byte. It's painful to do this, but it's much
1773 * better than pathological hash distribution. The efficacy of the
1774 * hashing algorithm (and a comparison with other algorithms) may be
1775 * found by running the ::dtrace_dynstat MDB dcmd.
1777 for (i
= 0; i
< nkeys
; i
++) {
1778 if (key
[i
].dttk_size
== 0) {
1779 uint64_t val
= key
[i
].dttk_value
;
1781 hashval
+= (val
>> 48) & 0xffff;
1782 hashval
+= (hashval
<< 10);
1783 hashval
^= (hashval
>> 6);
1785 hashval
+= (val
>> 32) & 0xffff;
1786 hashval
+= (hashval
<< 10);
1787 hashval
^= (hashval
>> 6);
1789 hashval
+= (val
>> 16) & 0xffff;
1790 hashval
+= (hashval
<< 10);
1791 hashval
^= (hashval
>> 6);
1793 hashval
+= val
& 0xffff;
1794 hashval
+= (hashval
<< 10);
1795 hashval
^= (hashval
>> 6);
1798 * This is incredibly painful, but it beats the hell
1799 * out of the alternative.
1801 uint64_t j
, size
= key
[i
].dttk_size
;
1802 uintptr_t base
= (uintptr_t)key
[i
].dttk_value
;
1804 if (!dtrace_canload(base
, size
, mstate
, vstate
))
1807 for (j
= 0; j
< size
; j
++) {
1808 hashval
+= dtrace_load8(base
+ j
);
1809 hashval
+= (hashval
<< 10);
1810 hashval
^= (hashval
>> 6);
1815 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT
))
1818 hashval
+= (hashval
<< 3);
1819 hashval
^= (hashval
>> 11);
1820 hashval
+= (hashval
<< 15);
1823 * There is a remote chance (ideally, 1 in 2^31) that our hashval
1824 * comes out to be one of our two sentinel hash values. If this
1825 * actually happens, we set the hashval to be a value known to be a
1826 * non-sentinel value.
1828 if (hashval
== DTRACE_DYNHASH_FREE
|| hashval
== DTRACE_DYNHASH_SINK
)
1829 hashval
= DTRACE_DYNHASH_VALID
;
1832 * Yes, it's painful to do a divide here. If the cycle count becomes
1833 * important here, tricks can be pulled to reduce it. (However, it's
1834 * critical that hash collisions be kept to an absolute minimum;
1835 * they're much more painful than a divide.) It's better to have a
1836 * solution that generates few collisions and still keeps things
1837 * relatively simple.
1839 bucket
= hashval
% dstate
->dtds_hashsize
;
1841 if (op
== DTRACE_DYNVAR_DEALLOC
) {
1842 volatile uintptr_t *lockp
= &hash
[bucket
].dtdh_lock
;
1845 while ((lock
= *lockp
) & 1)
1848 if (dtrace_casptr((void *)lockp
,
1849 (void *)lock
, (void *)(lock
+ 1)) == (void *)lock
)
1853 dtrace_membar_producer();
1858 lock
= hash
[bucket
].dtdh_lock
;
1860 dtrace_membar_consumer();
1862 start
= hash
[bucket
].dtdh_chain
;
1863 ASSERT(start
!= NULL
&& (start
->dtdv_hashval
== DTRACE_DYNHASH_SINK
||
1864 start
->dtdv_hashval
!= DTRACE_DYNHASH_FREE
||
1865 op
!= DTRACE_DYNVAR_DEALLOC
));
1867 for (dvar
= start
; dvar
!= NULL
; dvar
= dvar
->dtdv_next
) {
1868 dtrace_tuple_t
*dtuple
= &dvar
->dtdv_tuple
;
1869 dtrace_key_t
*dkey
= &dtuple
->dtt_key
[0];
1871 if (dvar
->dtdv_hashval
!= hashval
) {
1872 if (dvar
->dtdv_hashval
== DTRACE_DYNHASH_SINK
) {
1874 * We've reached the sink, and therefore the
1875 * end of the hash chain; we can kick out of
1876 * the loop knowing that we have seen a valid
1877 * snapshot of state.
1879 ASSERT(dvar
->dtdv_next
== NULL
);
1880 ASSERT(dvar
== &dtrace_dynhash_sink
);
1884 if (dvar
->dtdv_hashval
== DTRACE_DYNHASH_FREE
) {
1886 * We've gone off the rails: somewhere along
1887 * the line, one of the members of this hash
1888 * chain was deleted. Note that we could also
1889 * detect this by simply letting this loop run
1890 * to completion, as we would eventually hit
1891 * the end of the dirty list. However, we
1892 * want to avoid running the length of the
1893 * dirty list unnecessarily (it might be quite
1894 * long), so we catch this as early as
1895 * possible by detecting the hash marker. In
1896 * this case, we simply set dvar to NULL and
1897 * break; the conditional after the loop will
1898 * send us back to top.
1907 if (dtuple
->dtt_nkeys
!= nkeys
)
1910 for (i
= 0; i
< nkeys
; i
++, dkey
++) {
1911 if (dkey
->dttk_size
!= key
[i
].dttk_size
)
1912 goto next
; /* size or type mismatch */
1914 if (dkey
->dttk_size
!= 0) {
1916 (void *)(uintptr_t)key
[i
].dttk_value
,
1917 (void *)(uintptr_t)dkey
->dttk_value
,
1921 if (dkey
->dttk_value
!= key
[i
].dttk_value
)
1926 if (op
!= DTRACE_DYNVAR_DEALLOC
)
1929 ASSERT(dvar
->dtdv_next
== NULL
||
1930 dvar
->dtdv_next
->dtdv_hashval
!= DTRACE_DYNHASH_FREE
);
1933 ASSERT(hash
[bucket
].dtdh_chain
!= dvar
);
1934 ASSERT(start
!= dvar
);
1935 ASSERT(prev
->dtdv_next
== dvar
);
1936 prev
->dtdv_next
= dvar
->dtdv_next
;
1938 if (dtrace_casptr(&hash
[bucket
].dtdh_chain
,
1939 start
, dvar
->dtdv_next
) != start
) {
1941 * We have failed to atomically swing the
1942 * hash table head pointer, presumably because
1943 * of a conflicting allocation on another CPU.
1944 * We need to reread the hash chain and try
1951 dtrace_membar_producer();
1954 * Now set the hash value to indicate that it's free.
1956 ASSERT(hash
[bucket
].dtdh_chain
!= dvar
);
1957 dvar
->dtdv_hashval
= DTRACE_DYNHASH_FREE
;
1959 dtrace_membar_producer();
1962 * Set the next pointer to point at the dirty list, and
1963 * atomically swing the dirty pointer to the newly freed dvar.
1966 next
= dcpu
->dtdsc_dirty
;
1967 dvar
->dtdv_next
= next
;
1968 } while (dtrace_casptr(&dcpu
->dtdsc_dirty
, next
, dvar
) != next
);
1971 * Finally, unlock this hash bucket.
1973 ASSERT(hash
[bucket
].dtdh_lock
== lock
);
1975 hash
[bucket
].dtdh_lock
++;
1985 * If dvar is NULL, it is because we went off the rails:
1986 * one of the elements that we traversed in the hash chain
1987 * was deleted while we were traversing it. In this case,
1988 * we assert that we aren't doing a dealloc (deallocs lock
1989 * the hash bucket to prevent themselves from racing with
1990 * one another), and retry the hash chain traversal.
1992 ASSERT(op
!= DTRACE_DYNVAR_DEALLOC
);
1996 if (op
!= DTRACE_DYNVAR_ALLOC
) {
1998 * If we are not to allocate a new variable, we want to
1999 * return NULL now. Before we return, check that the value
2000 * of the lock word hasn't changed. If it has, we may have
2001 * seen an inconsistent snapshot.
2003 if (op
== DTRACE_DYNVAR_NOALLOC
) {
2004 if (hash
[bucket
].dtdh_lock
!= lock
)
2007 ASSERT(op
== DTRACE_DYNVAR_DEALLOC
);
2008 ASSERT(hash
[bucket
].dtdh_lock
== lock
);
2010 hash
[bucket
].dtdh_lock
++;
2017 * We need to allocate a new dynamic variable. The size we need is the
2018 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
2019 * size of any auxiliary key data (rounded up to 8-byte alignment) plus
2020 * the size of any referred-to data (dsize). We then round the final
2021 * size up to the chunksize for allocation.
2023 for (ksize
= 0, i
= 0; i
< nkeys
; i
++)
2024 ksize
+= P2ROUNDUP(key
[i
].dttk_size
, sizeof (uint64_t));
2027 * This should be pretty much impossible, but could happen if, say,
2028 * strange DIF specified the tuple. Ideally, this should be an
2029 * assertion and not an error condition -- but that requires that the
2030 * chunksize calculation in dtrace_difo_chunksize() be absolutely
2031 * bullet-proof. (That is, it must not be able to be fooled by
2032 * malicious DIF.) Given the lack of backwards branches in DIF,
2033 * solving this would presumably not amount to solving the Halting
2034 * Problem -- but it still seems awfully hard.
2036 if (sizeof (dtrace_dynvar_t
) + sizeof (dtrace_key_t
) * (nkeys
- 1) +
2037 ksize
+ dsize
> chunksize
) {
2038 dcpu
->dtdsc_drops
++;
2042 nstate
= DTRACE_DSTATE_EMPTY
;
2046 free
= dcpu
->dtdsc_free
;
2049 dtrace_dynvar_t
*clean
= dcpu
->dtdsc_clean
;
2052 if (clean
== NULL
) {
2054 * We're out of dynamic variable space on
2055 * this CPU. Unless we have tried all CPUs,
2056 * we'll try to allocate from a different
2059 switch (dstate
->dtds_state
) {
2060 case DTRACE_DSTATE_CLEAN
: {
2061 void *sp
= &dstate
->dtds_state
;
2066 if (dcpu
->dtdsc_dirty
!= NULL
&&
2067 nstate
== DTRACE_DSTATE_EMPTY
)
2068 nstate
= DTRACE_DSTATE_DIRTY
;
2070 if (dcpu
->dtdsc_rinsing
!= NULL
)
2071 nstate
= DTRACE_DSTATE_RINSING
;
2073 dcpu
= &dstate
->dtds_percpu
[cpu
];
2078 (void) dtrace_cas32(sp
,
2079 DTRACE_DSTATE_CLEAN
, nstate
);
2082 * To increment the correct bean
2083 * counter, take another lap.
2088 case DTRACE_DSTATE_DIRTY
:
2089 dcpu
->dtdsc_dirty_drops
++;
2092 case DTRACE_DSTATE_RINSING
:
2093 dcpu
->dtdsc_rinsing_drops
++;
2096 case DTRACE_DSTATE_EMPTY
:
2097 dcpu
->dtdsc_drops
++;
2101 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP
);
2106 * The clean list appears to be non-empty. We want to
2107 * move the clean list to the free list; we start by
2108 * moving the clean pointer aside.
2110 if (dtrace_casptr(&dcpu
->dtdsc_clean
,
2111 clean
, NULL
) != clean
) {
2113 * We are in one of two situations:
2115 * (a) The clean list was switched to the
2116 * free list by another CPU.
2118 * (b) The clean list was added to by the
2121 * In either of these situations, we can
2122 * just reattempt the free list allocation.
2127 ASSERT(clean
->dtdv_hashval
== DTRACE_DYNHASH_FREE
);
2130 * Now we'll move the clean list to our free list.
2131 * It's impossible for this to fail: the only way
2132 * the free list can be updated is through this
2133 * code path, and only one CPU can own the clean list.
2134 * Thus, it would only be possible for this to fail if
2135 * this code were racing with dtrace_dynvar_clean().
2136 * (That is, if dtrace_dynvar_clean() updated the clean
2137 * list, and we ended up racing to update the free
2138 * list.) This race is prevented by the dtrace_sync()
2139 * in dtrace_dynvar_clean() -- which flushes the
2140 * owners of the clean lists out before resetting
2143 dcpu
= &dstate
->dtds_percpu
[me
];
2144 rval
= dtrace_casptr(&dcpu
->dtdsc_free
, NULL
, clean
);
2145 ASSERT(rval
== NULL
);
2150 new_free
= dvar
->dtdv_next
;
2151 } while (dtrace_casptr(&dcpu
->dtdsc_free
, free
, new_free
) != free
);
2154 * We have now allocated a new chunk. We copy the tuple keys into the
2155 * tuple array and copy any referenced key data into the data space
2156 * following the tuple array. As we do this, we relocate dttk_value
2157 * in the final tuple to point to the key data address in the chunk.
2159 kdata
= (uintptr_t)&dvar
->dtdv_tuple
.dtt_key
[nkeys
];
2160 dvar
->dtdv_data
= (void *)(kdata
+ ksize
);
2161 dvar
->dtdv_tuple
.dtt_nkeys
= nkeys
;
2163 for (i
= 0; i
< nkeys
; i
++) {
2164 dtrace_key_t
*dkey
= &dvar
->dtdv_tuple
.dtt_key
[i
];
2165 size_t kesize
= key
[i
].dttk_size
;
2169 (const void *)(uintptr_t)key
[i
].dttk_value
,
2170 (void *)kdata
, kesize
);
2171 dkey
->dttk_value
= kdata
;
2172 kdata
+= P2ROUNDUP(kesize
, sizeof (uint64_t));
2174 dkey
->dttk_value
= key
[i
].dttk_value
;
2177 dkey
->dttk_size
= kesize
;
2180 ASSERT(dvar
->dtdv_hashval
== DTRACE_DYNHASH_FREE
);
2181 dvar
->dtdv_hashval
= hashval
;
2182 dvar
->dtdv_next
= start
;
2184 if (dtrace_casptr(&hash
[bucket
].dtdh_chain
, start
, dvar
) == start
)
2188 * The cas has failed. Either another CPU is adding an element to
2189 * this hash chain, or another CPU is deleting an element from this
2190 * hash chain. The simplest way to deal with both of these cases
2191 * (though not necessarily the most efficient) is to free our
2192 * allocated block and re-attempt it all. Note that the free is
2193 * to the dirty list and _not_ to the free list. This is to prevent
2194 * races with allocators, above.
2196 dvar
->dtdv_hashval
= DTRACE_DYNHASH_FREE
;
2198 dtrace_membar_producer();
2201 free
= dcpu
->dtdsc_dirty
;
2202 dvar
->dtdv_next
= free
;
2203 } while (dtrace_casptr(&dcpu
->dtdsc_dirty
, free
, dvar
) != free
);
2210 dtrace_aggregate_min(uint64_t *oval
, uint64_t nval
, uint64_t arg
)
2212 if ((int64_t)nval
< (int64_t)*oval
)
2218 dtrace_aggregate_max(uint64_t *oval
, uint64_t nval
, uint64_t arg
)
2220 if ((int64_t)nval
> (int64_t)*oval
)
2225 dtrace_aggregate_quantize(uint64_t *quanta
, uint64_t nval
, uint64_t incr
)
2227 int i
, zero
= DTRACE_QUANTIZE_ZEROBUCKET
;
2228 int64_t val
= (int64_t)nval
;
2231 for (i
= 0; i
< zero
; i
++) {
2232 if (val
<= DTRACE_QUANTIZE_BUCKETVAL(i
)) {
2238 for (i
= zero
+ 1; i
< DTRACE_QUANTIZE_NBUCKETS
; i
++) {
2239 if (val
< DTRACE_QUANTIZE_BUCKETVAL(i
)) {
2240 quanta
[i
- 1] += incr
;
2245 quanta
[DTRACE_QUANTIZE_NBUCKETS
- 1] += incr
;
2253 dtrace_aggregate_lquantize(uint64_t *lquanta
, uint64_t nval
, uint64_t incr
)
2255 uint64_t arg
= *lquanta
++;
2256 int32_t base
= DTRACE_LQUANTIZE_BASE(arg
);
2257 uint16_t step
= DTRACE_LQUANTIZE_STEP(arg
);
2258 uint16_t levels
= DTRACE_LQUANTIZE_LEVELS(arg
);
2259 int32_t val
= (int32_t)nval
, level
;
2262 ASSERT(levels
!= 0);
2266 * This is an underflow.
2272 level
= (val
- base
) / step
;
2274 if (level
< levels
) {
2275 lquanta
[level
+ 1] += incr
;
2280 * This is an overflow.
2282 lquanta
[levels
+ 1] += incr
;
2286 dtrace_aggregate_llquantize_bucket(uint16_t factor
, uint16_t low
,
2287 uint16_t high
, uint16_t nsteps
, int64_t value
)
2289 int64_t this = 1, last
, next
;
2290 int base
= 1, order
;
2292 ASSERT(factor
<= nsteps
);
2293 ASSERT(nsteps
% factor
== 0);
2295 for (order
= 0; order
< low
; order
++)
2299 * If our value is less than our factor taken to the power of the
2300 * low order of magnitude, it goes into the zeroth bucket.
2302 if (value
< (last
= this))
2305 for (this *= factor
; order
<= high
; order
++) {
2306 int nbuckets
= this > nsteps
? nsteps
: this;
2308 if ((next
= this * factor
) < this) {
2310 * We should not generally get log/linear quantizations
2311 * with a high magnitude that allows 64-bits to
2312 * overflow, but we nonetheless protect against this
2313 * by explicitly checking for overflow, and clamping
2314 * our value accordingly.
2321 * If our value lies within this order of magnitude,
2322 * determine its position by taking the offset within
2323 * the order of magnitude, dividing by the bucket
2324 * width, and adding to our (accumulated) base.
2326 return (base
+ (value
- last
) / (this / nbuckets
));
2329 base
+= nbuckets
- (nbuckets
/ factor
);
2335 * Our value is greater than or equal to our factor taken to the
2336 * power of one plus the high magnitude -- return the top bucket.
2342 dtrace_aggregate_llquantize(uint64_t *llquanta
, uint64_t nval
, uint64_t incr
)
2344 uint64_t arg
= *llquanta
++;
2345 uint16_t factor
= DTRACE_LLQUANTIZE_FACTOR(arg
);
2346 uint16_t low
= DTRACE_LLQUANTIZE_LOW(arg
);
2347 uint16_t high
= DTRACE_LLQUANTIZE_HIGH(arg
);
2348 uint16_t nsteps
= DTRACE_LLQUANTIZE_NSTEP(arg
);
2350 llquanta
[dtrace_aggregate_llquantize_bucket(factor
,
2351 low
, high
, nsteps
, nval
)] += incr
;
2356 dtrace_aggregate_avg(uint64_t *data
, uint64_t nval
, uint64_t arg
)
2364 dtrace_aggregate_stddev(uint64_t *data
, uint64_t nval
, uint64_t arg
)
2366 int64_t snval
= (int64_t)nval
;
2373 * What we want to say here is:
2375 * data[2] += nval * nval;
2377 * But given that nval is 64-bit, we could easily overflow, so
2378 * we do this as 128-bit arithmetic.
2383 dtrace_multiply_128((uint64_t)snval
, (uint64_t)snval
, tmp
);
2384 dtrace_add_128(data
+ 2, tmp
, data
+ 2);
2389 dtrace_aggregate_count(uint64_t *oval
, uint64_t nval
, uint64_t arg
)
2396 dtrace_aggregate_sum(uint64_t *oval
, uint64_t nval
, uint64_t arg
)
2402 * Aggregate given the tuple in the principal data buffer, and the aggregating
2403 * action denoted by the specified dtrace_aggregation_t. The aggregation
2404 * buffer is specified as the buf parameter. This routine does not return
2405 * failure; if there is no space in the aggregation buffer, the data will be
2406 * dropped, and a corresponding counter incremented.
2409 dtrace_aggregate(dtrace_aggregation_t
*agg
, dtrace_buffer_t
*dbuf
,
2410 intptr_t offset
, dtrace_buffer_t
*buf
, uint64_t expr
, uint64_t arg
)
2412 dtrace_recdesc_t
*rec
= &agg
->dtag_action
.dta_rec
;
2413 uint32_t i
, ndx
, size
, fsize
;
2414 uint32_t align
= sizeof (uint64_t) - 1;
2415 dtrace_aggbuffer_t
*agb
;
2416 dtrace_aggkey_t
*key
;
2417 uint32_t hashval
= 0, limit
, isstr
;
2418 caddr_t tomax
, data
, kdata
;
2419 dtrace_actkind_t action
;
2420 dtrace_action_t
*act
;
2426 if (!agg
->dtag_hasarg
) {
2428 * Currently, only quantize() and lquantize() take additional
2429 * arguments, and they have the same semantics: an increment
2430 * value that defaults to 1 when not present. If additional
2431 * aggregating actions take arguments, the setting of the
2432 * default argument value will presumably have to become more
2438 action
= agg
->dtag_action
.dta_kind
- DTRACEACT_AGGREGATION
;
2439 size
= rec
->dtrd_offset
- agg
->dtag_base
;
2440 fsize
= size
+ rec
->dtrd_size
;
2442 ASSERT(dbuf
->dtb_tomax
!= NULL
);
2443 data
= dbuf
->dtb_tomax
+ offset
+ agg
->dtag_base
;
2445 if ((tomax
= buf
->dtb_tomax
) == NULL
) {
2446 dtrace_buffer_drop(buf
);
2451 * The metastructure is always at the bottom of the buffer.
2453 agb
= (dtrace_aggbuffer_t
*)(tomax
+ buf
->dtb_size
-
2454 sizeof (dtrace_aggbuffer_t
));
2456 if (buf
->dtb_offset
== 0) {
2458 * We just kludge up approximately 1/8th of the size to be
2459 * buckets. If this guess ends up being routinely
2460 * off-the-mark, we may need to dynamically readjust this
2461 * based on past performance.
2463 uintptr_t hashsize
= (buf
->dtb_size
>> 3) / sizeof (uintptr_t);
2465 if ((uintptr_t)agb
- hashsize
* sizeof (dtrace_aggkey_t
*) <
2466 (uintptr_t)tomax
|| hashsize
== 0) {
2468 * We've been given a ludicrously small buffer;
2469 * increment our drop count and leave.
2471 dtrace_buffer_drop(buf
);
2476 * And now, a pathetic attempt to try to get a an odd (or
2477 * perchance, a prime) hash size for better hash distribution.
2479 if (hashsize
> (DTRACE_AGGHASHSIZE_SLEW
<< 3))
2480 hashsize
-= DTRACE_AGGHASHSIZE_SLEW
;
2482 agb
->dtagb_hashsize
= hashsize
;
2483 agb
->dtagb_hash
= (dtrace_aggkey_t
**)((uintptr_t)agb
-
2484 agb
->dtagb_hashsize
* sizeof (dtrace_aggkey_t
*));
2485 agb
->dtagb_free
= (uintptr_t)agb
->dtagb_hash
;
2487 for (i
= 0; i
< agb
->dtagb_hashsize
; i
++)
2488 agb
->dtagb_hash
[i
] = NULL
;
2491 ASSERT(agg
->dtag_first
!= NULL
);
2492 ASSERT(agg
->dtag_first
->dta_intuple
);
2495 * Calculate the hash value based on the key. Note that we _don't_
2496 * include the aggid in the hashing (but we will store it as part of
2497 * the key). The hashing algorithm is Bob Jenkins' "One-at-a-time"
2498 * algorithm: a simple, quick algorithm that has no known funnels, and
2499 * gets good distribution in practice. The efficacy of the hashing
2500 * algorithm (and a comparison with other algorithms) may be found by
2501 * running the ::dtrace_aggstat MDB dcmd.
2503 for (act
= agg
->dtag_first
; act
->dta_intuple
; act
= act
->dta_next
) {
2504 i
= act
->dta_rec
.dtrd_offset
- agg
->dtag_base
;
2505 limit
= i
+ act
->dta_rec
.dtrd_size
;
2506 ASSERT(limit
<= size
);
2507 isstr
= DTRACEACT_ISSTRING(act
);
2509 for (; i
< limit
; i
++) {
2511 hashval
+= (hashval
<< 10);
2512 hashval
^= (hashval
>> 6);
2514 if (isstr
&& data
[i
] == '\0')
2519 hashval
+= (hashval
<< 3);
2520 hashval
^= (hashval
>> 11);
2521 hashval
+= (hashval
<< 15);
2524 * Yes, the divide here is expensive -- but it's generally the least
2525 * of the performance issues given the amount of data that we iterate
2526 * over to compute hash values, compare data, etc.
2528 ndx
= hashval
% agb
->dtagb_hashsize
;
2530 for (key
= agb
->dtagb_hash
[ndx
]; key
!= NULL
; key
= key
->dtak_next
) {
2531 ASSERT((caddr_t
)key
>= tomax
);
2532 ASSERT((caddr_t
)key
< tomax
+ buf
->dtb_size
);
2534 if (hashval
!= key
->dtak_hashval
|| key
->dtak_size
!= size
)
2537 kdata
= key
->dtak_data
;
2538 ASSERT(kdata
>= tomax
&& kdata
< tomax
+ buf
->dtb_size
);
2540 for (act
= agg
->dtag_first
; act
->dta_intuple
;
2541 act
= act
->dta_next
) {
2542 i
= act
->dta_rec
.dtrd_offset
- agg
->dtag_base
;
2543 limit
= i
+ act
->dta_rec
.dtrd_size
;
2544 ASSERT(limit
<= size
);
2545 isstr
= DTRACEACT_ISSTRING(act
);
2547 for (; i
< limit
; i
++) {
2548 if (kdata
[i
] != data
[i
])
2551 if (isstr
&& data
[i
] == '\0')
2556 if (action
!= key
->dtak_action
) {
2558 * We are aggregating on the same value in the same
2559 * aggregation with two different aggregating actions.
2560 * (This should have been picked up in the compiler,
2561 * so we may be dealing with errant or devious DIF.)
2562 * This is an error condition; we indicate as much,
2565 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP
);
2570 * This is a hit: we need to apply the aggregator to
2571 * the value at this key.
2573 agg
->dtag_aggregate((uint64_t *)(kdata
+ size
), expr
, arg
);
2580 * We didn't find it. We need to allocate some zero-filled space,
2581 * link it into the hash table appropriately, and apply the aggregator
2582 * to the (zero-filled) value.
2584 offs
= buf
->dtb_offset
;
2585 while (offs
& (align
- 1))
2586 offs
+= sizeof (uint32_t);
2589 * If we don't have enough room to both allocate a new key _and_
2590 * its associated data, increment the drop count and return.
2592 if ((uintptr_t)tomax
+ offs
+ fsize
>
2593 agb
->dtagb_free
- sizeof (dtrace_aggkey_t
)) {
2594 dtrace_buffer_drop(buf
);
2599 ASSERT(!(sizeof (dtrace_aggkey_t
) & (sizeof (uintptr_t) - 1)));
2600 key
= (dtrace_aggkey_t
*)(agb
->dtagb_free
- sizeof (dtrace_aggkey_t
));
2601 agb
->dtagb_free
-= sizeof (dtrace_aggkey_t
);
2603 key
->dtak_data
= kdata
= tomax
+ offs
;
2604 buf
->dtb_offset
= offs
+ fsize
;
2607 * Now copy the data across.
2609 *((dtrace_aggid_t
*)kdata
) = agg
->dtag_id
;
2611 for (i
= sizeof (dtrace_aggid_t
); i
< size
; i
++)
2615 * Because strings are not zeroed out by default, we need to iterate
2616 * looking for actions that store strings, and we need to explicitly
2617 * pad these strings out with zeroes.
2619 for (act
= agg
->dtag_first
; act
->dta_intuple
; act
= act
->dta_next
) {
2622 if (!DTRACEACT_ISSTRING(act
))
2625 i
= act
->dta_rec
.dtrd_offset
- agg
->dtag_base
;
2626 limit
= i
+ act
->dta_rec
.dtrd_size
;
2627 ASSERT(limit
<= size
);
2629 for (nul
= 0; i
< limit
; i
++) {
2635 if (data
[i
] != '\0')
2642 for (i
= size
; i
< fsize
; i
++)
2645 key
->dtak_hashval
= hashval
;
2646 key
->dtak_size
= size
;
2647 key
->dtak_action
= action
;
2648 key
->dtak_next
= agb
->dtagb_hash
[ndx
];
2649 agb
->dtagb_hash
[ndx
] = key
;
2652 * Finally, apply the aggregator.
2654 *((uint64_t *)(key
->dtak_data
+ size
)) = agg
->dtag_initial
;
2655 agg
->dtag_aggregate((uint64_t *)(key
->dtak_data
+ size
), expr
, arg
);
2659 * Given consumer state, this routine finds a speculation in the INACTIVE
2660 * state and transitions it into the ACTIVE state. If there is no speculation
2661 * in the INACTIVE state, 0 is returned. In this case, no error counter is
2662 * incremented -- it is up to the caller to take appropriate action.
2665 dtrace_speculation(dtrace_state_t
*state
)
2668 dtrace_speculation_state_t current
;
2669 uint32_t *stat
= &state
->dts_speculations_unavail
, count
;
2671 while (i
< state
->dts_nspeculations
) {
2672 dtrace_speculation_t
*spec
= &state
->dts_speculations
[i
];
2674 current
= spec
->dtsp_state
;
2676 if (current
!= DTRACESPEC_INACTIVE
) {
2677 if (current
== DTRACESPEC_COMMITTINGMANY
||
2678 current
== DTRACESPEC_COMMITTING
||
2679 current
== DTRACESPEC_DISCARDING
)
2680 stat
= &state
->dts_speculations_busy
;
2685 if (dtrace_cas32((uint32_t *)&spec
->dtsp_state
,
2686 current
, DTRACESPEC_ACTIVE
) == current
)
2691 * We couldn't find a speculation. If we found as much as a single
2692 * busy speculation buffer, we'll attribute this failure as "busy"
2693 * instead of "unavail".
2697 } while (dtrace_cas32(stat
, count
, count
+ 1) != count
);
2703 * This routine commits an active speculation. If the specified speculation
2704 * is not in a valid state to perform a commit(), this routine will silently do
2705 * nothing. The state of the specified speculation is transitioned according
2706 * to the state transition diagram outlined in <sys/dtrace_impl.h>
2709 dtrace_speculation_commit(dtrace_state_t
*state
, processorid_t cpu
,
2710 dtrace_specid_t which
)
2712 dtrace_speculation_t
*spec
;
2713 dtrace_buffer_t
*src
, *dest
;
2714 uintptr_t daddr
, saddr
, dlimit
, slimit
;
2715 dtrace_speculation_state_t current
, new;
2722 if (which
> state
->dts_nspeculations
) {
2723 cpu_core
[cpu
].cpuc_dtrace_flags
|= CPU_DTRACE_ILLOP
;
2727 spec
= &state
->dts_speculations
[which
- 1];
2728 src
= &spec
->dtsp_buffer
[cpu
];
2729 dest
= &state
->dts_buffer
[cpu
];
2732 current
= spec
->dtsp_state
;
2734 if (current
== DTRACESPEC_COMMITTINGMANY
)
2738 case DTRACESPEC_INACTIVE
:
2739 case DTRACESPEC_DISCARDING
:
2742 case DTRACESPEC_COMMITTING
:
2744 * This is only possible if we are (a) commit()'ing
2745 * without having done a prior speculate() on this CPU
2746 * and (b) racing with another commit() on a different
2747 * CPU. There's nothing to do -- we just assert that
2750 ASSERT(src
->dtb_offset
== 0);
2753 case DTRACESPEC_ACTIVE
:
2754 new = DTRACESPEC_COMMITTING
;
2757 case DTRACESPEC_ACTIVEONE
:
2759 * This speculation is active on one CPU. If our
2760 * buffer offset is non-zero, we know that the one CPU
2761 * must be us. Otherwise, we are committing on a
2762 * different CPU from the speculate(), and we must
2763 * rely on being asynchronously cleaned.
2765 if (src
->dtb_offset
!= 0) {
2766 new = DTRACESPEC_COMMITTING
;
2771 case DTRACESPEC_ACTIVEMANY
:
2772 new = DTRACESPEC_COMMITTINGMANY
;
2778 } while (dtrace_cas32((uint32_t *)&spec
->dtsp_state
,
2779 current
, new) != current
);
2782 * We have set the state to indicate that we are committing this
2783 * speculation. Now reserve the necessary space in the destination
2786 if ((offs
= dtrace_buffer_reserve(dest
, src
->dtb_offset
,
2787 sizeof (uint64_t), state
, NULL
)) < 0) {
2788 dtrace_buffer_drop(dest
);
2793 * We have sufficient space to copy the speculative buffer into the
2794 * primary buffer. First, modify the speculative buffer, filling
2795 * in the timestamp of all entries with the current time. The data
2796 * must have the commit() time rather than the time it was traced,
2797 * so that all entries in the primary buffer are in timestamp order.
2799 timestamp
= dtrace_gethrtime();
2800 saddr
= (uintptr_t)src
->dtb_tomax
;
2801 slimit
= saddr
+ src
->dtb_offset
;
2802 while (saddr
< slimit
) {
2804 dtrace_rechdr_t
*dtrh
= (dtrace_rechdr_t
*)saddr
;
2806 if (dtrh
->dtrh_epid
== DTRACE_EPIDNONE
) {
2807 saddr
+= sizeof (dtrace_epid_t
);
2810 ASSERT3U(dtrh
->dtrh_epid
, <=, state
->dts_necbs
);
2811 size
= state
->dts_ecbs
[dtrh
->dtrh_epid
- 1]->dte_size
;
2813 ASSERT3U(saddr
+ size
, <=, slimit
);
2814 ASSERT3U(size
, >=, sizeof (dtrace_rechdr_t
));
2815 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh
), ==, UINT64_MAX
);
2817 DTRACE_RECORD_STORE_TIMESTAMP(dtrh
, timestamp
);
2823 * Copy the buffer across. (Note that this is a
2824 * highly subobtimal bcopy(); in the unlikely event that this becomes
2825 * a serious performance issue, a high-performance DTrace-specific
2826 * bcopy() should obviously be invented.)
2828 daddr
= (uintptr_t)dest
->dtb_tomax
+ offs
;
2829 dlimit
= daddr
+ src
->dtb_offset
;
2830 saddr
= (uintptr_t)src
->dtb_tomax
;
2833 * First, the aligned portion.
2835 while (dlimit
- daddr
>= sizeof (uint64_t)) {
2836 *((uint64_t *)daddr
) = *((uint64_t *)saddr
);
2838 daddr
+= sizeof (uint64_t);
2839 saddr
+= sizeof (uint64_t);
2843 * Now any left-over bit...
2845 while (dlimit
- daddr
)
2846 *((uint8_t *)daddr
++) = *((uint8_t *)saddr
++);
2849 * Finally, commit the reserved space in the destination buffer.
2851 dest
->dtb_offset
= offs
+ src
->dtb_offset
;
2855 * If we're lucky enough to be the only active CPU on this speculation
2856 * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2858 if (current
== DTRACESPEC_ACTIVE
||
2859 (current
== DTRACESPEC_ACTIVEONE
&& new == DTRACESPEC_COMMITTING
)) {
2860 uint32_t rval
= dtrace_cas32((uint32_t *)&spec
->dtsp_state
,
2861 DTRACESPEC_COMMITTING
, DTRACESPEC_INACTIVE
);
2863 ASSERT(rval
== DTRACESPEC_COMMITTING
);
2866 src
->dtb_offset
= 0;
2867 src
->dtb_xamot_drops
+= src
->dtb_drops
;
2872 * This routine discards an active speculation. If the specified speculation
2873 * is not in a valid state to perform a discard(), this routine will silently
2874 * do nothing. The state of the specified speculation is transitioned
2875 * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2878 dtrace_speculation_discard(dtrace_state_t
*state
, processorid_t cpu
,
2879 dtrace_specid_t which
)
2881 dtrace_speculation_t
*spec
;
2882 dtrace_speculation_state_t current
, new;
2883 dtrace_buffer_t
*buf
;
2888 if (which
> state
->dts_nspeculations
) {
2889 cpu_core
[cpu
].cpuc_dtrace_flags
|= CPU_DTRACE_ILLOP
;
2893 spec
= &state
->dts_speculations
[which
- 1];
2894 buf
= &spec
->dtsp_buffer
[cpu
];
2897 current
= spec
->dtsp_state
;
2900 case DTRACESPEC_INACTIVE
:
2901 case DTRACESPEC_COMMITTINGMANY
:
2902 case DTRACESPEC_COMMITTING
:
2903 case DTRACESPEC_DISCARDING
:
2906 case DTRACESPEC_ACTIVE
:
2907 case DTRACESPEC_ACTIVEMANY
:
2908 new = DTRACESPEC_DISCARDING
;
2911 case DTRACESPEC_ACTIVEONE
:
2912 if (buf
->dtb_offset
!= 0) {
2913 new = DTRACESPEC_INACTIVE
;
2915 new = DTRACESPEC_DISCARDING
;
2922 } while (dtrace_cas32((uint32_t *)&spec
->dtsp_state
,
2923 current
, new) != current
);
2925 buf
->dtb_offset
= 0;
2930 * Note: not called from probe context. This function is called
2931 * asynchronously from cross call context to clean any speculations that are
2932 * in the COMMITTINGMANY or DISCARDING states. These speculations may not be
2933 * transitioned back to the INACTIVE state until all CPUs have cleaned the
2937 dtrace_speculation_clean_here(dtrace_state_t
*state
)
2939 dtrace_icookie_t cookie
;
2940 processorid_t cpu
= CPU
->cpu_id
;
2941 dtrace_buffer_t
*dest
= &state
->dts_buffer
[cpu
];
2944 cookie
= dtrace_interrupt_disable();
2946 if (dest
->dtb_tomax
== NULL
) {
2947 dtrace_interrupt_enable(cookie
);
2951 for (i
= 0; i
< state
->dts_nspeculations
; i
++) {
2952 dtrace_speculation_t
*spec
= &state
->dts_speculations
[i
];
2953 dtrace_buffer_t
*src
= &spec
->dtsp_buffer
[cpu
];
2955 if (src
->dtb_tomax
== NULL
)
2958 if (spec
->dtsp_state
== DTRACESPEC_DISCARDING
) {
2959 src
->dtb_offset
= 0;
2963 if (spec
->dtsp_state
!= DTRACESPEC_COMMITTINGMANY
)
2966 if (src
->dtb_offset
== 0)
2969 dtrace_speculation_commit(state
, cpu
, i
+ 1);
2972 dtrace_interrupt_enable(cookie
);
2976 * Note: not called from probe context. This function is called
2977 * asynchronously (and at a regular interval) to clean any speculations that
2978 * are in the COMMITTINGMANY or DISCARDING states. If it discovers that there
2979 * is work to be done, it cross calls all CPUs to perform that work;
2980 * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2981 * INACTIVE state until they have been cleaned by all CPUs.
2984 dtrace_speculation_clean(dtrace_state_t
*state
)
2989 for (i
= 0; i
< state
->dts_nspeculations
; i
++) {
2990 dtrace_speculation_t
*spec
= &state
->dts_speculations
[i
];
2992 ASSERT(!spec
->dtsp_cleaning
);
2994 if (spec
->dtsp_state
!= DTRACESPEC_DISCARDING
&&
2995 spec
->dtsp_state
!= DTRACESPEC_COMMITTINGMANY
)
2999 spec
->dtsp_cleaning
= 1;
3005 dtrace_xcall(DTRACE_CPUALL
,
3006 (dtrace_xcall_t
)dtrace_speculation_clean_here
, state
);
3009 * We now know that all CPUs have committed or discarded their
3010 * speculation buffers, as appropriate. We can now set the state
3013 for (i
= 0; i
< state
->dts_nspeculations
; i
++) {
3014 dtrace_speculation_t
*spec
= &state
->dts_speculations
[i
];
3015 dtrace_speculation_state_t current
, new;
3017 if (!spec
->dtsp_cleaning
)
3020 current
= spec
->dtsp_state
;
3021 ASSERT(current
== DTRACESPEC_DISCARDING
||
3022 current
== DTRACESPEC_COMMITTINGMANY
);
3024 new = DTRACESPEC_INACTIVE
;
3026 rv
= dtrace_cas32((uint32_t *)&spec
->dtsp_state
, current
, new);
3027 ASSERT(rv
== current
);
3028 spec
->dtsp_cleaning
= 0;
3033 * Called as part of a speculate() to get the speculative buffer associated
3034 * with a given speculation. Returns NULL if the specified speculation is not
3035 * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and
3036 * the active CPU is not the specified CPU -- the speculation will be
3037 * atomically transitioned into the ACTIVEMANY state.
3039 static dtrace_buffer_t
*
3040 dtrace_speculation_buffer(dtrace_state_t
*state
, processorid_t cpuid
,
3041 dtrace_specid_t which
)
3043 dtrace_speculation_t
*spec
;
3044 dtrace_speculation_state_t current
, new;
3045 dtrace_buffer_t
*buf
;
3050 if (which
> state
->dts_nspeculations
) {
3051 cpu_core
[cpuid
].cpuc_dtrace_flags
|= CPU_DTRACE_ILLOP
;
3055 spec
= &state
->dts_speculations
[which
- 1];
3056 buf
= &spec
->dtsp_buffer
[cpuid
];
3059 current
= spec
->dtsp_state
;
3062 case DTRACESPEC_INACTIVE
:
3063 case DTRACESPEC_COMMITTINGMANY
:
3064 case DTRACESPEC_DISCARDING
:
3067 case DTRACESPEC_COMMITTING
:
3068 ASSERT(buf
->dtb_offset
== 0);
3071 case DTRACESPEC_ACTIVEONE
:
3073 * This speculation is currently active on one CPU.
3074 * Check the offset in the buffer; if it's non-zero,
3075 * that CPU must be us (and we leave the state alone).
3076 * If it's zero, assume that we're starting on a new
3077 * CPU -- and change the state to indicate that the
3078 * speculation is active on more than one CPU.
3080 if (buf
->dtb_offset
!= 0)
3083 new = DTRACESPEC_ACTIVEMANY
;
3086 case DTRACESPEC_ACTIVEMANY
:
3089 case DTRACESPEC_ACTIVE
:
3090 new = DTRACESPEC_ACTIVEONE
;
3096 } while (dtrace_cas32((uint32_t *)&spec
->dtsp_state
,
3097 current
, new) != current
);
3099 ASSERT(new == DTRACESPEC_ACTIVEONE
|| new == DTRACESPEC_ACTIVEMANY
);
3104 * Return a string. In the event that the user lacks the privilege to access
3105 * arbitrary kernel memory, we copy the string out to scratch memory so that we
3106 * don't fail access checking.
3108 * dtrace_dif_variable() uses this routine as a helper for various
3109 * builtin values such as 'execname' and 'probefunc.'
3112 dtrace_dif_varstr(uintptr_t addr
, dtrace_state_t
*state
,
3113 dtrace_mstate_t
*mstate
)
3115 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
3120 * The easy case: this probe is allowed to read all of memory, so
3121 * we can just return this as a vanilla pointer.
3123 if ((mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) != 0)
3127 * This is the tougher case: we copy the string in question from
3128 * kernel memory into scratch memory and return it that way: this
3129 * ensures that we won't trip up when access checking tests the
3130 * BYREF return value.
3132 strsz
= dtrace_strlen((char *)addr
, size
) + 1;
3134 if (mstate
->dtms_scratch_ptr
+ strsz
>
3135 mstate
->dtms_scratch_base
+ mstate
->dtms_scratch_size
) {
3136 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
3140 dtrace_strcpy((const void *)addr
, (void *)mstate
->dtms_scratch_ptr
,
3142 ret
= mstate
->dtms_scratch_ptr
;
3143 mstate
->dtms_scratch_ptr
+= strsz
;
3148 * This function implements the DIF emulator's variable lookups. The emulator
3149 * passes a reserved variable identifier and optional built-in array index.
3152 dtrace_dif_variable(dtrace_mstate_t
*mstate
, dtrace_state_t
*state
, uint64_t v
,
3156 * If we're accessing one of the uncached arguments, we'll turn this
3157 * into a reference in the args array.
3159 if (v
>= DIF_VAR_ARG0
&& v
<= DIF_VAR_ARG9
) {
3160 ndx
= v
- DIF_VAR_ARG0
;
3166 if (!(mstate
->dtms_access
& DTRACE_ACCESS_ARGS
)) {
3167 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
|=
3172 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_ARGS
);
3173 if (ndx
>= sizeof (mstate
->dtms_arg
) /
3174 sizeof (mstate
->dtms_arg
[0])) {
3175 int aframes
= mstate
->dtms_probe
->dtpr_aframes
+ 2;
3176 dtrace_provider_t
*pv
;
3179 pv
= mstate
->dtms_probe
->dtpr_provider
;
3180 if (pv
->dtpv_pops
.dtps_getargval
!= NULL
)
3181 val
= pv
->dtpv_pops
.dtps_getargval(pv
->dtpv_arg
,
3182 mstate
->dtms_probe
->dtpr_id
,
3183 mstate
->dtms_probe
->dtpr_arg
, ndx
, aframes
);
3185 val
= dtrace_getarg(ndx
, aframes
);
3188 * This is regrettably required to keep the compiler
3189 * from tail-optimizing the call to dtrace_getarg().
3190 * The condition always evaluates to true, but the
3191 * compiler has no way of figuring that out a priori.
3192 * (None of this would be necessary if the compiler
3193 * could be relied upon to _always_ tail-optimize
3194 * the call to dtrace_getarg() -- but it can't.)
3196 if (mstate
->dtms_probe
!= NULL
)
3202 return (mstate
->dtms_arg
[ndx
]);
3204 case DIF_VAR_UREGS
: {
3207 if (!dtrace_priv_proc(state
, mstate
))
3210 if ((lwp
= curthread
->t_lwp
) == NULL
) {
3211 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR
);
3212 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
= NULL
;
3216 return (dtrace_getreg(lwp
->lwp_regs
, ndx
));
3219 case DIF_VAR_VMREGS
: {
3222 if (!dtrace_priv_kernel(state
))
3225 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
3227 rval
= dtrace_getvmreg(ndx
,
3228 &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
);
3230 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
3235 case DIF_VAR_CURTHREAD
:
3236 if (!dtrace_priv_proc(state
, mstate
))
3238 return ((uint64_t)(uintptr_t)curthread
);
3240 case DIF_VAR_TIMESTAMP
:
3241 if (!(mstate
->dtms_present
& DTRACE_MSTATE_TIMESTAMP
)) {
3242 mstate
->dtms_timestamp
= dtrace_gethrtime();
3243 mstate
->dtms_present
|= DTRACE_MSTATE_TIMESTAMP
;
3245 return (mstate
->dtms_timestamp
);
3247 case DIF_VAR_VTIMESTAMP
:
3248 ASSERT(dtrace_vtime_references
!= 0);
3249 return (curthread
->t_dtrace_vtime
);
3251 case DIF_VAR_WALLTIMESTAMP
:
3252 if (!(mstate
->dtms_present
& DTRACE_MSTATE_WALLTIMESTAMP
)) {
3253 mstate
->dtms_walltimestamp
= dtrace_gethrestime();
3254 mstate
->dtms_present
|= DTRACE_MSTATE_WALLTIMESTAMP
;
3256 return (mstate
->dtms_walltimestamp
);
3259 if (!dtrace_priv_kernel(state
))
3261 if (!(mstate
->dtms_present
& DTRACE_MSTATE_IPL
)) {
3262 mstate
->dtms_ipl
= dtrace_getipl();
3263 mstate
->dtms_present
|= DTRACE_MSTATE_IPL
;
3265 return (mstate
->dtms_ipl
);
3268 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_EPID
);
3269 return (mstate
->dtms_epid
);
3272 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_PROBE
);
3273 return (mstate
->dtms_probe
->dtpr_id
);
3275 case DIF_VAR_STACKDEPTH
:
3276 if (!dtrace_priv_kernel(state
))
3278 if (!(mstate
->dtms_present
& DTRACE_MSTATE_STACKDEPTH
)) {
3279 int aframes
= mstate
->dtms_probe
->dtpr_aframes
+ 2;
3281 mstate
->dtms_stackdepth
= dtrace_getstackdepth(aframes
);
3282 mstate
->dtms_present
|= DTRACE_MSTATE_STACKDEPTH
;
3284 return (mstate
->dtms_stackdepth
);
3286 case DIF_VAR_USTACKDEPTH
:
3287 if (!dtrace_priv_proc(state
, mstate
))
3289 if (!(mstate
->dtms_present
& DTRACE_MSTATE_USTACKDEPTH
)) {
3291 * See comment in DIF_VAR_PID.
3293 if (DTRACE_ANCHORED(mstate
->dtms_probe
) &&
3295 mstate
->dtms_ustackdepth
= 0;
3297 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
3298 mstate
->dtms_ustackdepth
=
3299 dtrace_getustackdepth();
3300 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
3302 mstate
->dtms_present
|= DTRACE_MSTATE_USTACKDEPTH
;
3304 return (mstate
->dtms_ustackdepth
);
3306 case DIF_VAR_CALLER
:
3307 if (!dtrace_priv_kernel(state
))
3309 if (!(mstate
->dtms_present
& DTRACE_MSTATE_CALLER
)) {
3310 int aframes
= mstate
->dtms_probe
->dtpr_aframes
+ 2;
3312 if (!DTRACE_ANCHORED(mstate
->dtms_probe
)) {
3314 * If this is an unanchored probe, we are
3315 * required to go through the slow path:
3316 * dtrace_caller() only guarantees correct
3317 * results for anchored probes.
3321 dtrace_getpcstack(caller
, 2, aframes
,
3322 (uint32_t *)(uintptr_t)mstate
->dtms_arg
[0]);
3323 mstate
->dtms_caller
= caller
[1];
3324 } else if ((mstate
->dtms_caller
=
3325 dtrace_caller(aframes
)) == -1) {
3327 * We have failed to do this the quick way;
3328 * we must resort to the slower approach of
3329 * calling dtrace_getpcstack().
3333 dtrace_getpcstack(&caller
, 1, aframes
, NULL
);
3334 mstate
->dtms_caller
= caller
;
3337 mstate
->dtms_present
|= DTRACE_MSTATE_CALLER
;
3339 return (mstate
->dtms_caller
);
3341 case DIF_VAR_UCALLER
:
3342 if (!dtrace_priv_proc(state
, mstate
))
3345 if (!(mstate
->dtms_present
& DTRACE_MSTATE_UCALLER
)) {
3349 * dtrace_getupcstack() fills in the first uint64_t
3350 * with the current PID. The second uint64_t will
3351 * be the program counter at user-level. The third
3352 * uint64_t will contain the caller, which is what
3356 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
3357 dtrace_getupcstack(ustack
, 3);
3358 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
3359 mstate
->dtms_ucaller
= ustack
[2];
3360 mstate
->dtms_present
|= DTRACE_MSTATE_UCALLER
;
3363 return (mstate
->dtms_ucaller
);
3365 case DIF_VAR_PROBEPROV
:
3366 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_PROBE
);
3367 return (dtrace_dif_varstr(
3368 (uintptr_t)mstate
->dtms_probe
->dtpr_provider
->dtpv_name
,
3371 case DIF_VAR_PROBEMOD
:
3372 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_PROBE
);
3373 return (dtrace_dif_varstr(
3374 (uintptr_t)mstate
->dtms_probe
->dtpr_mod
,
3377 case DIF_VAR_PROBEFUNC
:
3378 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_PROBE
);
3379 return (dtrace_dif_varstr(
3380 (uintptr_t)mstate
->dtms_probe
->dtpr_func
,
3383 case DIF_VAR_PROBENAME
:
3384 ASSERT(mstate
->dtms_present
& DTRACE_MSTATE_PROBE
);
3385 return (dtrace_dif_varstr(
3386 (uintptr_t)mstate
->dtms_probe
->dtpr_name
,
3390 if (!dtrace_priv_proc(state
, mstate
))
3394 * Note that we are assuming that an unanchored probe is
3395 * always due to a high-level interrupt. (And we're assuming
3396 * that there is only a single high level interrupt.)
3398 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3399 return (pid0
.pid_id
);
3402 * It is always safe to dereference one's own t_procp pointer:
3403 * it always points to a valid, allocated proc structure.
3404 * Further, it is always safe to dereference the p_pidp member
3405 * of one's own proc structure. (These are truisms becuase
3406 * threads and processes don't clean up their own state --
3407 * they leave that task to whomever reaps them.)
3409 return ((uint64_t)curthread
->t_procp
->p_pidp
->pid_id
);
3412 if (!dtrace_priv_proc(state
, mstate
))
3416 * See comment in DIF_VAR_PID.
3418 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3419 return (pid0
.pid_id
);
3422 * It is always safe to dereference one's own t_procp pointer:
3423 * it always points to a valid, allocated proc structure.
3424 * (This is true because threads don't clean up their own
3425 * state -- they leave that task to whomever reaps them.)
3427 return ((uint64_t)curthread
->t_procp
->p_ppid
);
3431 * See comment in DIF_VAR_PID.
3433 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3436 return ((uint64_t)curthread
->t_tid
);
3438 case DIF_VAR_EXECNAME
:
3439 if (!dtrace_priv_proc(state
, mstate
))
3443 * See comment in DIF_VAR_PID.
3445 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3446 return ((uint64_t)(uintptr_t)p0
.p_user
.u_comm
);
3449 * It is always safe to dereference one's own t_procp pointer:
3450 * it always points to a valid, allocated proc structure.
3451 * (This is true because threads don't clean up their own
3452 * state -- they leave that task to whomever reaps them.)
3454 return (dtrace_dif_varstr(
3455 (uintptr_t)curthread
->t_procp
->p_user
.u_comm
,
3458 case DIF_VAR_ZONENAME
:
3459 if (!dtrace_priv_proc(state
, mstate
))
3463 * See comment in DIF_VAR_PID.
3465 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3466 return ((uint64_t)(uintptr_t)p0
.p_zone
->zone_name
);
3469 * It is always safe to dereference one's own t_procp pointer:
3470 * it always points to a valid, allocated proc structure.
3471 * (This is true because threads don't clean up their own
3472 * state -- they leave that task to whomever reaps them.)
3474 return (dtrace_dif_varstr(
3475 (uintptr_t)curthread
->t_procp
->p_zone
->zone_name
,
3479 if (!dtrace_priv_proc(state
, mstate
))
3483 * See comment in DIF_VAR_PID.
3485 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3486 return ((uint64_t)p0
.p_cred
->cr_uid
);
3489 * It is always safe to dereference one's own t_procp pointer:
3490 * it always points to a valid, allocated proc structure.
3491 * (This is true because threads don't clean up their own
3492 * state -- they leave that task to whomever reaps them.)
3494 * Additionally, it is safe to dereference one's own process
3495 * credential, since this is never NULL after process birth.
3497 return ((uint64_t)curthread
->t_procp
->p_cred
->cr_uid
);
3500 if (!dtrace_priv_proc(state
, mstate
))
3504 * See comment in DIF_VAR_PID.
3506 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3507 return ((uint64_t)p0
.p_cred
->cr_gid
);
3510 * It is always safe to dereference one's own t_procp pointer:
3511 * it always points to a valid, allocated proc structure.
3512 * (This is true because threads don't clean up their own
3513 * state -- they leave that task to whomever reaps them.)
3515 * Additionally, it is safe to dereference one's own process
3516 * credential, since this is never NULL after process birth.
3518 return ((uint64_t)curthread
->t_procp
->p_cred
->cr_gid
);
3520 case DIF_VAR_ERRNO
: {
3522 if (!dtrace_priv_proc(state
, mstate
))
3526 * See comment in DIF_VAR_PID.
3528 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3532 * It is always safe to dereference one's own t_lwp pointer in
3533 * the event that this pointer is non-NULL. (This is true
3534 * because threads and lwps don't clean up their own state --
3535 * they leave that task to whomever reaps them.)
3537 if ((lwp
= curthread
->t_lwp
) == NULL
)
3540 return ((uint64_t)lwp
->lwp_errno
);
3543 case DIF_VAR_THREADNAME
:
3545 * See comment in DIF_VAR_PID.
3547 if (DTRACE_ANCHORED(mstate
->dtms_probe
) && CPU_ON_INTR(CPU
))
3550 if (curthread
->t_name
== NULL
)
3554 * Once set, ->t_name itself is never changed: any updates are
3555 * made to the same buffer that we are pointing out. So we are
3556 * safe to dereference it here.
3558 return (dtrace_dif_varstr((uintptr_t)curthread
->t_name
,
3562 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP
);
3568 dtrace_dif_variable_write(dtrace_mstate_t
*mstate
, dtrace_state_t
*state
,
3569 uint64_t v
, uint64_t ndx
, uint64_t data
)
3572 case DIF_VAR_UREGS
: {
3575 if (dtrace_destructive_disallow
||
3576 !dtrace_priv_proc_control(state
, mstate
)) {
3580 if ((lwp
= curthread
->t_lwp
) == NULL
) {
3581 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR
);
3582 cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
= NULL
;
3586 dtrace_setreg(lwp
->lwp_regs
, ndx
, data
);
3591 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP
);
3596 typedef enum dtrace_json_state
{
3597 DTRACE_JSON_REST
= 1,
3600 DTRACE_JSON_STRING_ESCAPE
,
3601 DTRACE_JSON_STRING_ESCAPE_UNICODE
,
3605 DTRACE_JSON_IDENTIFIER
,
3607 DTRACE_JSON_NUMBER_FRAC
,
3608 DTRACE_JSON_NUMBER_EXP
,
3609 DTRACE_JSON_COLLECT_OBJECT
3610 } dtrace_json_state_t
;
3613 * This function possesses just enough knowledge about JSON to extract a single
3614 * value from a JSON string and store it in the scratch buffer. It is able
3615 * to extract nested object values, and members of arrays by index.
3617 * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to
3618 * be looked up as we descend into the object tree. e.g.
3620 * foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL
3623 * The run time of this function must be bounded above by strsize to limit the
3624 * amount of work done in probe context. As such, it is implemented as a
3625 * simple state machine, reading one character at a time using safe loads
3626 * until we find the requested element, hit a parsing error or run off the
3627 * end of the object or string.
3629 * As there is no way for a subroutine to return an error without interrupting
3630 * clause execution, we simply return NULL in the event of a missing key or any
3631 * other error condition. Each NULL return in this function is commented with
3632 * the error condition it represents -- parsing or otherwise.
3634 * The set of states for the state machine closely matches the JSON
3635 * specification (http://json.org/). Briefly:
3638 * Skip whitespace until we find either a top-level Object, moving
3639 * to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE.
3641 * DTRACE_JSON_OBJECT:
3642 * Locate the next key String in an Object. Sets a flag to denote
3643 * the next String as a key string and moves to DTRACE_JSON_STRING.
3645 * DTRACE_JSON_COLON:
3646 * Skip whitespace until we find the colon that separates key Strings
3647 * from their values. Once found, move to DTRACE_JSON_VALUE.
3649 * DTRACE_JSON_VALUE:
3650 * Detects the type of the next value (String, Number, Identifier, Object
3651 * or Array) and routes to the states that process that type. Here we also
3652 * deal with the element selector list if we are requested to traverse down
3653 * into the object tree.
3655 * DTRACE_JSON_COMMA:
3656 * Skip whitespace until we find the comma that separates key-value pairs
3657 * in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays
3658 * (similarly DTRACE_JSON_VALUE). All following literal value processing
3659 * states return to this state at the end of their value, unless otherwise
3662 * DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP:
3663 * Processes a Number literal from the JSON, including any exponent
3664 * component that may be present. Numbers are returned as strings, which
3665 * may be passed to strtoll() if an integer is required.
3667 * DTRACE_JSON_IDENTIFIER:
3668 * Processes a "true", "false" or "null" literal in the JSON.
3670 * DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE,
3671 * DTRACE_JSON_STRING_ESCAPE_UNICODE:
3672 * Processes a String literal from the JSON, whether the String denotes
3673 * a key, a value or part of a larger Object. Handles all escape sequences
3674 * present in the specification, including four-digit unicode characters,
3675 * but merely includes the escape sequence without converting it to the
3676 * actual escaped character. If the String is flagged as a key, we
3677 * move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA.
3679 * DTRACE_JSON_COLLECT_OBJECT:
3680 * This state collects an entire Object (or Array), correctly handling
3681 * embedded strings. If the full element selector list matches this nested
3682 * object, we return the Object in full as a string. If not, we use this
3683 * state to skip to the next value at this level and continue processing.
3685 * NOTE: This function uses various macros from strtolctype.h to manipulate
3686 * digit values, etc -- these have all been checked to ensure they make
3687 * no additional function calls.
3690 dtrace_json(uint64_t size
, uintptr_t json
, char *elemlist
, int nelems
,
3693 dtrace_json_state_t state
= DTRACE_JSON_REST
;
3694 int64_t array_elem
= INT64_MIN
;
3695 int64_t array_pos
= 0;
3696 uint8_t escape_unicount
= 0;
3697 boolean_t string_is_key
= B_FALSE
;
3698 boolean_t collect_object
= B_FALSE
;
3699 boolean_t found_key
= B_FALSE
;
3700 boolean_t in_array
= B_FALSE
;
3701 uint32_t braces
= 0, brackets
= 0;
3702 char *elem
= elemlist
;
3706 for (cur
= json
; cur
< json
+ size
; cur
++) {
3707 char cc
= dtrace_load8(cur
);
3712 case DTRACE_JSON_REST
:
3717 state
= DTRACE_JSON_OBJECT
;
3724 array_elem
= dtrace_strtoll(elem
, 10, size
);
3725 found_key
= array_elem
== 0 ? B_TRUE
: B_FALSE
;
3726 state
= DTRACE_JSON_VALUE
;
3731 * ERROR: expected to find a top-level object or array.
3734 case DTRACE_JSON_OBJECT
:
3739 state
= DTRACE_JSON_STRING
;
3740 string_is_key
= B_TRUE
;
3745 * ERROR: either the object did not start with a key
3746 * string, or we've run off the end of the object
3747 * without finding the requested key.
3750 case DTRACE_JSON_STRING
:
3753 state
= DTRACE_JSON_STRING_ESCAPE
;
3758 if (collect_object
) {
3760 * We don't reset the dest here, as
3761 * the string is part of a larger
3762 * object being collected.
3765 collect_object
= B_FALSE
;
3766 state
= DTRACE_JSON_COLLECT_OBJECT
;
3770 dd
= dest
; /* reset string buffer */
3771 if (string_is_key
) {
3772 if (dtrace_strncmp(dest
, elem
,
3775 } else if (found_key
) {
3778 * We expected an object, not
3785 state
= string_is_key
? DTRACE_JSON_COLON
:
3787 string_is_key
= B_FALSE
;
3793 case DTRACE_JSON_STRING_ESCAPE
:
3796 escape_unicount
= 0;
3797 state
= DTRACE_JSON_STRING_ESCAPE_UNICODE
;
3799 state
= DTRACE_JSON_STRING
;
3802 case DTRACE_JSON_STRING_ESCAPE_UNICODE
:
3803 if (!isxdigit(cc
)) {
3805 * ERROR: invalid unicode escape, expected
3806 * four valid hexidecimal digits.
3812 if (++escape_unicount
== 4)
3813 state
= DTRACE_JSON_STRING
;
3815 case DTRACE_JSON_COLON
:
3820 state
= DTRACE_JSON_VALUE
;
3825 * ERROR: expected a colon.
3828 case DTRACE_JSON_COMMA
:
3834 state
= DTRACE_JSON_VALUE
;
3835 if (++array_pos
== array_elem
)
3838 state
= DTRACE_JSON_OBJECT
;
3844 * ERROR: either we hit an unexpected character, or
3845 * we reached the end of the object or array without
3846 * finding the requested key.
3849 case DTRACE_JSON_IDENTIFIER
:
3856 dd
= dest
; /* reset string buffer */
3858 if (dtrace_strncmp(dest
, "true", 5) == 0 ||
3859 dtrace_strncmp(dest
, "false", 6) == 0 ||
3860 dtrace_strncmp(dest
, "null", 5) == 0) {
3864 * ERROR: We expected an object,
3865 * not this identifier.
3872 state
= DTRACE_JSON_COMMA
;
3878 * ERROR: we did not recognise the identifier as one
3879 * of those in the JSON specification.
3882 case DTRACE_JSON_NUMBER
:
3885 state
= DTRACE_JSON_NUMBER_FRAC
;
3889 if (cc
== 'x' || cc
== 'X') {
3891 * ERROR: specification explicitly excludes
3892 * hexidecimal or octal numbers.
3898 case DTRACE_JSON_NUMBER_FRAC
:
3899 if (cc
== 'e' || cc
== 'E') {
3901 state
= DTRACE_JSON_NUMBER_EXP
;
3905 if (cc
== '+' || cc
== '-') {
3907 * ERROR: expect sign as part of exponent only.
3912 case DTRACE_JSON_NUMBER_EXP
:
3913 if (isdigit(cc
) || cc
== '+' || cc
== '-') {
3919 dd
= dest
; /* reset string buffer */
3923 * ERROR: We expected an object, not
3932 state
= DTRACE_JSON_COMMA
;
3934 case DTRACE_JSON_VALUE
:
3938 if (cc
== '{' || cc
== '[') {
3939 if (nelems
> 1 && found_key
) {
3940 in_array
= cc
== '[' ? B_TRUE
: B_FALSE
;
3942 * If our element selector directs us
3943 * to descend into this nested object,
3944 * then move to the next selector
3945 * element in the list and restart the
3948 while (*elem
!= '\0')
3950 elem
++; /* skip the inter-element NUL */
3954 state
= DTRACE_JSON_VALUE
;
3956 array_elem
= dtrace_strtoll(
3958 found_key
= array_elem
== 0 ?
3961 found_key
= B_FALSE
;
3962 state
= DTRACE_JSON_OBJECT
;
3968 * Otherwise, we wish to either skip this
3969 * nested object or return it in full.
3976 state
= DTRACE_JSON_COLLECT_OBJECT
;
3981 state
= DTRACE_JSON_STRING
;
3987 * Here we deal with true, false and null.
3990 state
= DTRACE_JSON_IDENTIFIER
;
3994 if (cc
== '-' || isdigit(cc
)) {
3996 state
= DTRACE_JSON_NUMBER
;
4001 * ERROR: unexpected character at start of value.
4004 case DTRACE_JSON_COLLECT_OBJECT
:
4007 * ERROR: unexpected end of input.
4013 collect_object
= B_TRUE
;
4014 state
= DTRACE_JSON_STRING
;
4019 if (brackets
-- == 0) {
4021 * ERROR: unbalanced brackets.
4025 } else if (cc
== '}') {
4026 if (braces
-- == 0) {
4028 * ERROR: unbalanced braces.
4032 } else if (cc
== '{') {
4034 } else if (cc
== '[') {
4038 if (brackets
== 0 && braces
== 0) {
4043 dd
= dest
; /* reset string buffer */
4044 state
= DTRACE_JSON_COMMA
;
4053 * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
4054 * Notice that we don't bother validating the proper number of arguments or
4055 * their types in the tuple stack. This isn't needed because all argument
4056 * interpretation is safe because of our load safety -- the worst that can
4057 * happen is that a bogus program can obtain bogus results.
4060 dtrace_dif_subr(uint_t subr
, uint_t rd
, uint64_t *regs
,
4061 dtrace_key_t
*tupregs
, int nargs
,
4062 dtrace_mstate_t
*mstate
, dtrace_state_t
*state
)
4064 volatile uint16_t *flags
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
4065 volatile uintptr_t *illval
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
;
4066 dtrace_vstate_t
*vstate
= &state
->dts_vstate
;
4080 regs
[rd
] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
4083 case DIF_SUBR_MUTEX_OWNED
:
4084 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (kmutex_t
),
4090 m
.mx
= dtrace_load64(tupregs
[0].dttk_value
);
4091 if (MUTEX_TYPE_ADAPTIVE(&m
.mi
))
4092 regs
[rd
] = MUTEX_OWNER(&m
.mi
) != MUTEX_NO_OWNER
;
4094 regs
[rd
] = LOCK_HELD(&m
.mi
.m_spin
.m_spinlock
);
4097 case DIF_SUBR_MUTEX_OWNER
:
4098 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (kmutex_t
),
4104 m
.mx
= dtrace_load64(tupregs
[0].dttk_value
);
4105 if (MUTEX_TYPE_ADAPTIVE(&m
.mi
) &&
4106 MUTEX_OWNER(&m
.mi
) != MUTEX_NO_OWNER
)
4107 regs
[rd
] = (uintptr_t)MUTEX_OWNER(&m
.mi
);
4112 case DIF_SUBR_MUTEX_TYPE_ADAPTIVE
:
4113 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (kmutex_t
),
4119 m
.mx
= dtrace_load64(tupregs
[0].dttk_value
);
4120 regs
[rd
] = MUTEX_TYPE_ADAPTIVE(&m
.mi
);
4123 case DIF_SUBR_MUTEX_TYPE_SPIN
:
4124 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (kmutex_t
),
4130 m
.mx
= dtrace_load64(tupregs
[0].dttk_value
);
4131 regs
[rd
] = MUTEX_TYPE_SPIN(&m
.mi
);
4134 case DIF_SUBR_RW_READ_HELD
: {
4137 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (uintptr_t),
4143 r
.rw
= dtrace_loadptr(tupregs
[0].dttk_value
);
4144 regs
[rd
] = _RW_READ_HELD(&r
.ri
, tmp
);
4148 case DIF_SUBR_RW_WRITE_HELD
:
4149 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (krwlock_t
),
4155 r
.rw
= dtrace_loadptr(tupregs
[0].dttk_value
);
4156 regs
[rd
] = _RW_WRITE_HELD(&r
.ri
);
4159 case DIF_SUBR_RW_ISWRITER
:
4160 if (!dtrace_canload(tupregs
[0].dttk_value
, sizeof (krwlock_t
),
4166 r
.rw
= dtrace_loadptr(tupregs
[0].dttk_value
);
4167 regs
[rd
] = _RW_ISWRITER(&r
.ri
);
4170 case DIF_SUBR_BCOPY
: {
4172 * We need to be sure that the destination is in the scratch
4173 * region -- no other region is allowed.
4175 uintptr_t src
= tupregs
[0].dttk_value
;
4176 uintptr_t dest
= tupregs
[1].dttk_value
;
4177 size_t size
= tupregs
[2].dttk_value
;
4179 if (!dtrace_inscratch(dest
, size
, mstate
)) {
4180 *flags
|= CPU_DTRACE_BADADDR
;
4185 if (!dtrace_canload(src
, size
, mstate
, vstate
)) {
4190 dtrace_bcopy((void *)src
, (void *)dest
, size
);
4194 case DIF_SUBR_ALLOCA
:
4195 case DIF_SUBR_COPYIN
: {
4196 uintptr_t dest
= P2ROUNDUP(mstate
->dtms_scratch_ptr
, 8);
4198 tupregs
[subr
== DIF_SUBR_ALLOCA
? 0 : 1].dttk_value
;
4199 size_t scratch_size
= (dest
- mstate
->dtms_scratch_ptr
) + size
;
4202 * This action doesn't require any credential checks since
4203 * probes will not activate in user contexts to which the
4204 * enabling user does not have permissions.
4208 * Rounding up the user allocation size could have overflowed
4209 * a large, bogus allocation (like -1ULL) to 0.
4211 if (scratch_size
< size
||
4212 !DTRACE_INSCRATCH(mstate
, scratch_size
)) {
4213 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4218 if (subr
== DIF_SUBR_COPYIN
) {
4219 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4220 dtrace_copyin(tupregs
[0].dttk_value
, dest
, size
, flags
);
4221 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4224 mstate
->dtms_scratch_ptr
+= scratch_size
;
4229 case DIF_SUBR_COPYINTO
: {
4230 uint64_t size
= tupregs
[1].dttk_value
;
4231 uintptr_t dest
= tupregs
[2].dttk_value
;
4234 * This action doesn't require any credential checks since
4235 * probes will not activate in user contexts to which the
4236 * enabling user does not have permissions.
4238 if (!dtrace_inscratch(dest
, size
, mstate
)) {
4239 *flags
|= CPU_DTRACE_BADADDR
;
4244 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4245 dtrace_copyin(tupregs
[0].dttk_value
, dest
, size
, flags
);
4246 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4250 case DIF_SUBR_COPYINSTR
: {
4251 uintptr_t dest
= mstate
->dtms_scratch_ptr
;
4252 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4254 if (nargs
> 1 && tupregs
[1].dttk_value
< size
)
4255 size
= tupregs
[1].dttk_value
+ 1;
4258 * This action doesn't require any credential checks since
4259 * probes will not activate in user contexts to which the
4260 * enabling user does not have permissions.
4262 if (!DTRACE_INSCRATCH(mstate
, size
)) {
4263 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4268 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4269 dtrace_copyinstr(tupregs
[0].dttk_value
, dest
, size
, flags
);
4270 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4272 ((char *)dest
)[size
- 1] = '\0';
4273 mstate
->dtms_scratch_ptr
+= size
;
4278 case DIF_SUBR_MSGSIZE
:
4279 case DIF_SUBR_MSGDSIZE
: {
4280 uintptr_t baddr
= tupregs
[0].dttk_value
, daddr
;
4281 uintptr_t wptr
, rptr
;
4285 while (baddr
!= NULL
&& !(*flags
& CPU_DTRACE_FAULT
)) {
4287 if (!dtrace_canload(baddr
, sizeof (mblk_t
), mstate
,
4293 wptr
= dtrace_loadptr(baddr
+
4294 offsetof(mblk_t
, b_wptr
));
4296 rptr
= dtrace_loadptr(baddr
+
4297 offsetof(mblk_t
, b_rptr
));
4300 *flags
|= CPU_DTRACE_BADADDR
;
4301 *illval
= tupregs
[0].dttk_value
;
4305 daddr
= dtrace_loadptr(baddr
+
4306 offsetof(mblk_t
, b_datap
));
4308 baddr
= dtrace_loadptr(baddr
+
4309 offsetof(mblk_t
, b_cont
));
4312 * We want to prevent against denial-of-service here,
4313 * so we're only going to search the list for
4314 * dtrace_msgdsize_max mblks.
4316 if (cont
++ > dtrace_msgdsize_max
) {
4317 *flags
|= CPU_DTRACE_ILLOP
;
4321 if (subr
== DIF_SUBR_MSGDSIZE
) {
4322 if (dtrace_load8(daddr
+
4323 offsetof(dblk_t
, db_type
)) != M_DATA
)
4327 count
+= wptr
- rptr
;
4330 if (!(*flags
& CPU_DTRACE_FAULT
))
4336 case DIF_SUBR_PROGENYOF
: {
4337 pid_t pid
= tupregs
[0].dttk_value
;
4341 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4343 for (p
= curthread
->t_procp
; p
!= NULL
; p
= p
->p_parent
) {
4344 if (p
->p_pidp
->pid_id
== pid
) {
4350 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4356 case DIF_SUBR_SPECULATION
:
4357 regs
[rd
] = dtrace_speculation(state
);
4360 case DIF_SUBR_COPYOUT
: {
4361 uintptr_t kaddr
= tupregs
[0].dttk_value
;
4362 uintptr_t uaddr
= tupregs
[1].dttk_value
;
4363 uint64_t size
= tupregs
[2].dttk_value
;
4365 if (!dtrace_destructive_disallow
&&
4366 dtrace_priv_proc_control(state
, mstate
) &&
4367 !dtrace_istoxic(kaddr
, size
) &&
4368 dtrace_canload(kaddr
, size
, mstate
, vstate
)) {
4369 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4370 dtrace_copyout(kaddr
, uaddr
, size
, flags
);
4371 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4376 case DIF_SUBR_COPYOUTSTR
: {
4377 uintptr_t kaddr
= tupregs
[0].dttk_value
;
4378 uintptr_t uaddr
= tupregs
[1].dttk_value
;
4379 uint64_t size
= tupregs
[2].dttk_value
;
4382 if (!dtrace_destructive_disallow
&&
4383 dtrace_priv_proc_control(state
, mstate
) &&
4384 !dtrace_istoxic(kaddr
, size
) &&
4385 dtrace_strcanload(kaddr
, size
, &lim
, mstate
, vstate
)) {
4386 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
4387 dtrace_copyoutstr(kaddr
, uaddr
, lim
, flags
);
4388 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
4393 case DIF_SUBR_STRLEN
: {
4394 size_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4395 uintptr_t addr
= (uintptr_t)tupregs
[0].dttk_value
;
4398 if (!dtrace_strcanload(addr
, size
, &lim
, mstate
, vstate
)) {
4402 regs
[rd
] = dtrace_strlen((char *)addr
, lim
);
4407 case DIF_SUBR_STRCHR
:
4408 case DIF_SUBR_STRRCHR
: {
4410 * We're going to iterate over the string looking for the
4411 * specified character. We will iterate until we have reached
4412 * the string length or we have found the character. If this
4413 * is DIF_SUBR_STRRCHR, we will look for the last occurrence
4414 * of the specified character instead of the first.
4416 uintptr_t addr
= tupregs
[0].dttk_value
;
4417 uintptr_t addr_limit
;
4418 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4420 char c
, target
= (char)tupregs
[1].dttk_value
;
4422 if (!dtrace_strcanload(addr
, size
, &lim
, mstate
, vstate
)) {
4426 addr_limit
= addr
+ lim
;
4428 for (regs
[rd
] = NULL
; addr
< addr_limit
; addr
++) {
4429 if ((c
= dtrace_load8(addr
)) == target
) {
4432 if (subr
== DIF_SUBR_STRCHR
)
4442 case DIF_SUBR_STRSTR
:
4443 case DIF_SUBR_INDEX
:
4444 case DIF_SUBR_RINDEX
: {
4446 * We're going to iterate over the string looking for the
4447 * specified string. We will iterate until we have reached
4448 * the string length or we have found the string. (Yes, this
4449 * is done in the most naive way possible -- but considering
4450 * that the string we're searching for is likely to be
4451 * relatively short, the complexity of Rabin-Karp or similar
4452 * hardly seems merited.)
4454 char *addr
= (char *)(uintptr_t)tupregs
[0].dttk_value
;
4455 char *substr
= (char *)(uintptr_t)tupregs
[1].dttk_value
;
4456 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4457 size_t len
= dtrace_strlen(addr
, size
);
4458 size_t sublen
= dtrace_strlen(substr
, size
);
4459 char *limit
= addr
+ len
, *orig
= addr
;
4460 int notfound
= subr
== DIF_SUBR_STRSTR
? 0 : -1;
4463 regs
[rd
] = notfound
;
4465 if (!dtrace_canload((uintptr_t)addr
, len
+ 1, mstate
, vstate
)) {
4470 if (!dtrace_canload((uintptr_t)substr
, sublen
+ 1, mstate
,
4477 * strstr() and index()/rindex() have similar semantics if
4478 * both strings are the empty string: strstr() returns a
4479 * pointer to the (empty) string, and index() and rindex()
4480 * both return index 0 (regardless of any position argument).
4482 if (sublen
== 0 && len
== 0) {
4483 if (subr
== DIF_SUBR_STRSTR
)
4484 regs
[rd
] = (uintptr_t)addr
;
4490 if (subr
!= DIF_SUBR_STRSTR
) {
4491 if (subr
== DIF_SUBR_RINDEX
) {
4498 * Both index() and rindex() take an optional position
4499 * argument that denotes the starting position.
4502 int64_t pos
= (int64_t)tupregs
[2].dttk_value
;
4505 * If the position argument to index() is
4506 * negative, Perl implicitly clamps it at
4507 * zero. This semantic is a little surprising
4508 * given the special meaning of negative
4509 * positions to similar Perl functions like
4510 * substr(), but it appears to reflect a
4511 * notion that index() can start from a
4512 * negative index and increment its way up to
4513 * the string. Given this notion, Perl's
4514 * rindex() is at least self-consistent in
4515 * that it implicitly clamps positions greater
4516 * than the string length to be the string
4517 * length. Where Perl completely loses
4518 * coherence, however, is when the specified
4519 * substring is the empty string (""). In
4520 * this case, even if the position is
4521 * negative, rindex() returns 0 -- and even if
4522 * the position is greater than the length,
4523 * index() returns the string length. These
4524 * semantics violate the notion that index()
4525 * should never return a value less than the
4526 * specified position and that rindex() should
4527 * never return a value greater than the
4528 * specified position. (One assumes that
4529 * these semantics are artifacts of Perl's
4530 * implementation and not the results of
4531 * deliberate design -- it beggars belief that
4532 * even Larry Wall could desire such oddness.)
4533 * While in the abstract one would wish for
4534 * consistent position semantics across
4535 * substr(), index() and rindex() -- or at the
4536 * very least self-consistent position
4537 * semantics for index() and rindex() -- we
4538 * instead opt to keep with the extant Perl
4539 * semantics, in all their broken glory. (Do
4540 * we have more desire to maintain Perl's
4541 * semantics than Perl does? Probably.)
4543 if (subr
== DIF_SUBR_RINDEX
) {
4567 for (regs
[rd
] = notfound
; addr
!= limit
; addr
+= inc
) {
4568 if (dtrace_strncmp(addr
, substr
, sublen
) == 0) {
4569 if (subr
!= DIF_SUBR_STRSTR
) {
4571 * As D index() and rindex() are
4572 * modeled on Perl (and not on awk),
4573 * we return a zero-based (and not a
4574 * one-based) index. (For you Perl
4575 * weenies: no, we're not going to add
4576 * $[ -- and shouldn't you be at a con
4579 regs
[rd
] = (uintptr_t)(addr
- orig
);
4583 ASSERT(subr
== DIF_SUBR_STRSTR
);
4584 regs
[rd
] = (uintptr_t)addr
;
4592 case DIF_SUBR_STRTOK
: {
4593 uintptr_t addr
= tupregs
[0].dttk_value
;
4594 uintptr_t tokaddr
= tupregs
[1].dttk_value
;
4595 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4596 uintptr_t limit
, toklimit
;
4598 uint8_t c
, tokmap
[32]; /* 256 / 8 */
4599 char *dest
= (char *)mstate
->dtms_scratch_ptr
;
4603 * Check both the token buffer and (later) the input buffer,
4604 * since both could be non-scratch addresses.
4606 if (!dtrace_strcanload(tokaddr
, size
, &clim
, mstate
, vstate
)) {
4610 toklimit
= tokaddr
+ clim
;
4612 if (!DTRACE_INSCRATCH(mstate
, size
)) {
4613 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4620 * If the address specified is NULL, we use our saved
4621 * strtok pointer from the mstate. Note that this
4622 * means that the saved strtok pointer is _only_
4623 * valid within multiple enablings of the same probe --
4624 * it behaves like an implicit clause-local variable.
4626 addr
= mstate
->dtms_strtok
;
4627 limit
= mstate
->dtms_strtok_limit
;
4630 * If the user-specified address is non-NULL we must
4631 * access check it. This is the only time we have
4632 * a chance to do so, since this address may reside
4633 * in the string table of this clause-- future calls
4634 * (when we fetch addr from mstate->dtms_strtok)
4635 * would fail this access check.
4637 if (!dtrace_strcanload(addr
, size
, &clim
, mstate
,
4642 limit
= addr
+ clim
;
4646 * First, zero the token map, and then process the token
4647 * string -- setting a bit in the map for every character
4648 * found in the token string.
4650 for (i
= 0; i
< sizeof (tokmap
); i
++)
4653 for (; tokaddr
< toklimit
; tokaddr
++) {
4654 if ((c
= dtrace_load8(tokaddr
)) == '\0')
4657 ASSERT((c
>> 3) < sizeof (tokmap
));
4658 tokmap
[c
>> 3] |= (1 << (c
& 0x7));
4661 for (; addr
< limit
; addr
++) {
4663 * We're looking for a character that is _not_
4664 * contained in the token string.
4666 if ((c
= dtrace_load8(addr
)) == '\0')
4669 if (!(tokmap
[c
>> 3] & (1 << (c
& 0x7))))
4675 * We reached the end of the string without finding
4676 * any character that was not in the token string.
4677 * We return NULL in this case, and we set the saved
4678 * address to NULL as well.
4681 mstate
->dtms_strtok
= NULL
;
4682 mstate
->dtms_strtok_limit
= NULL
;
4687 * From here on, we're copying into the destination string.
4689 for (i
= 0; addr
< limit
&& i
< size
- 1; addr
++) {
4690 if ((c
= dtrace_load8(addr
)) == '\0')
4693 if (tokmap
[c
>> 3] & (1 << (c
& 0x7)))
4702 regs
[rd
] = (uintptr_t)dest
;
4703 mstate
->dtms_scratch_ptr
+= size
;
4704 mstate
->dtms_strtok
= addr
;
4705 mstate
->dtms_strtok_limit
= limit
;
4709 case DIF_SUBR_SUBSTR
: {
4710 uintptr_t s
= tupregs
[0].dttk_value
;
4711 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4712 char *d
= (char *)mstate
->dtms_scratch_ptr
;
4713 int64_t index
= (int64_t)tupregs
[1].dttk_value
;
4714 int64_t remaining
= (int64_t)tupregs
[2].dttk_value
;
4715 size_t len
= dtrace_strlen((char *)s
, size
);
4718 if (!dtrace_canload(s
, len
+ 1, mstate
, vstate
)) {
4723 if (!DTRACE_INSCRATCH(mstate
, size
)) {
4724 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4730 remaining
= (int64_t)size
;
4735 if (index
< 0 && index
+ remaining
> 0) {
4741 if (index
>= len
|| index
< 0) {
4743 } else if (remaining
< 0) {
4744 remaining
+= len
- index
;
4745 } else if (index
+ remaining
> size
) {
4746 remaining
= size
- index
;
4749 for (i
= 0; i
< remaining
; i
++) {
4750 if ((d
[i
] = dtrace_load8(s
+ index
+ i
)) == '\0')
4756 mstate
->dtms_scratch_ptr
+= size
;
4757 regs
[rd
] = (uintptr_t)d
;
4761 case DIF_SUBR_JSON
: {
4762 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4763 uintptr_t json
= tupregs
[0].dttk_value
;
4764 size_t jsonlen
= dtrace_strlen((char *)json
, size
);
4765 uintptr_t elem
= tupregs
[1].dttk_value
;
4766 size_t elemlen
= dtrace_strlen((char *)elem
, size
);
4768 char *dest
= (char *)mstate
->dtms_scratch_ptr
;
4769 char *elemlist
= (char *)mstate
->dtms_scratch_ptr
+ jsonlen
+ 1;
4770 char *ee
= elemlist
;
4774 if (!dtrace_canload(json
, jsonlen
+ 1, mstate
, vstate
) ||
4775 !dtrace_canload(elem
, elemlen
+ 1, mstate
, vstate
)) {
4780 if (!DTRACE_INSCRATCH(mstate
, jsonlen
+ 1 + elemlen
+ 1)) {
4781 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4787 * Read the element selector and split it up into a packed list
4790 for (cur
= elem
; cur
< elem
+ elemlen
; cur
++) {
4791 char cc
= dtrace_load8(cur
);
4793 if (cur
== elem
&& cc
== '[') {
4795 * If the first element selector key is
4796 * actually an array index then ignore the
4805 if (cc
== '.' || cc
== '[') {
4814 if ((regs
[rd
] = (uintptr_t)dtrace_json(size
, json
, elemlist
,
4815 nelems
, dest
)) != NULL
)
4816 mstate
->dtms_scratch_ptr
+= jsonlen
+ 1;
4820 case DIF_SUBR_TOUPPER
:
4821 case DIF_SUBR_TOLOWER
: {
4822 uintptr_t s
= tupregs
[0].dttk_value
;
4823 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4824 char *dest
= (char *)mstate
->dtms_scratch_ptr
, c
;
4825 size_t len
= dtrace_strlen((char *)s
, size
);
4826 char lower
, upper
, convert
;
4829 if (subr
== DIF_SUBR_TOUPPER
) {
4839 if (!dtrace_canload(s
, len
+ 1, mstate
, vstate
)) {
4844 if (!DTRACE_INSCRATCH(mstate
, size
)) {
4845 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4850 for (i
= 0; i
< size
- 1; i
++) {
4851 if ((c
= dtrace_load8(s
+ i
)) == '\0')
4854 if (c
>= lower
&& c
<= upper
)
4855 c
= convert
+ (c
- lower
);
4862 regs
[rd
] = (uintptr_t)dest
;
4863 mstate
->dtms_scratch_ptr
+= size
;
4867 case DIF_SUBR_GETMAJOR
:
4869 regs
[rd
] = (tupregs
[0].dttk_value
>> NBITSMINOR64
) & MAXMAJ64
;
4871 regs
[rd
] = (tupregs
[0].dttk_value
>> NBITSMINOR
) & MAXMAJ
;
4875 case DIF_SUBR_GETMINOR
:
4877 regs
[rd
] = tupregs
[0].dttk_value
& MAXMIN64
;
4879 regs
[rd
] = tupregs
[0].dttk_value
& MAXMIN
;
4883 case DIF_SUBR_DDI_PATHNAME
: {
4885 * This one is a galactic mess. We are going to roughly
4886 * emulate ddi_pathname(), but it's made more complicated
4887 * by the fact that we (a) want to include the minor name and
4888 * (b) must proceed iteratively instead of recursively.
4890 uintptr_t dest
= mstate
->dtms_scratch_ptr
;
4891 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
4892 char *start
= (char *)dest
, *end
= start
+ size
- 1;
4893 uintptr_t daddr
= tupregs
[0].dttk_value
;
4894 int64_t minor
= (int64_t)tupregs
[1].dttk_value
;
4896 int i
, len
, depth
= 0;
4899 * Due to all the pointer jumping we do and context we must
4900 * rely upon, we just mandate that the user must have kernel
4901 * read privileges to use this routine.
4903 if ((mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) == 0) {
4904 *flags
|= CPU_DTRACE_KPRIV
;
4909 if (!DTRACE_INSCRATCH(mstate
, size
)) {
4910 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
4918 * We want to have a name for the minor. In order to do this,
4919 * we need to walk the minor list from the devinfo. We want
4920 * to be sure that we don't infinitely walk a circular list,
4921 * so we check for circularity by sending a scout pointer
4922 * ahead two elements for every element that we iterate over;
4923 * if the list is circular, these will ultimately point to the
4924 * same element. You may recognize this little trick as the
4925 * answer to a stupid interview question -- one that always
4926 * seems to be asked by those who had to have it laboriously
4927 * explained to them, and who can't even concisely describe
4928 * the conditions under which one would be forced to resort to
4929 * this technique. Needless to say, those conditions are
4930 * found here -- and probably only here. Is this the only use
4931 * of this infamous trick in shipping, production code? If it
4932 * isn't, it probably should be...
4935 uintptr_t maddr
= dtrace_loadptr(daddr
+
4936 offsetof(struct dev_info
, devi_minor
));
4938 uintptr_t next
= offsetof(struct ddi_minor_data
, next
);
4939 uintptr_t name
= offsetof(struct ddi_minor_data
,
4940 d_minor
) + offsetof(struct ddi_minor
, name
);
4941 uintptr_t dev
= offsetof(struct ddi_minor_data
,
4942 d_minor
) + offsetof(struct ddi_minor
, dev
);
4946 scout
= dtrace_loadptr(maddr
+ next
);
4948 while (maddr
!= NULL
&& !(*flags
& CPU_DTRACE_FAULT
)) {
4951 m
= dtrace_load64(maddr
+ dev
) & MAXMIN64
;
4953 m
= dtrace_load32(maddr
+ dev
) & MAXMIN
;
4956 maddr
= dtrace_loadptr(maddr
+ next
);
4961 scout
= dtrace_loadptr(scout
+ next
);
4966 scout
= dtrace_loadptr(scout
+ next
);
4971 if (scout
== maddr
) {
4972 *flags
|= CPU_DTRACE_ILLOP
;
4980 * We have the minor data. Now we need to
4981 * copy the minor's name into the end of the
4984 s
= (char *)dtrace_loadptr(maddr
+ name
);
4985 len
= dtrace_strlen(s
, size
);
4987 if (*flags
& CPU_DTRACE_FAULT
)
4991 if ((end
-= (len
+ 1)) < start
)
4997 for (i
= 1; i
<= len
; i
++)
4998 end
[i
] = dtrace_load8((uintptr_t)s
++);
5003 while (daddr
!= NULL
&& !(*flags
& CPU_DTRACE_FAULT
)) {
5004 ddi_node_state_t devi_state
;
5006 devi_state
= dtrace_load32(daddr
+
5007 offsetof(struct dev_info
, devi_node_state
));
5009 if (*flags
& CPU_DTRACE_FAULT
)
5012 if (devi_state
>= DS_INITIALIZED
) {
5013 s
= (char *)dtrace_loadptr(daddr
+
5014 offsetof(struct dev_info
, devi_addr
));
5015 len
= dtrace_strlen(s
, size
);
5017 if (*flags
& CPU_DTRACE_FAULT
)
5021 if ((end
-= (len
+ 1)) < start
)
5027 for (i
= 1; i
<= len
; i
++)
5028 end
[i
] = dtrace_load8((uintptr_t)s
++);
5032 * Now for the node name...
5034 s
= (char *)dtrace_loadptr(daddr
+
5035 offsetof(struct dev_info
, devi_node_name
));
5037 daddr
= dtrace_loadptr(daddr
+
5038 offsetof(struct dev_info
, devi_parent
));
5041 * If our parent is NULL (that is, if we're the root
5042 * node), we're going to use the special path
5048 len
= dtrace_strlen(s
, size
);
5049 if (*flags
& CPU_DTRACE_FAULT
)
5052 if ((end
-= (len
+ 1)) < start
)
5055 for (i
= 1; i
<= len
; i
++)
5056 end
[i
] = dtrace_load8((uintptr_t)s
++);
5059 if (depth
++ > dtrace_devdepth_max
) {
5060 *flags
|= CPU_DTRACE_ILLOP
;
5066 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5068 if (daddr
== NULL
) {
5069 regs
[rd
] = (uintptr_t)end
;
5070 mstate
->dtms_scratch_ptr
+= size
;
5076 case DIF_SUBR_STRJOIN
: {
5077 char *d
= (char *)mstate
->dtms_scratch_ptr
;
5078 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
5079 uintptr_t s1
= tupregs
[0].dttk_value
;
5080 uintptr_t s2
= tupregs
[1].dttk_value
;
5085 if (!dtrace_strcanload(s1
, size
, &lim1
, mstate
, vstate
) ||
5086 !dtrace_strcanload(s2
, size
, &lim2
, mstate
, vstate
)) {
5091 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5092 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5099 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5103 c
= (i
>= lim1
) ? '\0' : dtrace_load8(s1
++);
5104 if ((d
[i
++] = c
) == '\0') {
5112 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5117 c
= (j
++ >= lim2
) ? '\0' : dtrace_load8(s2
++);
5118 if ((d
[i
++] = c
) == '\0')
5123 mstate
->dtms_scratch_ptr
+= i
;
5124 regs
[rd
] = (uintptr_t)d
;
5130 case DIF_SUBR_STRTOLL
: {
5131 uintptr_t s
= tupregs
[0].dttk_value
;
5132 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
5137 if ((base
= tupregs
[1].dttk_value
) <= 1 ||
5138 base
> ('z' - 'a' + 1) + ('9' - '0' + 1)) {
5139 *flags
|= CPU_DTRACE_ILLOP
;
5144 if (!dtrace_strcanload(s
, size
, &lim
, mstate
, vstate
)) {
5145 regs
[rd
] = INT64_MIN
;
5149 regs
[rd
] = dtrace_strtoll((char *)s
, base
, lim
);
5153 case DIF_SUBR_LLTOSTR
: {
5154 int64_t i
= (int64_t)tupregs
[0].dttk_value
;
5155 uint64_t val
, digit
;
5156 uint64_t size
= 65; /* enough room for 2^64 in binary */
5157 char *end
= (char *)mstate
->dtms_scratch_ptr
+ size
- 1;
5161 if ((base
= tupregs
[1].dttk_value
) <= 1 ||
5162 base
> ('z' - 'a' + 1) + ('9' - '0' + 1)) {
5163 *flags
|= CPU_DTRACE_ILLOP
;
5168 val
= (base
== 10 && i
< 0) ? i
* -1 : i
;
5170 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5171 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5176 for (*end
-- = '\0'; val
; val
/= base
) {
5177 if ((digit
= val
% base
) <= '9' - '0') {
5178 *end
-- = '0' + digit
;
5180 *end
-- = 'a' + (digit
- ('9' - '0') - 1);
5184 if (i
== 0 && base
== 16)
5190 if (i
== 0 || base
== 8 || base
== 16)
5193 if (i
< 0 && base
== 10)
5196 regs
[rd
] = (uintptr_t)end
+ 1;
5197 mstate
->dtms_scratch_ptr
+= size
;
5201 case DIF_SUBR_HTONS
:
5202 case DIF_SUBR_NTOHS
:
5204 regs
[rd
] = (uint16_t)tupregs
[0].dttk_value
;
5206 regs
[rd
] = DT_BSWAP_16((uint16_t)tupregs
[0].dttk_value
);
5211 case DIF_SUBR_HTONL
:
5212 case DIF_SUBR_NTOHL
:
5214 regs
[rd
] = (uint32_t)tupregs
[0].dttk_value
;
5216 regs
[rd
] = DT_BSWAP_32((uint32_t)tupregs
[0].dttk_value
);
5221 case DIF_SUBR_HTONLL
:
5222 case DIF_SUBR_NTOHLL
:
5224 regs
[rd
] = (uint64_t)tupregs
[0].dttk_value
;
5226 regs
[rd
] = DT_BSWAP_64((uint64_t)tupregs
[0].dttk_value
);
5231 case DIF_SUBR_DIRNAME
:
5232 case DIF_SUBR_BASENAME
: {
5233 char *dest
= (char *)mstate
->dtms_scratch_ptr
;
5234 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
5235 uintptr_t src
= tupregs
[0].dttk_value
;
5236 int i
, j
, len
= dtrace_strlen((char *)src
, size
);
5237 int lastbase
= -1, firstbase
= -1, lastdir
= -1;
5240 if (!dtrace_canload(src
, len
+ 1, mstate
, vstate
)) {
5245 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5246 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5252 * The basename and dirname for a zero-length string is
5257 src
= (uintptr_t)".";
5261 * Start from the back of the string, moving back toward the
5262 * front until we see a character that isn't a slash. That
5263 * character is the last character in the basename.
5265 for (i
= len
- 1; i
>= 0; i
--) {
5266 if (dtrace_load8(src
+ i
) != '/')
5274 * Starting from the last character in the basename, move
5275 * towards the front until we find a slash. The character
5276 * that we processed immediately before that is the first
5277 * character in the basename.
5279 for (; i
>= 0; i
--) {
5280 if (dtrace_load8(src
+ i
) == '/')
5288 * Now keep going until we find a non-slash character. That
5289 * character is the last character in the dirname.
5291 for (; i
>= 0; i
--) {
5292 if (dtrace_load8(src
+ i
) != '/')
5299 ASSERT(!(lastbase
== -1 && firstbase
!= -1));
5300 ASSERT(!(firstbase
== -1 && lastdir
!= -1));
5302 if (lastbase
== -1) {
5304 * We didn't find a non-slash character. We know that
5305 * the length is non-zero, so the whole string must be
5306 * slashes. In either the dirname or the basename
5307 * case, we return '/'.
5309 ASSERT(firstbase
== -1);
5310 firstbase
= lastbase
= lastdir
= 0;
5313 if (firstbase
== -1) {
5315 * The entire string consists only of a basename
5316 * component. If we're looking for dirname, we need
5317 * to change our string to be just "."; if we're
5318 * looking for a basename, we'll just set the first
5319 * character of the basename to be 0.
5321 if (subr
== DIF_SUBR_DIRNAME
) {
5322 ASSERT(lastdir
== -1);
5323 src
= (uintptr_t)".";
5330 if (subr
== DIF_SUBR_DIRNAME
) {
5331 if (lastdir
== -1) {
5333 * We know that we have a slash in the name --
5334 * or lastdir would be set to 0, above. And
5335 * because lastdir is -1, we know that this
5336 * slash must be the first character. (That
5337 * is, the full string must be of the form
5338 * "/basename".) In this case, the last
5339 * character of the directory name is 0.
5347 ASSERT(subr
== DIF_SUBR_BASENAME
);
5348 ASSERT(firstbase
!= -1 && lastbase
!= -1);
5353 for (i
= start
, j
= 0; i
<= end
&& j
< size
- 1; i
++, j
++)
5354 dest
[j
] = dtrace_load8(src
+ i
);
5357 regs
[rd
] = (uintptr_t)dest
;
5358 mstate
->dtms_scratch_ptr
+= size
;
5362 case DIF_SUBR_GETF
: {
5363 uintptr_t fd
= tupregs
[0].dttk_value
;
5364 uf_info_t
*finfo
= &curthread
->t_procp
->p_user
.u_finfo
;
5367 if (!dtrace_priv_proc(state
, mstate
)) {
5373 * This is safe because fi_nfiles only increases, and the
5374 * fi_list array is not freed when the array size doubles.
5375 * (See the comment in flist_grow() for details on the
5376 * management of the u_finfo structure.)
5378 fp
= fd
< finfo
->fi_nfiles
? finfo
->fi_list
[fd
].uf_file
: NULL
;
5380 mstate
->dtms_getf
= fp
;
5381 regs
[rd
] = (uintptr_t)fp
;
5385 case DIF_SUBR_CLEANPATH
: {
5386 char *dest
= (char *)mstate
->dtms_scratch_ptr
, c
;
5387 uint64_t size
= state
->dts_options
[DTRACEOPT_STRSIZE
];
5388 uintptr_t src
= tupregs
[0].dttk_value
;
5393 if (!dtrace_strcanload(src
, size
, &lim
, mstate
, vstate
)) {
5398 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5399 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5405 * Move forward, loading each character.
5408 c
= (i
>= lim
) ? '\0' : dtrace_load8(src
+ i
++);
5410 if (j
+ 5 >= size
) /* 5 = strlen("/..c\0") */
5418 c
= (i
>= lim
) ? '\0' : dtrace_load8(src
+ i
++);
5422 * We have two slashes -- we can just advance
5423 * to the next character.
5430 * This is not "." and it's not ".." -- we can
5431 * just store the "/" and this character and
5439 c
= (i
>= lim
) ? '\0' : dtrace_load8(src
+ i
++);
5443 * This is a "/./" component. We're not going
5444 * to store anything in the destination buffer;
5445 * we're just going to go to the next component.
5452 * This is not ".." -- we can just store the
5453 * "/." and this character and continue
5462 c
= (i
>= lim
) ? '\0' : dtrace_load8(src
+ i
++);
5464 if (c
!= '/' && c
!= '\0') {
5466 * This is not ".." -- it's "..[mumble]".
5467 * We'll store the "/.." and this character
5468 * and continue processing.
5478 * This is "/../" or "/..\0". We need to back up
5479 * our destination pointer until we find a "/".
5482 while (j
!= 0 && dest
[--j
] != '/')
5487 } while (c
!= '\0');
5491 if (mstate
->dtms_getf
!= NULL
&&
5492 !(mstate
->dtms_access
& DTRACE_ACCESS_KERNEL
) &&
5493 (z
= state
->dts_cred
.dcr_cred
->cr_zone
) != kcred
->cr_zone
) {
5495 * If we've done a getf() as a part of this ECB and we
5496 * don't have kernel access (and we're not in the global
5497 * zone), check if the path we cleaned up begins with
5498 * the zone's root path, and trim it off if so. Note
5499 * that this is an output cleanliness issue, not a
5500 * security issue: knowing one's zone root path does
5501 * not enable privilege escalation.
5503 if (strstr(dest
, z
->zone_rootpath
) == dest
)
5504 dest
+= strlen(z
->zone_rootpath
) - 1;
5507 regs
[rd
] = (uintptr_t)dest
;
5508 mstate
->dtms_scratch_ptr
+= size
;
5512 case DIF_SUBR_INET_NTOA
:
5513 case DIF_SUBR_INET_NTOA6
:
5514 case DIF_SUBR_INET_NTOP
: {
5519 if (subr
== DIF_SUBR_INET_NTOP
) {
5520 af
= (int)tupregs
[0].dttk_value
;
5523 af
= subr
== DIF_SUBR_INET_NTOA
? AF_INET
: AF_INET6
;
5527 if (af
== AF_INET
) {
5531 if (!dtrace_canload(tupregs
[argi
].dttk_value
,
5532 sizeof (ipaddr_t
), mstate
, vstate
)) {
5538 * Safely load the IPv4 address.
5540 ip4
= dtrace_load32(tupregs
[argi
].dttk_value
);
5543 * Check an IPv4 string will fit in scratch.
5545 size
= INET_ADDRSTRLEN
;
5546 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5547 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5551 base
= (char *)mstate
->dtms_scratch_ptr
;
5552 end
= (char *)mstate
->dtms_scratch_ptr
+ size
- 1;
5555 * Stringify as a dotted decimal quad.
5558 ptr8
= (uint8_t *)&ip4
;
5559 for (i
= 3; i
>= 0; i
--) {
5565 for (; val
; val
/= 10) {
5566 *end
-- = '0' + (val
% 10);
5573 ASSERT(end
+ 1 >= base
);
5575 } else if (af
== AF_INET6
) {
5576 struct in6_addr ip6
;
5577 int firstzero
, tryzero
, numzero
, v6end
;
5579 const char digits
[] = "0123456789abcdef";
5582 * Stringify using RFC 1884 convention 2 - 16 bit
5583 * hexadecimal values with a zero-run compression.
5584 * Lower case hexadecimal digits are used.
5585 * eg, fe80::214:4fff:fe0b:76c8.
5586 * The IPv4 embedded form is returned for inet_ntop,
5587 * just the IPv4 string is returned for inet_ntoa6.
5590 if (!dtrace_canload(tupregs
[argi
].dttk_value
,
5591 sizeof (struct in6_addr
), mstate
, vstate
)) {
5597 * Safely load the IPv6 address.
5600 (void *)(uintptr_t)tupregs
[argi
].dttk_value
,
5601 (void *)(uintptr_t)&ip6
, sizeof (struct in6_addr
));
5604 * Check an IPv6 string will fit in scratch.
5606 size
= INET6_ADDRSTRLEN
;
5607 if (!DTRACE_INSCRATCH(mstate
, size
)) {
5608 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
5612 base
= (char *)mstate
->dtms_scratch_ptr
;
5613 end
= (char *)mstate
->dtms_scratch_ptr
+ size
- 1;
5617 * Find the longest run of 16 bit zero values
5618 * for the single allowed zero compression - "::".
5623 for (i
= 0; i
< sizeof (struct in6_addr
); i
++) {
5624 if (ip6
._S6_un
._S6_u8
[i
] == 0 &&
5625 tryzero
== -1 && i
% 2 == 0) {
5630 if (tryzero
!= -1 &&
5631 (ip6
._S6_un
._S6_u8
[i
] != 0 ||
5632 i
== sizeof (struct in6_addr
) - 1)) {
5634 if (i
- tryzero
<= numzero
) {
5639 firstzero
= tryzero
;
5640 numzero
= i
- i
% 2 - tryzero
;
5643 if (ip6
._S6_un
._S6_u8
[i
] == 0 &&
5644 i
== sizeof (struct in6_addr
) - 1)
5648 ASSERT(firstzero
+ numzero
<= sizeof (struct in6_addr
));
5651 * Check for an IPv4 embedded address.
5653 v6end
= sizeof (struct in6_addr
) - 2;
5654 if (IN6_IS_ADDR_V4MAPPED(&ip6
) ||
5655 IN6_IS_ADDR_V4COMPAT(&ip6
)) {
5656 for (i
= sizeof (struct in6_addr
) - 1;
5657 i
>= DTRACE_V4MAPPED_OFFSET
; i
--) {
5658 ASSERT(end
>= base
);
5660 val
= ip6
._S6_un
._S6_u8
[i
];
5665 for (; val
; val
/= 10) {
5666 *end
-- = '0' + val
% 10;
5670 if (i
> DTRACE_V4MAPPED_OFFSET
)
5674 if (subr
== DIF_SUBR_INET_NTOA6
)
5678 * Set v6end to skip the IPv4 address that
5679 * we have already stringified.
5685 * Build the IPv6 string by working through the
5686 * address in reverse.
5688 for (i
= v6end
; i
>= 0; i
-= 2) {
5689 ASSERT(end
>= base
);
5691 if (i
== firstzero
+ numzero
- 2) {
5698 if (i
< 14 && i
!= firstzero
- 2)
5701 val
= (ip6
._S6_un
._S6_u8
[i
] << 8) +
5702 ip6
._S6_un
._S6_u8
[i
+ 1];
5707 for (; val
; val
/= 16) {
5708 *end
-- = digits
[val
% 16];
5712 ASSERT(end
+ 1 >= base
);
5716 * The user didn't use AH_INET or AH_INET6.
5718 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP
);
5723 inetout
: regs
[rd
] = (uintptr_t)end
+ 1;
5724 mstate
->dtms_scratch_ptr
+= size
;
5732 * Emulate the execution of DTrace IR instructions specified by the given
5733 * DIF object. This function is deliberately void of assertions as all of
5734 * the necessary checks are handled by a call to dtrace_difo_validate().
5737 dtrace_dif_emulate(dtrace_difo_t
*difo
, dtrace_mstate_t
*mstate
,
5738 dtrace_vstate_t
*vstate
, dtrace_state_t
*state
)
5740 const dif_instr_t
*text
= difo
->dtdo_buf
;
5741 const uint_t textlen
= difo
->dtdo_len
;
5742 const char *strtab
= difo
->dtdo_strtab
;
5743 const uint64_t *inttab
= difo
->dtdo_inttab
;
5746 dtrace_statvar_t
*svar
;
5747 dtrace_dstate_t
*dstate
= &vstate
->dtvs_dynvars
;
5749 volatile uint16_t *flags
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
5750 volatile uintptr_t *illval
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
;
5752 dtrace_key_t tupregs
[DIF_DTR_NREGS
+ 2]; /* +2 for thread and id */
5753 uint64_t regs
[DIF_DIR_NREGS
];
5756 uint8_t cc_n
= 0, cc_z
= 0, cc_v
= 0, cc_c
= 0;
5758 uint_t pc
= 0, id
, opc
;
5764 * We stash the current DIF object into the machine state: we need it
5765 * for subsequent access checking.
5767 mstate
->dtms_difo
= difo
;
5769 regs
[DIF_REG_R0
] = 0; /* %r0 is fixed at zero */
5771 while (pc
< textlen
&& !(*flags
& CPU_DTRACE_FAULT
)) {
5775 r1
= DIF_INSTR_R1(instr
);
5776 r2
= DIF_INSTR_R2(instr
);
5777 rd
= DIF_INSTR_RD(instr
);
5779 switch (DIF_INSTR_OP(instr
)) {
5781 regs
[rd
] = regs
[r1
] | regs
[r2
];
5784 regs
[rd
] = regs
[r1
] ^ regs
[r2
];
5787 regs
[rd
] = regs
[r1
] & regs
[r2
];
5790 regs
[rd
] = regs
[r1
] << regs
[r2
];
5793 regs
[rd
] = regs
[r1
] >> regs
[r2
];
5796 regs
[rd
] = regs
[r1
] - regs
[r2
];
5799 regs
[rd
] = regs
[r1
] + regs
[r2
];
5802 regs
[rd
] = regs
[r1
] * regs
[r2
];
5805 if (regs
[r2
] == 0) {
5807 *flags
|= CPU_DTRACE_DIVZERO
;
5809 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5810 regs
[rd
] = (int64_t)regs
[r1
] /
5812 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5817 if (regs
[r2
] == 0) {
5819 *flags
|= CPU_DTRACE_DIVZERO
;
5821 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5822 regs
[rd
] = regs
[r1
] / regs
[r2
];
5823 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5828 if (regs
[r2
] == 0) {
5830 *flags
|= CPU_DTRACE_DIVZERO
;
5832 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5833 regs
[rd
] = (int64_t)regs
[r1
] %
5835 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5840 if (regs
[r2
] == 0) {
5842 *flags
|= CPU_DTRACE_DIVZERO
;
5844 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5845 regs
[rd
] = regs
[r1
] % regs
[r2
];
5846 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5851 regs
[rd
] = ~regs
[r1
];
5854 regs
[rd
] = regs
[r1
];
5857 cc_r
= regs
[r1
] - regs
[r2
];
5861 cc_c
= regs
[r1
] < regs
[r2
];
5864 cc_n
= cc_v
= cc_c
= 0;
5865 cc_z
= regs
[r1
] == 0;
5868 pc
= DIF_INSTR_LABEL(instr
);
5872 pc
= DIF_INSTR_LABEL(instr
);
5876 pc
= DIF_INSTR_LABEL(instr
);
5879 if ((cc_z
| (cc_n
^ cc_v
)) == 0)
5880 pc
= DIF_INSTR_LABEL(instr
);
5883 if ((cc_c
| cc_z
) == 0)
5884 pc
= DIF_INSTR_LABEL(instr
);
5887 if ((cc_n
^ cc_v
) == 0)
5888 pc
= DIF_INSTR_LABEL(instr
);
5892 pc
= DIF_INSTR_LABEL(instr
);
5896 pc
= DIF_INSTR_LABEL(instr
);
5900 pc
= DIF_INSTR_LABEL(instr
);
5903 if (cc_z
| (cc_n
^ cc_v
))
5904 pc
= DIF_INSTR_LABEL(instr
);
5908 pc
= DIF_INSTR_LABEL(instr
);
5911 if (!dtrace_canload(regs
[r1
], 1, mstate
, vstate
))
5915 regs
[rd
] = (int8_t)dtrace_load8(regs
[r1
]);
5918 if (!dtrace_canload(regs
[r1
], 2, mstate
, vstate
))
5922 regs
[rd
] = (int16_t)dtrace_load16(regs
[r1
]);
5925 if (!dtrace_canload(regs
[r1
], 4, mstate
, vstate
))
5929 regs
[rd
] = (int32_t)dtrace_load32(regs
[r1
]);
5932 if (!dtrace_canload(regs
[r1
], 1, mstate
, vstate
))
5936 regs
[rd
] = dtrace_load8(regs
[r1
]);
5939 if (!dtrace_canload(regs
[r1
], 2, mstate
, vstate
))
5943 regs
[rd
] = dtrace_load16(regs
[r1
]);
5946 if (!dtrace_canload(regs
[r1
], 4, mstate
, vstate
))
5950 regs
[rd
] = dtrace_load32(regs
[r1
]);
5953 if (!dtrace_canload(regs
[r1
], 8, mstate
, vstate
))
5957 regs
[rd
] = dtrace_load64(regs
[r1
]);
5960 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5962 dtrace_fuword8((void *)(uintptr_t)regs
[r1
]);
5963 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5966 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5967 regs
[rd
] = (int16_t)
5968 dtrace_fuword16((void *)(uintptr_t)regs
[r1
]);
5969 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5972 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5973 regs
[rd
] = (int32_t)
5974 dtrace_fuword32((void *)(uintptr_t)regs
[r1
]);
5975 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5978 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5980 dtrace_fuword8((void *)(uintptr_t)regs
[r1
]);
5981 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5984 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5986 dtrace_fuword16((void *)(uintptr_t)regs
[r1
]);
5987 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5990 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5992 dtrace_fuword32((void *)(uintptr_t)regs
[r1
]);
5993 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
5996 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
5998 dtrace_fuword64((void *)(uintptr_t)regs
[r1
]);
5999 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
6008 regs
[rd
] = inttab
[DIF_INSTR_INTEGER(instr
)];
6011 regs
[rd
] = (uint64_t)(uintptr_t)
6012 (strtab
+ DIF_INSTR_STRING(instr
));
6015 size_t sz
= state
->dts_options
[DTRACEOPT_STRSIZE
];
6016 uintptr_t s1
= regs
[r1
];
6017 uintptr_t s2
= regs
[r2
];
6021 !dtrace_strcanload(s1
, sz
, &lim1
, mstate
, vstate
))
6024 !dtrace_strcanload(s2
, sz
, &lim2
, mstate
, vstate
))
6027 cc_r
= dtrace_strncmp((char *)s1
, (char *)s2
,
6036 regs
[rd
] = dtrace_dif_variable(mstate
, state
,
6040 id
= DIF_INSTR_VAR(instr
);
6042 if (id
>= DIF_VAR_OTHER_UBASE
) {
6045 id
-= DIF_VAR_OTHER_UBASE
;
6046 svar
= vstate
->dtvs_globals
[id
];
6047 ASSERT(svar
!= NULL
);
6048 v
= &svar
->dtsv_var
;
6050 if (!(v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
)) {
6051 regs
[rd
] = svar
->dtsv_data
;
6055 a
= (uintptr_t)svar
->dtsv_data
;
6057 if (*(uint8_t *)a
== UINT8_MAX
) {
6059 * If the 0th byte is set to UINT8_MAX
6060 * then this is to be treated as a
6061 * reference to a NULL variable.
6065 regs
[rd
] = a
+ sizeof (uint64_t);
6071 regs
[rd
] = dtrace_dif_variable(mstate
, state
, id
, 0);
6075 dtrace_dif_variable_write(mstate
, state
, r1
, regs
[r2
],
6080 id
= DIF_INSTR_VAR(instr
);
6082 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6083 id
-= DIF_VAR_OTHER_UBASE
;
6085 VERIFY(id
< vstate
->dtvs_nglobals
);
6086 svar
= vstate
->dtvs_globals
[id
];
6087 ASSERT(svar
!= NULL
);
6088 v
= &svar
->dtsv_var
;
6090 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6091 uintptr_t a
= (uintptr_t)svar
->dtsv_data
;
6095 ASSERT(svar
->dtsv_size
!= 0);
6097 if (regs
[rd
] == NULL
) {
6098 *(uint8_t *)a
= UINT8_MAX
;
6102 a
+= sizeof (uint64_t);
6104 if (!dtrace_vcanload(
6105 (void *)(uintptr_t)regs
[rd
], &v
->dtdv_type
,
6106 &lim
, mstate
, vstate
))
6109 dtrace_vcopy((void *)(uintptr_t)regs
[rd
],
6110 (void *)a
, &v
->dtdv_type
, lim
);
6114 svar
->dtsv_data
= regs
[rd
];
6119 * There are no DTrace built-in thread-local arrays at
6120 * present. This opcode is saved for future work.
6122 *flags
|= CPU_DTRACE_ILLOP
;
6127 id
= DIF_INSTR_VAR(instr
);
6129 if (id
< DIF_VAR_OTHER_UBASE
) {
6131 * For now, this has no meaning.
6137 id
-= DIF_VAR_OTHER_UBASE
;
6139 ASSERT(id
< vstate
->dtvs_nlocals
);
6140 ASSERT(vstate
->dtvs_locals
!= NULL
);
6142 svar
= vstate
->dtvs_locals
[id
];
6143 ASSERT(svar
!= NULL
);
6144 v
= &svar
->dtsv_var
;
6146 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6147 uintptr_t a
= (uintptr_t)svar
->dtsv_data
;
6148 size_t sz
= v
->dtdv_type
.dtdt_size
;
6150 sz
+= sizeof (uint64_t);
6151 ASSERT(svar
->dtsv_size
== NCPU
* sz
);
6152 a
+= CPU
->cpu_id
* sz
;
6154 if (*(uint8_t *)a
== UINT8_MAX
) {
6156 * If the 0th byte is set to UINT8_MAX
6157 * then this is to be treated as a
6158 * reference to a NULL variable.
6162 regs
[rd
] = a
+ sizeof (uint64_t);
6168 ASSERT(svar
->dtsv_size
== NCPU
* sizeof (uint64_t));
6169 tmp
= (uint64_t *)(uintptr_t)svar
->dtsv_data
;
6170 regs
[rd
] = tmp
[CPU
->cpu_id
];
6174 id
= DIF_INSTR_VAR(instr
);
6176 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6177 id
-= DIF_VAR_OTHER_UBASE
;
6178 VERIFY(id
< vstate
->dtvs_nlocals
);
6180 ASSERT(vstate
->dtvs_locals
!= NULL
);
6181 svar
= vstate
->dtvs_locals
[id
];
6182 ASSERT(svar
!= NULL
);
6183 v
= &svar
->dtsv_var
;
6185 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6186 uintptr_t a
= (uintptr_t)svar
->dtsv_data
;
6187 size_t sz
= v
->dtdv_type
.dtdt_size
;
6190 sz
+= sizeof (uint64_t);
6191 ASSERT(svar
->dtsv_size
== NCPU
* sz
);
6192 a
+= CPU
->cpu_id
* sz
;
6194 if (regs
[rd
] == NULL
) {
6195 *(uint8_t *)a
= UINT8_MAX
;
6199 a
+= sizeof (uint64_t);
6202 if (!dtrace_vcanload(
6203 (void *)(uintptr_t)regs
[rd
], &v
->dtdv_type
,
6204 &lim
, mstate
, vstate
))
6207 dtrace_vcopy((void *)(uintptr_t)regs
[rd
],
6208 (void *)a
, &v
->dtdv_type
, lim
);
6212 ASSERT(svar
->dtsv_size
== NCPU
* sizeof (uint64_t));
6213 tmp
= (uint64_t *)(uintptr_t)svar
->dtsv_data
;
6214 tmp
[CPU
->cpu_id
] = regs
[rd
];
6218 dtrace_dynvar_t
*dvar
;
6221 id
= DIF_INSTR_VAR(instr
);
6222 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6223 id
-= DIF_VAR_OTHER_UBASE
;
6224 v
= &vstate
->dtvs_tlocals
[id
];
6226 key
= &tupregs
[DIF_DTR_NREGS
];
6227 key
[0].dttk_value
= (uint64_t)id
;
6228 key
[0].dttk_size
= 0;
6229 DTRACE_TLS_THRKEY(key
[1].dttk_value
);
6230 key
[1].dttk_size
= 0;
6232 dvar
= dtrace_dynvar(dstate
, 2, key
,
6233 sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC
,
6241 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6242 regs
[rd
] = (uint64_t)(uintptr_t)dvar
->dtdv_data
;
6244 regs
[rd
] = *((uint64_t *)dvar
->dtdv_data
);
6251 dtrace_dynvar_t
*dvar
;
6254 id
= DIF_INSTR_VAR(instr
);
6255 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6256 id
-= DIF_VAR_OTHER_UBASE
;
6257 VERIFY(id
< vstate
->dtvs_ntlocals
);
6259 key
= &tupregs
[DIF_DTR_NREGS
];
6260 key
[0].dttk_value
= (uint64_t)id
;
6261 key
[0].dttk_size
= 0;
6262 DTRACE_TLS_THRKEY(key
[1].dttk_value
);
6263 key
[1].dttk_size
= 0;
6264 v
= &vstate
->dtvs_tlocals
[id
];
6266 dvar
= dtrace_dynvar(dstate
, 2, key
,
6267 v
->dtdv_type
.dtdt_size
> sizeof (uint64_t) ?
6268 v
->dtdv_type
.dtdt_size
: sizeof (uint64_t),
6269 regs
[rd
] ? DTRACE_DYNVAR_ALLOC
:
6270 DTRACE_DYNVAR_DEALLOC
, mstate
, vstate
);
6273 * Given that we're storing to thread-local data,
6274 * we need to flush our predicate cache.
6276 curthread
->t_predcache
= NULL
;
6281 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6284 if (!dtrace_vcanload(
6285 (void *)(uintptr_t)regs
[rd
],
6286 &v
->dtdv_type
, &lim
, mstate
, vstate
))
6289 dtrace_vcopy((void *)(uintptr_t)regs
[rd
],
6290 dvar
->dtdv_data
, &v
->dtdv_type
, lim
);
6292 *((uint64_t *)dvar
->dtdv_data
) = regs
[rd
];
6299 regs
[rd
] = (int64_t)regs
[r1
] >> regs
[r2
];
6303 dtrace_dif_subr(DIF_INSTR_SUBR(instr
), rd
,
6304 regs
, tupregs
, ttop
, mstate
, state
);
6308 if (ttop
== DIF_DTR_NREGS
) {
6309 *flags
|= CPU_DTRACE_TUPOFLOW
;
6313 if (r1
== DIF_TYPE_STRING
) {
6315 * If this is a string type and the size is 0,
6316 * we'll use the system-wide default string
6317 * size. Note that we are _not_ looking at
6318 * the value of the DTRACEOPT_STRSIZE option;
6319 * had this been set, we would expect to have
6320 * a non-zero size value in the "pushtr".
6322 tupregs
[ttop
].dttk_size
=
6323 dtrace_strlen((char *)(uintptr_t)regs
[rd
],
6324 regs
[r2
] ? regs
[r2
] :
6325 dtrace_strsize_default
) + 1;
6327 if (regs
[r2
] > LONG_MAX
) {
6328 *flags
|= CPU_DTRACE_ILLOP
;
6332 tupregs
[ttop
].dttk_size
= regs
[r2
];
6335 tupregs
[ttop
++].dttk_value
= regs
[rd
];
6339 if (ttop
== DIF_DTR_NREGS
) {
6340 *flags
|= CPU_DTRACE_TUPOFLOW
;
6344 tupregs
[ttop
].dttk_value
= regs
[rd
];
6345 tupregs
[ttop
++].dttk_size
= 0;
6353 case DIF_OP_FLUSHTS
:
6358 case DIF_OP_LDTAA
: {
6359 dtrace_dynvar_t
*dvar
;
6360 dtrace_key_t
*key
= tupregs
;
6361 uint_t nkeys
= ttop
;
6363 id
= DIF_INSTR_VAR(instr
);
6364 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6365 id
-= DIF_VAR_OTHER_UBASE
;
6367 key
[nkeys
].dttk_value
= (uint64_t)id
;
6368 key
[nkeys
++].dttk_size
= 0;
6370 if (DIF_INSTR_OP(instr
) == DIF_OP_LDTAA
) {
6371 DTRACE_TLS_THRKEY(key
[nkeys
].dttk_value
);
6372 key
[nkeys
++].dttk_size
= 0;
6373 VERIFY(id
< vstate
->dtvs_ntlocals
);
6374 v
= &vstate
->dtvs_tlocals
[id
];
6376 VERIFY(id
< vstate
->dtvs_nglobals
);
6377 v
= &vstate
->dtvs_globals
[id
]->dtsv_var
;
6380 dvar
= dtrace_dynvar(dstate
, nkeys
, key
,
6381 v
->dtdv_type
.dtdt_size
> sizeof (uint64_t) ?
6382 v
->dtdv_type
.dtdt_size
: sizeof (uint64_t),
6383 DTRACE_DYNVAR_NOALLOC
, mstate
, vstate
);
6390 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6391 regs
[rd
] = (uint64_t)(uintptr_t)dvar
->dtdv_data
;
6393 regs
[rd
] = *((uint64_t *)dvar
->dtdv_data
);
6400 case DIF_OP_STTAA
: {
6401 dtrace_dynvar_t
*dvar
;
6402 dtrace_key_t
*key
= tupregs
;
6403 uint_t nkeys
= ttop
;
6405 id
= DIF_INSTR_VAR(instr
);
6406 ASSERT(id
>= DIF_VAR_OTHER_UBASE
);
6407 id
-= DIF_VAR_OTHER_UBASE
;
6409 key
[nkeys
].dttk_value
= (uint64_t)id
;
6410 key
[nkeys
++].dttk_size
= 0;
6412 if (DIF_INSTR_OP(instr
) == DIF_OP_STTAA
) {
6413 DTRACE_TLS_THRKEY(key
[nkeys
].dttk_value
);
6414 key
[nkeys
++].dttk_size
= 0;
6415 VERIFY(id
< vstate
->dtvs_ntlocals
);
6416 v
= &vstate
->dtvs_tlocals
[id
];
6418 VERIFY(id
< vstate
->dtvs_nglobals
);
6419 v
= &vstate
->dtvs_globals
[id
]->dtsv_var
;
6422 dvar
= dtrace_dynvar(dstate
, nkeys
, key
,
6423 v
->dtdv_type
.dtdt_size
> sizeof (uint64_t) ?
6424 v
->dtdv_type
.dtdt_size
: sizeof (uint64_t),
6425 regs
[rd
] ? DTRACE_DYNVAR_ALLOC
:
6426 DTRACE_DYNVAR_DEALLOC
, mstate
, vstate
);
6431 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
) {
6434 if (!dtrace_vcanload(
6435 (void *)(uintptr_t)regs
[rd
], &v
->dtdv_type
,
6436 &lim
, mstate
, vstate
))
6439 dtrace_vcopy((void *)(uintptr_t)regs
[rd
],
6440 dvar
->dtdv_data
, &v
->dtdv_type
, lim
);
6442 *((uint64_t *)dvar
->dtdv_data
) = regs
[rd
];
6448 case DIF_OP_ALLOCS
: {
6449 uintptr_t ptr
= P2ROUNDUP(mstate
->dtms_scratch_ptr
, 8);
6450 size_t size
= ptr
- mstate
->dtms_scratch_ptr
+ regs
[r1
];
6453 * Rounding up the user allocation size could have
6454 * overflowed large, bogus allocations (like -1ULL) to
6457 if (size
< regs
[r1
] ||
6458 !DTRACE_INSCRATCH(mstate
, size
)) {
6459 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
6464 dtrace_bzero((void *) mstate
->dtms_scratch_ptr
, size
);
6465 mstate
->dtms_scratch_ptr
+= size
;
6471 if (!dtrace_canstore(regs
[rd
], regs
[r2
],
6473 *flags
|= CPU_DTRACE_BADADDR
;
6478 if (!dtrace_canload(regs
[r1
], regs
[r2
], mstate
, vstate
))
6481 dtrace_bcopy((void *)(uintptr_t)regs
[r1
],
6482 (void *)(uintptr_t)regs
[rd
], (size_t)regs
[r2
]);
6486 if (!dtrace_canstore(regs
[rd
], 1, mstate
, vstate
)) {
6487 *flags
|= CPU_DTRACE_BADADDR
;
6491 *((uint8_t *)(uintptr_t)regs
[rd
]) = (uint8_t)regs
[r1
];
6495 if (!dtrace_canstore(regs
[rd
], 2, mstate
, vstate
)) {
6496 *flags
|= CPU_DTRACE_BADADDR
;
6501 *flags
|= CPU_DTRACE_BADALIGN
;
6505 *((uint16_t *)(uintptr_t)regs
[rd
]) = (uint16_t)regs
[r1
];
6509 if (!dtrace_canstore(regs
[rd
], 4, mstate
, vstate
)) {
6510 *flags
|= CPU_DTRACE_BADADDR
;
6515 *flags
|= CPU_DTRACE_BADALIGN
;
6519 *((uint32_t *)(uintptr_t)regs
[rd
]) = (uint32_t)regs
[r1
];
6523 if (!dtrace_canstore(regs
[rd
], 8, mstate
, vstate
)) {
6524 *flags
|= CPU_DTRACE_BADADDR
;
6529 *flags
|= CPU_DTRACE_BADALIGN
;
6533 *((uint64_t *)(uintptr_t)regs
[rd
]) = regs
[r1
];
6538 if (!(*flags
& CPU_DTRACE_FAULT
))
6541 mstate
->dtms_fltoffs
= opc
* sizeof (dif_instr_t
);
6542 mstate
->dtms_present
|= DTRACE_MSTATE_FLTOFFS
;
6548 dtrace_action_breakpoint(dtrace_ecb_t
*ecb
)
6550 dtrace_probe_t
*probe
= ecb
->dte_probe
;
6551 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
6552 char c
[DTRACE_FULLNAMELEN
+ 80], *str
;
6553 char *msg
= "dtrace: breakpoint action at probe ";
6554 char *ecbmsg
= " (ecb ";
6555 uintptr_t mask
= (0xf << (sizeof (uintptr_t) * NBBY
/ 4));
6556 uintptr_t val
= (uintptr_t)ecb
;
6557 int shift
= (sizeof (uintptr_t) * NBBY
) - 4, i
= 0;
6559 if (dtrace_destructive_disallow
)
6563 * It's impossible to be taking action on the NULL probe.
6565 ASSERT(probe
!= NULL
);
6568 * This is a poor man's (destitute man's?) sprintf(): we want to
6569 * print the provider name, module name, function name and name of
6570 * the probe, along with the hex address of the ECB with the breakpoint
6571 * action -- all of which we must place in the character buffer by
6574 while (*msg
!= '\0')
6577 for (str
= prov
->dtpv_name
; *str
!= '\0'; str
++)
6581 for (str
= probe
->dtpr_mod
; *str
!= '\0'; str
++)
6585 for (str
= probe
->dtpr_func
; *str
!= '\0'; str
++)
6589 for (str
= probe
->dtpr_name
; *str
!= '\0'; str
++)
6592 while (*ecbmsg
!= '\0')
6595 while (shift
>= 0) {
6596 mask
= (uintptr_t)0xf << shift
;
6598 if (val
>= ((uintptr_t)1 << shift
))
6599 c
[i
++] = "0123456789abcdef"[(val
& mask
) >> shift
];
6610 dtrace_action_panic(dtrace_ecb_t
*ecb
)
6612 dtrace_probe_t
*probe
= ecb
->dte_probe
;
6615 * It's impossible to be taking action on the NULL probe.
6617 ASSERT(probe
!= NULL
);
6619 if (dtrace_destructive_disallow
)
6622 if (dtrace_panicked
!= NULL
)
6625 if (dtrace_casptr(&dtrace_panicked
, NULL
, curthread
) != NULL
)
6629 * We won the right to panic. (We want to be sure that only one
6630 * thread calls panic() from dtrace_probe(), and that panic() is
6631 * called exactly once.)
6633 dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
6634 probe
->dtpr_provider
->dtpv_name
, probe
->dtpr_mod
,
6635 probe
->dtpr_func
, probe
->dtpr_name
, (void *)ecb
);
6639 dtrace_action_raise(uint64_t sig
)
6641 if (dtrace_destructive_disallow
)
6645 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP
);
6650 * raise() has a queue depth of 1 -- we ignore all subsequent
6651 * invocations of the raise() action.
6653 if (curthread
->t_dtrace_sig
== 0)
6654 curthread
->t_dtrace_sig
= (uint8_t)sig
;
6656 curthread
->t_sig_check
= 1;
6661 dtrace_action_stop(void)
6663 if (dtrace_destructive_disallow
)
6666 if (!curthread
->t_dtrace_stop
) {
6667 curthread
->t_dtrace_stop
= 1;
6668 curthread
->t_sig_check
= 1;
6674 dtrace_action_chill(dtrace_mstate_t
*mstate
, hrtime_t val
)
6677 volatile uint16_t *flags
;
6680 if (dtrace_destructive_disallow
)
6683 flags
= (volatile uint16_t *)&cpu_core
[cpu
->cpu_id
].cpuc_dtrace_flags
;
6685 now
= dtrace_gethrtime();
6687 if (now
- cpu
->cpu_dtrace_chillmark
> dtrace_chill_interval
) {
6689 * We need to advance the mark to the current time.
6691 cpu
->cpu_dtrace_chillmark
= now
;
6692 cpu
->cpu_dtrace_chilled
= 0;
6696 * Now check to see if the requested chill time would take us over
6697 * the maximum amount of time allowed in the chill interval. (Or
6698 * worse, if the calculation itself induces overflow.)
6700 if (cpu
->cpu_dtrace_chilled
+ val
> dtrace_chill_max
||
6701 cpu
->cpu_dtrace_chilled
+ val
< cpu
->cpu_dtrace_chilled
) {
6702 *flags
|= CPU_DTRACE_ILLOP
;
6706 while (dtrace_gethrtime() - now
< val
)
6710 * Normally, we assure that the value of the variable "timestamp" does
6711 * not change within an ECB. The presence of chill() represents an
6712 * exception to this rule, however.
6714 mstate
->dtms_present
&= ~DTRACE_MSTATE_TIMESTAMP
;
6715 cpu
->cpu_dtrace_chilled
+= val
;
6719 dtrace_action_ustack(dtrace_mstate_t
*mstate
, dtrace_state_t
*state
,
6720 uint64_t *buf
, uint64_t arg
)
6722 int nframes
= DTRACE_USTACK_NFRAMES(arg
);
6723 int strsize
= DTRACE_USTACK_STRSIZE(arg
);
6724 uint64_t *pcs
= &buf
[1], *fps
;
6725 char *str
= (char *)&pcs
[nframes
];
6726 int size
, offs
= 0, i
, j
;
6728 uintptr_t old
= mstate
->dtms_scratch_ptr
, saved
;
6729 uint16_t *flags
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
6733 * Should be taking a faster path if string space has not been
6736 ASSERT(strsize
!= 0);
6739 * We will first allocate some temporary space for the frame pointers.
6741 fps
= (uint64_t *)P2ROUNDUP(mstate
->dtms_scratch_ptr
, 8);
6742 size
= (uintptr_t)fps
- mstate
->dtms_scratch_ptr
+
6743 (nframes
* sizeof (uint64_t));
6745 if (!DTRACE_INSCRATCH(mstate
, size
)) {
6747 * Not enough room for our frame pointers -- need to indicate
6748 * that we ran out of scratch space.
6750 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH
);
6754 mstate
->dtms_scratch_ptr
+= size
;
6755 saved
= mstate
->dtms_scratch_ptr
;
6758 * Now get a stack with both program counters and frame pointers.
6760 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
6761 dtrace_getufpstack(buf
, fps
, nframes
+ 1);
6762 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
6765 * If that faulted, we're cooked.
6767 if (*flags
& CPU_DTRACE_FAULT
)
6771 * Now we want to walk up the stack, calling the USTACK helper. For
6772 * each iteration, we restore the scratch pointer.
6774 for (i
= 0; i
< nframes
; i
++) {
6775 mstate
->dtms_scratch_ptr
= saved
;
6777 if (offs
>= strsize
)
6780 sym
= (char *)(uintptr_t)dtrace_helper(
6781 DTRACE_HELPER_ACTION_USTACK
,
6782 mstate
, state
, pcs
[i
], fps
[i
]);
6785 * If we faulted while running the helper, we're going to
6786 * clear the fault and null out the corresponding string.
6788 if (*flags
& CPU_DTRACE_FAULT
) {
6789 *flags
&= ~CPU_DTRACE_FAULT
;
6799 if (!dtrace_strcanload((uintptr_t)sym
, strsize
, &rem
, mstate
,
6800 &(state
->dts_vstate
))) {
6805 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
6808 * Now copy in the string that the helper returned to us.
6810 for (j
= 0; offs
+ j
< strsize
&& j
< rem
; j
++) {
6811 if ((str
[offs
+ j
] = sym
[j
]) == '\0')
6815 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
6820 if (offs
>= strsize
) {
6822 * If we didn't have room for all of the strings, we don't
6823 * abort processing -- this needn't be a fatal error -- but we
6824 * still want to increment a counter (dts_stkstroverflows) to
6825 * allow this condition to be warned about. (If this is from
6826 * a jstack() action, it is easily tuned via jstackstrsize.)
6828 dtrace_error(&state
->dts_stkstroverflows
);
6831 while (offs
< strsize
)
6835 mstate
->dtms_scratch_ptr
= old
;
6839 dtrace_store_by_ref(dtrace_difo_t
*dp
, caddr_t tomax
, size_t size
,
6840 size_t *valoffsp
, uint64_t *valp
, uint64_t end
, int intuple
, int dtkind
)
6842 volatile uint16_t *flags
;
6843 uint64_t val
= *valp
;
6844 size_t valoffs
= *valoffsp
;
6846 flags
= (volatile uint16_t *)&cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
6847 ASSERT(dtkind
== DIF_TF_BYREF
|| dtkind
== DIF_TF_BYUREF
);
6850 * If this is a string, we're going to only load until we find the zero
6851 * byte -- after which we'll store zero bytes.
6853 if (dp
->dtdo_rtype
.dtdt_kind
== DIF_TYPE_STRING
) {
6857 for (s
= 0; s
< size
; s
++) {
6858 if (c
!= '\0' && dtkind
== DIF_TF_BYREF
) {
6859 c
= dtrace_load8(val
++);
6860 } else if (c
!= '\0' && dtkind
== DIF_TF_BYUREF
) {
6861 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
6862 c
= dtrace_fuword8((void *)(uintptr_t)val
++);
6863 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
6864 if (*flags
& CPU_DTRACE_FAULT
)
6868 DTRACE_STORE(uint8_t, tomax
, valoffs
++, c
);
6870 if (c
== '\0' && intuple
)
6875 while (valoffs
< end
) {
6876 if (dtkind
== DIF_TF_BYREF
) {
6877 c
= dtrace_load8(val
++);
6878 } else if (dtkind
== DIF_TF_BYUREF
) {
6879 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
6880 c
= dtrace_fuword8((void *)(uintptr_t)val
++);
6881 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
6882 if (*flags
& CPU_DTRACE_FAULT
)
6886 DTRACE_STORE(uint8_t, tomax
,
6892 *valoffsp
= valoffs
;
6896 * If you're looking for the epicenter of DTrace, you just found it. This
6897 * is the function called by the provider to fire a probe -- from which all
6898 * subsequent probe-context DTrace activity emanates.
6901 dtrace_probe(dtrace_id_t id
, uintptr_t arg0
, uintptr_t arg1
,
6902 uintptr_t arg2
, uintptr_t arg3
, uintptr_t arg4
)
6904 processorid_t cpuid
;
6905 dtrace_icookie_t cookie
;
6906 dtrace_probe_t
*probe
;
6907 dtrace_mstate_t mstate
;
6909 dtrace_action_t
*act
;
6913 volatile uint16_t *flags
;
6917 * Kick out immediately if this CPU is still being born (in which case
6918 * curthread will be set to -1) or the current thread can't allow
6919 * probes in its current context.
6921 if (((uintptr_t)curthread
& 1) || (curthread
->t_flag
& T_DONTDTRACE
))
6924 cookie
= dtrace_interrupt_disable();
6925 probe
= dtrace_probes
[id
- 1];
6926 cpuid
= CPU
->cpu_id
;
6927 onintr
= CPU_ON_INTR(CPU
);
6929 CPU
->cpu_dtrace_probes
++;
6931 if (!onintr
&& probe
->dtpr_predcache
!= DTRACE_CACHEIDNONE
&&
6932 probe
->dtpr_predcache
== curthread
->t_predcache
) {
6934 * We have hit in the predicate cache; we know that
6935 * this predicate would evaluate to be false.
6937 dtrace_interrupt_enable(cookie
);
6941 if (panic_quiesce
) {
6943 * We don't trace anything if we're panicking.
6945 dtrace_interrupt_enable(cookie
);
6949 now
= mstate
.dtms_timestamp
= dtrace_gethrtime();
6950 mstate
.dtms_present
|= DTRACE_MSTATE_TIMESTAMP
;
6951 vtime
= dtrace_vtime_references
!= 0;
6953 if (vtime
&& curthread
->t_dtrace_start
)
6954 curthread
->t_dtrace_vtime
+= now
- curthread
->t_dtrace_start
;
6956 mstate
.dtms_difo
= NULL
;
6957 mstate
.dtms_probe
= probe
;
6958 mstate
.dtms_strtok
= NULL
;
6959 mstate
.dtms_arg
[0] = arg0
;
6960 mstate
.dtms_arg
[1] = arg1
;
6961 mstate
.dtms_arg
[2] = arg2
;
6962 mstate
.dtms_arg
[3] = arg3
;
6963 mstate
.dtms_arg
[4] = arg4
;
6965 flags
= (volatile uint16_t *)&cpu_core
[cpuid
].cpuc_dtrace_flags
;
6967 for (ecb
= probe
->dtpr_ecb
; ecb
!= NULL
; ecb
= ecb
->dte_next
) {
6968 dtrace_predicate_t
*pred
= ecb
->dte_predicate
;
6969 dtrace_state_t
*state
= ecb
->dte_state
;
6970 dtrace_buffer_t
*buf
= &state
->dts_buffer
[cpuid
];
6971 dtrace_buffer_t
*aggbuf
= &state
->dts_aggbuffer
[cpuid
];
6972 dtrace_vstate_t
*vstate
= &state
->dts_vstate
;
6973 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
6974 uint64_t tracememsize
= 0;
6979 * A little subtlety with the following (seemingly innocuous)
6980 * declaration of the automatic 'val': by looking at the
6981 * code, you might think that it could be declared in the
6982 * action processing loop, below. (That is, it's only used in
6983 * the action processing loop.) However, it must be declared
6984 * out of that scope because in the case of DIF expression
6985 * arguments to aggregating actions, one iteration of the
6986 * action loop will use the last iteration's value.
6994 mstate
.dtms_present
= DTRACE_MSTATE_ARGS
| DTRACE_MSTATE_PROBE
;
6995 mstate
.dtms_access
= DTRACE_ACCESS_ARGS
| DTRACE_ACCESS_PROC
;
6996 mstate
.dtms_getf
= NULL
;
6998 *flags
&= ~CPU_DTRACE_ERROR
;
7000 if (prov
== dtrace_provider
) {
7002 * If dtrace itself is the provider of this probe,
7003 * we're only going to continue processing the ECB if
7004 * arg0 (the dtrace_state_t) is equal to the ECB's
7005 * creating state. (This prevents disjoint consumers
7006 * from seeing one another's metaprobes.)
7008 if (arg0
!= (uint64_t)(uintptr_t)state
)
7012 if (state
->dts_activity
!= DTRACE_ACTIVITY_ACTIVE
) {
7014 * We're not currently active. If our provider isn't
7015 * the dtrace pseudo provider, we're not interested.
7017 if (prov
!= dtrace_provider
)
7021 * Now we must further check if we are in the BEGIN
7022 * probe. If we are, we will only continue processing
7023 * if we're still in WARMUP -- if one BEGIN enabling
7024 * has invoked the exit() action, we don't want to
7025 * evaluate subsequent BEGIN enablings.
7027 if (probe
->dtpr_id
== dtrace_probeid_begin
&&
7028 state
->dts_activity
!= DTRACE_ACTIVITY_WARMUP
) {
7029 ASSERT(state
->dts_activity
==
7030 DTRACE_ACTIVITY_DRAINING
);
7035 if (ecb
->dte_cond
&& !dtrace_priv_probe(state
, &mstate
, ecb
))
7038 if (now
- state
->dts_alive
> dtrace_deadman_timeout
) {
7040 * We seem to be dead. Unless we (a) have kernel
7041 * destructive permissions (b) have explicitly enabled
7042 * destructive actions and (c) destructive actions have
7043 * not been disabled, we're going to transition into
7044 * the KILLED state, from which no further processing
7045 * on this state will be performed.
7047 if (!dtrace_priv_kernel_destructive(state
) ||
7048 !state
->dts_cred
.dcr_destructive
||
7049 dtrace_destructive_disallow
) {
7050 void *activity
= &state
->dts_activity
;
7051 dtrace_activity_t current
;
7054 current
= state
->dts_activity
;
7055 } while (dtrace_cas32(activity
, current
,
7056 DTRACE_ACTIVITY_KILLED
) != current
);
7062 if ((offs
= dtrace_buffer_reserve(buf
, ecb
->dte_needed
,
7063 ecb
->dte_alignment
, state
, &mstate
)) < 0)
7066 tomax
= buf
->dtb_tomax
;
7067 ASSERT(tomax
!= NULL
);
7069 if (ecb
->dte_size
!= 0) {
7070 dtrace_rechdr_t dtrh
;
7071 if (!(mstate
.dtms_present
& DTRACE_MSTATE_TIMESTAMP
)) {
7072 mstate
.dtms_timestamp
= dtrace_gethrtime();
7073 mstate
.dtms_present
|= DTRACE_MSTATE_TIMESTAMP
;
7075 ASSERT3U(ecb
->dte_size
, >=, sizeof (dtrace_rechdr_t
));
7076 dtrh
.dtrh_epid
= ecb
->dte_epid
;
7077 DTRACE_RECORD_STORE_TIMESTAMP(&dtrh
,
7078 mstate
.dtms_timestamp
);
7079 *((dtrace_rechdr_t
*)(tomax
+ offs
)) = dtrh
;
7082 mstate
.dtms_epid
= ecb
->dte_epid
;
7083 mstate
.dtms_present
|= DTRACE_MSTATE_EPID
;
7085 if (state
->dts_cred
.dcr_visible
& DTRACE_CRV_KERNEL
)
7086 mstate
.dtms_access
|= DTRACE_ACCESS_KERNEL
;
7089 dtrace_difo_t
*dp
= pred
->dtp_difo
;
7092 rval
= dtrace_dif_emulate(dp
, &mstate
, vstate
, state
);
7094 if (!(*flags
& CPU_DTRACE_ERROR
) && !rval
) {
7095 dtrace_cacheid_t cid
= probe
->dtpr_predcache
;
7097 if (cid
!= DTRACE_CACHEIDNONE
&& !onintr
) {
7099 * Update the predicate cache...
7101 ASSERT(cid
== pred
->dtp_cacheid
);
7102 curthread
->t_predcache
= cid
;
7109 for (act
= ecb
->dte_action
; !(*flags
& CPU_DTRACE_ERROR
) &&
7110 act
!= NULL
; act
= act
->dta_next
) {
7113 dtrace_recdesc_t
*rec
= &act
->dta_rec
;
7115 size
= rec
->dtrd_size
;
7116 valoffs
= offs
+ rec
->dtrd_offset
;
7118 if (DTRACEACT_ISAGG(act
->dta_kind
)) {
7120 dtrace_aggregation_t
*agg
;
7122 agg
= (dtrace_aggregation_t
*)act
;
7124 if ((dp
= act
->dta_difo
) != NULL
)
7125 v
= dtrace_dif_emulate(dp
,
7126 &mstate
, vstate
, state
);
7128 if (*flags
& CPU_DTRACE_ERROR
)
7132 * Note that we always pass the expression
7133 * value from the previous iteration of the
7134 * action loop. This value will only be used
7135 * if there is an expression argument to the
7136 * aggregating action, denoted by the
7137 * dtag_hasarg field.
7139 dtrace_aggregate(agg
, buf
,
7140 offs
, aggbuf
, v
, val
);
7144 switch (act
->dta_kind
) {
7145 case DTRACEACT_STOP
:
7146 if (dtrace_priv_proc_destructive(state
,
7148 dtrace_action_stop();
7151 case DTRACEACT_BREAKPOINT
:
7152 if (dtrace_priv_kernel_destructive(state
))
7153 dtrace_action_breakpoint(ecb
);
7156 case DTRACEACT_PANIC
:
7157 if (dtrace_priv_kernel_destructive(state
))
7158 dtrace_action_panic(ecb
);
7161 case DTRACEACT_STACK
:
7162 if (!dtrace_priv_kernel(state
))
7165 dtrace_getpcstack((pc_t
*)(tomax
+ valoffs
),
7166 size
/ sizeof (pc_t
), probe
->dtpr_aframes
,
7167 DTRACE_ANCHORED(probe
) ? NULL
:
7172 case DTRACEACT_JSTACK
:
7173 case DTRACEACT_USTACK
:
7174 if (!dtrace_priv_proc(state
, &mstate
))
7178 * See comment in DIF_VAR_PID.
7180 if (DTRACE_ANCHORED(mstate
.dtms_probe
) &&
7182 int depth
= DTRACE_USTACK_NFRAMES(
7185 dtrace_bzero((void *)(tomax
+ valoffs
),
7186 DTRACE_USTACK_STRSIZE(rec
->dtrd_arg
)
7187 + depth
* sizeof (uint64_t));
7192 if (DTRACE_USTACK_STRSIZE(rec
->dtrd_arg
) != 0 &&
7193 curproc
->p_dtrace_helpers
!= NULL
) {
7195 * This is the slow path -- we have
7196 * allocated string space, and we're
7197 * getting the stack of a process that
7198 * has helpers. Call into a separate
7199 * routine to perform this processing.
7201 dtrace_action_ustack(&mstate
, state
,
7202 (uint64_t *)(tomax
+ valoffs
),
7208 * Clear the string space, since there's no
7209 * helper to do it for us.
7211 if (DTRACE_USTACK_STRSIZE(rec
->dtrd_arg
) != 0) {
7212 int depth
= DTRACE_USTACK_NFRAMES(
7214 size_t strsize
= DTRACE_USTACK_STRSIZE(
7216 uint64_t *buf
= (uint64_t *)(tomax
+
7218 void *strspace
= &buf
[depth
+ 1];
7220 dtrace_bzero(strspace
,
7221 MIN(depth
, strsize
));
7224 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
7225 dtrace_getupcstack((uint64_t *)
7227 DTRACE_USTACK_NFRAMES(rec
->dtrd_arg
) + 1);
7228 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
);
7238 val
= dtrace_dif_emulate(dp
, &mstate
, vstate
, state
);
7240 if (*flags
& CPU_DTRACE_ERROR
)
7243 switch (act
->dta_kind
) {
7244 case DTRACEACT_SPECULATE
: {
7245 dtrace_rechdr_t
*dtrh
;
7247 ASSERT(buf
== &state
->dts_buffer
[cpuid
]);
7248 buf
= dtrace_speculation_buffer(state
,
7252 *flags
|= CPU_DTRACE_DROP
;
7256 offs
= dtrace_buffer_reserve(buf
,
7257 ecb
->dte_needed
, ecb
->dte_alignment
,
7261 *flags
|= CPU_DTRACE_DROP
;
7265 tomax
= buf
->dtb_tomax
;
7266 ASSERT(tomax
!= NULL
);
7268 if (ecb
->dte_size
== 0)
7271 ASSERT3U(ecb
->dte_size
, >=,
7272 sizeof (dtrace_rechdr_t
));
7273 dtrh
= ((void *)(tomax
+ offs
));
7274 dtrh
->dtrh_epid
= ecb
->dte_epid
;
7276 * When the speculation is committed, all of
7277 * the records in the speculative buffer will
7278 * have their timestamps set to the commit
7279 * time. Until then, it is set to a sentinel
7280 * value, for debugability.
7282 DTRACE_RECORD_STORE_TIMESTAMP(dtrh
, UINT64_MAX
);
7286 case DTRACEACT_CHILL
:
7287 if (dtrace_priv_kernel_destructive(state
))
7288 dtrace_action_chill(&mstate
, val
);
7291 case DTRACEACT_RAISE
:
7292 if (dtrace_priv_proc_destructive(state
,
7294 dtrace_action_raise(val
);
7297 case DTRACEACT_COMMIT
:
7301 * We need to commit our buffer state.
7304 buf
->dtb_offset
= offs
+ ecb
->dte_size
;
7305 buf
= &state
->dts_buffer
[cpuid
];
7306 dtrace_speculation_commit(state
, cpuid
, val
);
7310 case DTRACEACT_DISCARD
:
7311 dtrace_speculation_discard(state
, cpuid
, val
);
7314 case DTRACEACT_DIFEXPR
:
7315 case DTRACEACT_LIBACT
:
7316 case DTRACEACT_PRINTF
:
7317 case DTRACEACT_PRINTA
:
7318 case DTRACEACT_SYSTEM
:
7319 case DTRACEACT_FREOPEN
:
7320 case DTRACEACT_TRACEMEM
:
7323 case DTRACEACT_TRACEMEM_DYNSIZE
:
7329 if (!dtrace_priv_kernel(state
))
7333 case DTRACEACT_USYM
:
7334 case DTRACEACT_UMOD
:
7335 case DTRACEACT_UADDR
: {
7336 struct pid
*pid
= curthread
->t_procp
->p_pidp
;
7338 if (!dtrace_priv_proc(state
, &mstate
))
7341 DTRACE_STORE(uint64_t, tomax
,
7342 valoffs
, (uint64_t)pid
->pid_id
);
7343 DTRACE_STORE(uint64_t, tomax
,
7344 valoffs
+ sizeof (uint64_t), val
);
7349 case DTRACEACT_EXIT
: {
7351 * For the exit action, we are going to attempt
7352 * to atomically set our activity to be
7353 * draining. If this fails (either because
7354 * another CPU has beat us to the exit action,
7355 * or because our current activity is something
7356 * other than ACTIVE or WARMUP), we will
7357 * continue. This assures that the exit action
7358 * can be successfully recorded at most once
7359 * when we're in the ACTIVE state. If we're
7360 * encountering the exit() action while in
7361 * COOLDOWN, however, we want to honor the new
7362 * status code. (We know that we're the only
7363 * thread in COOLDOWN, so there is no race.)
7365 void *activity
= &state
->dts_activity
;
7366 dtrace_activity_t current
= state
->dts_activity
;
7368 if (current
== DTRACE_ACTIVITY_COOLDOWN
)
7371 if (current
!= DTRACE_ACTIVITY_WARMUP
)
7372 current
= DTRACE_ACTIVITY_ACTIVE
;
7374 if (dtrace_cas32(activity
, current
,
7375 DTRACE_ACTIVITY_DRAINING
) != current
) {
7376 *flags
|= CPU_DTRACE_DROP
;
7387 if (dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
||
7388 dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYUREF
) {
7389 uintptr_t end
= valoffs
+ size
;
7391 if (tracememsize
!= 0 &&
7392 valoffs
+ tracememsize
< end
) {
7393 end
= valoffs
+ tracememsize
;
7397 if (dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
&&
7398 !dtrace_vcanload((void *)(uintptr_t)val
,
7399 &dp
->dtdo_rtype
, NULL
, &mstate
, vstate
))
7402 dtrace_store_by_ref(dp
, tomax
, size
, &valoffs
,
7403 &val
, end
, act
->dta_intuple
,
7404 dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
?
7405 DIF_TF_BYREF
: DIF_TF_BYUREF
);
7413 case sizeof (uint8_t):
7414 DTRACE_STORE(uint8_t, tomax
, valoffs
, val
);
7416 case sizeof (uint16_t):
7417 DTRACE_STORE(uint16_t, tomax
, valoffs
, val
);
7419 case sizeof (uint32_t):
7420 DTRACE_STORE(uint32_t, tomax
, valoffs
, val
);
7422 case sizeof (uint64_t):
7423 DTRACE_STORE(uint64_t, tomax
, valoffs
, val
);
7427 * Any other size should have been returned by
7428 * reference, not by value.
7435 if (*flags
& CPU_DTRACE_DROP
)
7438 if (*flags
& CPU_DTRACE_FAULT
) {
7440 dtrace_action_t
*err
;
7444 if (probe
->dtpr_id
== dtrace_probeid_error
) {
7446 * There's nothing we can do -- we had an
7447 * error on the error probe. We bump an
7448 * error counter to at least indicate that
7449 * this condition happened.
7451 dtrace_error(&state
->dts_dblerrors
);
7457 * Before recursing on dtrace_probe(), we
7458 * need to explicitly clear out our start
7459 * time to prevent it from being accumulated
7460 * into t_dtrace_vtime.
7462 curthread
->t_dtrace_start
= 0;
7466 * Iterate over the actions to figure out which action
7467 * we were processing when we experienced the error.
7468 * Note that act points _past_ the faulting action; if
7469 * act is ecb->dte_action, the fault was in the
7470 * predicate, if it's ecb->dte_action->dta_next it's
7471 * in action #1, and so on.
7473 for (err
= ecb
->dte_action
, ndx
= 0;
7474 err
!= act
; err
= err
->dta_next
, ndx
++)
7477 dtrace_probe_error(state
, ecb
->dte_epid
, ndx
,
7478 (mstate
.dtms_present
& DTRACE_MSTATE_FLTOFFS
) ?
7479 mstate
.dtms_fltoffs
: -1, DTRACE_FLAGS2FLT(*flags
),
7480 cpu_core
[cpuid
].cpuc_dtrace_illval
);
7486 buf
->dtb_offset
= offs
+ ecb
->dte_size
;
7489 end
= dtrace_gethrtime();
7491 curthread
->t_dtrace_start
= end
;
7493 CPU
->cpu_dtrace_nsec
+= end
- now
;
7495 dtrace_interrupt_enable(cookie
);
7499 * DTrace Probe Hashing Functions
7501 * The functions in this section (and indeed, the functions in remaining
7502 * sections) are not _called_ from probe context. (Any exceptions to this are
7503 * marked with a "Note:".) Rather, they are called from elsewhere in the
7504 * DTrace framework to look-up probes in, add probes to and remove probes from
7505 * the DTrace probe hashes. (Each probe is hashed by each element of the
7506 * probe tuple -- allowing for fast lookups, regardless of what was
7510 dtrace_hash_str(char *p
)
7516 hval
= (hval
<< 4) + *p
++;
7517 if ((g
= (hval
& 0xf0000000)) != 0)
7524 static dtrace_hash_t
*
7525 dtrace_hash_create(uintptr_t stroffs
, uintptr_t nextoffs
, uintptr_t prevoffs
)
7527 dtrace_hash_t
*hash
= kmem_zalloc(sizeof (dtrace_hash_t
), KM_SLEEP
);
7529 hash
->dth_stroffs
= stroffs
;
7530 hash
->dth_nextoffs
= nextoffs
;
7531 hash
->dth_prevoffs
= prevoffs
;
7534 hash
->dth_mask
= hash
->dth_size
- 1;
7536 hash
->dth_tab
= kmem_zalloc(hash
->dth_size
*
7537 sizeof (dtrace_hashbucket_t
*), KM_SLEEP
);
7543 dtrace_hash_destroy(dtrace_hash_t
*hash
)
7548 for (i
= 0; i
< hash
->dth_size
; i
++)
7549 ASSERT(hash
->dth_tab
[i
] == NULL
);
7552 kmem_free(hash
->dth_tab
,
7553 hash
->dth_size
* sizeof (dtrace_hashbucket_t
*));
7554 kmem_free(hash
, sizeof (dtrace_hash_t
));
7558 dtrace_hash_resize(dtrace_hash_t
*hash
)
7560 int size
= hash
->dth_size
, i
, ndx
;
7561 int new_size
= hash
->dth_size
<< 1;
7562 int new_mask
= new_size
- 1;
7563 dtrace_hashbucket_t
**new_tab
, *bucket
, *next
;
7565 ASSERT((new_size
& new_mask
) == 0);
7567 new_tab
= kmem_zalloc(new_size
* sizeof (void *), KM_SLEEP
);
7569 for (i
= 0; i
< size
; i
++) {
7570 for (bucket
= hash
->dth_tab
[i
]; bucket
!= NULL
; bucket
= next
) {
7571 dtrace_probe_t
*probe
= bucket
->dthb_chain
;
7573 ASSERT(probe
!= NULL
);
7574 ndx
= DTRACE_HASHSTR(hash
, probe
) & new_mask
;
7576 next
= bucket
->dthb_next
;
7577 bucket
->dthb_next
= new_tab
[ndx
];
7578 new_tab
[ndx
] = bucket
;
7582 kmem_free(hash
->dth_tab
, hash
->dth_size
* sizeof (void *));
7583 hash
->dth_tab
= new_tab
;
7584 hash
->dth_size
= new_size
;
7585 hash
->dth_mask
= new_mask
;
7589 dtrace_hash_add(dtrace_hash_t
*hash
, dtrace_probe_t
*new)
7591 int hashval
= DTRACE_HASHSTR(hash
, new);
7592 int ndx
= hashval
& hash
->dth_mask
;
7593 dtrace_hashbucket_t
*bucket
= hash
->dth_tab
[ndx
];
7594 dtrace_probe_t
**nextp
, **prevp
;
7596 for (; bucket
!= NULL
; bucket
= bucket
->dthb_next
) {
7597 if (DTRACE_HASHEQ(hash
, bucket
->dthb_chain
, new))
7601 if ((hash
->dth_nbuckets
>> 1) > hash
->dth_size
) {
7602 dtrace_hash_resize(hash
);
7603 dtrace_hash_add(hash
, new);
7607 bucket
= kmem_zalloc(sizeof (dtrace_hashbucket_t
), KM_SLEEP
);
7608 bucket
->dthb_next
= hash
->dth_tab
[ndx
];
7609 hash
->dth_tab
[ndx
] = bucket
;
7610 hash
->dth_nbuckets
++;
7613 nextp
= DTRACE_HASHNEXT(hash
, new);
7614 ASSERT(*nextp
== NULL
&& *(DTRACE_HASHPREV(hash
, new)) == NULL
);
7615 *nextp
= bucket
->dthb_chain
;
7617 if (bucket
->dthb_chain
!= NULL
) {
7618 prevp
= DTRACE_HASHPREV(hash
, bucket
->dthb_chain
);
7619 ASSERT(*prevp
== NULL
);
7623 bucket
->dthb_chain
= new;
7627 static dtrace_probe_t
*
7628 dtrace_hash_lookup(dtrace_hash_t
*hash
, dtrace_probe_t
*template)
7630 int hashval
= DTRACE_HASHSTR(hash
, template);
7631 int ndx
= hashval
& hash
->dth_mask
;
7632 dtrace_hashbucket_t
*bucket
= hash
->dth_tab
[ndx
];
7634 for (; bucket
!= NULL
; bucket
= bucket
->dthb_next
) {
7635 if (DTRACE_HASHEQ(hash
, bucket
->dthb_chain
, template))
7636 return (bucket
->dthb_chain
);
7643 dtrace_hash_collisions(dtrace_hash_t
*hash
, dtrace_probe_t
*template)
7645 int hashval
= DTRACE_HASHSTR(hash
, template);
7646 int ndx
= hashval
& hash
->dth_mask
;
7647 dtrace_hashbucket_t
*bucket
= hash
->dth_tab
[ndx
];
7649 for (; bucket
!= NULL
; bucket
= bucket
->dthb_next
) {
7650 if (DTRACE_HASHEQ(hash
, bucket
->dthb_chain
, template))
7651 return (bucket
->dthb_len
);
7658 dtrace_hash_remove(dtrace_hash_t
*hash
, dtrace_probe_t
*probe
)
7660 int ndx
= DTRACE_HASHSTR(hash
, probe
) & hash
->dth_mask
;
7661 dtrace_hashbucket_t
*bucket
= hash
->dth_tab
[ndx
];
7663 dtrace_probe_t
**prevp
= DTRACE_HASHPREV(hash
, probe
);
7664 dtrace_probe_t
**nextp
= DTRACE_HASHNEXT(hash
, probe
);
7667 * Find the bucket that we're removing this probe from.
7669 for (; bucket
!= NULL
; bucket
= bucket
->dthb_next
) {
7670 if (DTRACE_HASHEQ(hash
, bucket
->dthb_chain
, probe
))
7674 ASSERT(bucket
!= NULL
);
7676 if (*prevp
== NULL
) {
7677 if (*nextp
== NULL
) {
7679 * The removed probe was the only probe on this
7680 * bucket; we need to remove the bucket.
7682 dtrace_hashbucket_t
*b
= hash
->dth_tab
[ndx
];
7684 ASSERT(bucket
->dthb_chain
== probe
);
7688 hash
->dth_tab
[ndx
] = bucket
->dthb_next
;
7690 while (b
->dthb_next
!= bucket
)
7692 b
->dthb_next
= bucket
->dthb_next
;
7695 ASSERT(hash
->dth_nbuckets
> 0);
7696 hash
->dth_nbuckets
--;
7697 kmem_free(bucket
, sizeof (dtrace_hashbucket_t
));
7701 bucket
->dthb_chain
= *nextp
;
7703 *(DTRACE_HASHNEXT(hash
, *prevp
)) = *nextp
;
7707 *(DTRACE_HASHPREV(hash
, *nextp
)) = *prevp
;
7711 * DTrace Utility Functions
7713 * These are random utility functions that are _not_ called from probe context.
7716 dtrace_badattr(const dtrace_attribute_t
*a
)
7718 return (a
->dtat_name
> DTRACE_STABILITY_MAX
||
7719 a
->dtat_data
> DTRACE_STABILITY_MAX
||
7720 a
->dtat_class
> DTRACE_CLASS_MAX
);
7724 * Return a duplicate copy of a string. If the specified string is NULL,
7725 * this function returns a zero-length string.
7728 dtrace_strdup(const char *str
)
7730 char *new = kmem_zalloc((str
!= NULL
? strlen(str
) : 0) + 1, KM_SLEEP
);
7733 (void) strcpy(new, str
);
7738 #define DTRACE_ISALPHA(c) \
7739 (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
7742 dtrace_badname(const char *s
)
7746 if (s
== NULL
|| (c
= *s
++) == '\0')
7749 if (!DTRACE_ISALPHA(c
) && c
!= '-' && c
!= '_' && c
!= '.')
7752 while ((c
= *s
++) != '\0') {
7753 if (!DTRACE_ISALPHA(c
) && (c
< '0' || c
> '9') &&
7754 c
!= '-' && c
!= '_' && c
!= '.' && c
!= '`')
7762 dtrace_cred2priv(cred_t
*cr
, uint32_t *privp
, uid_t
*uidp
, zoneid_t
*zoneidp
)
7766 if (cr
== NULL
|| PRIV_POLICY_ONLY(cr
, PRIV_ALL
, B_FALSE
)) {
7768 * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
7770 priv
= DTRACE_PRIV_ALL
;
7772 *uidp
= crgetuid(cr
);
7773 *zoneidp
= crgetzoneid(cr
);
7776 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_KERNEL
, B_FALSE
))
7777 priv
|= DTRACE_PRIV_KERNEL
| DTRACE_PRIV_USER
;
7778 else if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_USER
, B_FALSE
))
7779 priv
|= DTRACE_PRIV_USER
;
7780 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_PROC
, B_FALSE
))
7781 priv
|= DTRACE_PRIV_PROC
;
7782 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_OWNER
, B_FALSE
))
7783 priv
|= DTRACE_PRIV_OWNER
;
7784 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_ZONE
, B_FALSE
))
7785 priv
|= DTRACE_PRIV_ZONEOWNER
;
7791 #ifdef DTRACE_ERRDEBUG
7793 dtrace_errdebug(const char *str
)
7795 int hval
= dtrace_hash_str((char *)str
) % DTRACE_ERRHASHSZ
;
7798 mutex_enter(&dtrace_errlock
);
7799 dtrace_errlast
= str
;
7800 dtrace_errthread
= curthread
;
7802 while (occupied
++ < DTRACE_ERRHASHSZ
) {
7803 if (dtrace_errhash
[hval
].dter_msg
== str
) {
7804 dtrace_errhash
[hval
].dter_count
++;
7808 if (dtrace_errhash
[hval
].dter_msg
!= NULL
) {
7809 hval
= (hval
+ 1) % DTRACE_ERRHASHSZ
;
7813 dtrace_errhash
[hval
].dter_msg
= str
;
7814 dtrace_errhash
[hval
].dter_count
= 1;
7818 panic("dtrace: undersized error hash");
7820 mutex_exit(&dtrace_errlock
);
7825 * DTrace Matching Functions
7827 * These functions are used to match groups of probes, given some elements of
7828 * a probe tuple, or some globbed expressions for elements of a probe tuple.
7831 dtrace_match_priv(const dtrace_probe_t
*prp
, uint32_t priv
, uid_t uid
,
7834 if (priv
!= DTRACE_PRIV_ALL
) {
7835 uint32_t ppriv
= prp
->dtpr_provider
->dtpv_priv
.dtpp_flags
;
7836 uint32_t match
= priv
& ppriv
;
7839 * No PRIV_DTRACE_* privileges...
7841 if ((priv
& (DTRACE_PRIV_PROC
| DTRACE_PRIV_USER
|
7842 DTRACE_PRIV_KERNEL
)) == 0)
7846 * No matching bits, but there were bits to match...
7848 if (match
== 0 && ppriv
!= 0)
7852 * Need to have permissions to the process, but don't...
7854 if (((ppriv
& ~match
) & DTRACE_PRIV_OWNER
) != 0 &&
7855 uid
!= prp
->dtpr_provider
->dtpv_priv
.dtpp_uid
) {
7860 * Need to be in the same zone unless we possess the
7861 * privilege to examine all zones.
7863 if (((ppriv
& ~match
) & DTRACE_PRIV_ZONEOWNER
) != 0 &&
7864 zoneid
!= prp
->dtpr_provider
->dtpv_priv
.dtpp_zoneid
) {
7873 * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
7874 * consists of input pattern strings and an ops-vector to evaluate them.
7875 * This function returns >0 for match, 0 for no match, and <0 for error.
7878 dtrace_match_probe(const dtrace_probe_t
*prp
, const dtrace_probekey_t
*pkp
,
7879 uint32_t priv
, uid_t uid
, zoneid_t zoneid
)
7881 dtrace_provider_t
*pvp
= prp
->dtpr_provider
;
7884 if (pvp
->dtpv_defunct
)
7887 if ((rv
= pkp
->dtpk_pmatch(pvp
->dtpv_name
, pkp
->dtpk_prov
, 0)) <= 0)
7890 if ((rv
= pkp
->dtpk_mmatch(prp
->dtpr_mod
, pkp
->dtpk_mod
, 0)) <= 0)
7893 if ((rv
= pkp
->dtpk_fmatch(prp
->dtpr_func
, pkp
->dtpk_func
, 0)) <= 0)
7896 if ((rv
= pkp
->dtpk_nmatch(prp
->dtpr_name
, pkp
->dtpk_name
, 0)) <= 0)
7899 if (dtrace_match_priv(prp
, priv
, uid
, zoneid
) == 0)
7906 * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
7907 * interface for matching a glob pattern 'p' to an input string 's'. Unlike
7908 * libc's version, the kernel version only applies to 8-bit ASCII strings.
7909 * In addition, all of the recursion cases except for '*' matching have been
7910 * unwound. For '*', we still implement recursive evaluation, but a depth
7911 * counter is maintained and matching is aborted if we recurse too deep.
7912 * The function returns 0 if no match, >0 if match, and <0 if recursion error.
7915 dtrace_match_glob(const char *s
, const char *p
, int depth
)
7921 if (depth
> DTRACE_PROBEKEY_MAXDEPTH
)
7925 s
= ""; /* treat NULL as empty string */
7934 if ((c
= *p
++) == '\0')
7935 return (s1
== '\0');
7939 int ok
= 0, notflag
= 0;
7950 if ((c
= *p
++) == '\0')
7954 if (c
== '-' && lc
!= '\0' && *p
!= ']') {
7955 if ((c
= *p
++) == '\0')
7957 if (c
== '\\' && (c
= *p
++) == '\0')
7961 if (s1
< lc
|| s1
> c
)
7965 } else if (lc
<= s1
&& s1
<= c
)
7968 } else if (c
== '\\' && (c
= *p
++) == '\0')
7971 lc
= c
; /* save left-hand 'c' for next iteration */
7981 if ((c
= *p
++) == '\0')
7993 if ((c
= *p
++) == '\0')
8009 p
++; /* consecutive *'s are identical to a single one */
8014 for (s
= olds
; *s
!= '\0'; s
++) {
8015 if ((gs
= dtrace_match_glob(s
, p
, depth
+ 1)) != 0)
8025 dtrace_match_string(const char *s
, const char *p
, int depth
)
8027 return (s
!= NULL
&& strcmp(s
, p
) == 0);
8032 dtrace_match_nul(const char *s
, const char *p
, int depth
)
8034 return (1); /* always match the empty pattern */
8039 dtrace_match_nonzero(const char *s
, const char *p
, int depth
)
8041 return (s
!= NULL
&& s
[0] != '\0');
8045 dtrace_match(const dtrace_probekey_t
*pkp
, uint32_t priv
, uid_t uid
,
8046 zoneid_t zoneid
, int (*matched
)(dtrace_probe_t
*, void *), void *arg
)
8048 dtrace_probe_t
template, *probe
;
8049 dtrace_hash_t
*hash
= NULL
;
8050 int len
, rc
, best
= INT_MAX
, nmatched
= 0;
8053 ASSERT(MUTEX_HELD(&dtrace_lock
));
8056 * If the probe ID is specified in the key, just lookup by ID and
8057 * invoke the match callback once if a matching probe is found.
8059 if (pkp
->dtpk_id
!= DTRACE_IDNONE
) {
8060 if ((probe
= dtrace_probe_lookup_id(pkp
->dtpk_id
)) != NULL
&&
8061 dtrace_match_probe(probe
, pkp
, priv
, uid
, zoneid
) > 0) {
8062 if ((*matched
)(probe
, arg
) == DTRACE_MATCH_FAIL
)
8063 return (DTRACE_MATCH_FAIL
);
8069 template.dtpr_mod
= (char *)pkp
->dtpk_mod
;
8070 template.dtpr_func
= (char *)pkp
->dtpk_func
;
8071 template.dtpr_name
= (char *)pkp
->dtpk_name
;
8074 * We want to find the most distinct of the module name, function
8075 * name, and name. So for each one that is not a glob pattern or
8076 * empty string, we perform a lookup in the corresponding hash and
8077 * use the hash table with the fewest collisions to do our search.
8079 if (pkp
->dtpk_mmatch
== &dtrace_match_string
&&
8080 (len
= dtrace_hash_collisions(dtrace_bymod
, &template)) < best
) {
8082 hash
= dtrace_bymod
;
8085 if (pkp
->dtpk_fmatch
== &dtrace_match_string
&&
8086 (len
= dtrace_hash_collisions(dtrace_byfunc
, &template)) < best
) {
8088 hash
= dtrace_byfunc
;
8091 if (pkp
->dtpk_nmatch
== &dtrace_match_string
&&
8092 (len
= dtrace_hash_collisions(dtrace_byname
, &template)) < best
) {
8094 hash
= dtrace_byname
;
8098 * If we did not select a hash table, iterate over every probe and
8099 * invoke our callback for each one that matches our input probe key.
8102 for (i
= 0; i
< dtrace_nprobes
; i
++) {
8103 if ((probe
= dtrace_probes
[i
]) == NULL
||
8104 dtrace_match_probe(probe
, pkp
, priv
, uid
,
8110 if ((rc
= (*matched
)(probe
, arg
)) !=
8111 DTRACE_MATCH_NEXT
) {
8112 if (rc
== DTRACE_MATCH_FAIL
)
8113 return (DTRACE_MATCH_FAIL
);
8122 * If we selected a hash table, iterate over each probe of the same key
8123 * name and invoke the callback for every probe that matches the other
8124 * attributes of our input probe key.
8126 for (probe
= dtrace_hash_lookup(hash
, &template); probe
!= NULL
;
8127 probe
= *(DTRACE_HASHNEXT(hash
, probe
))) {
8129 if (dtrace_match_probe(probe
, pkp
, priv
, uid
, zoneid
) <= 0)
8134 if ((rc
= (*matched
)(probe
, arg
)) != DTRACE_MATCH_NEXT
) {
8135 if (rc
== DTRACE_MATCH_FAIL
)
8136 return (DTRACE_MATCH_FAIL
);
8145 * Return the function pointer dtrace_probecmp() should use to compare the
8146 * specified pattern with a string. For NULL or empty patterns, we select
8147 * dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob().
8148 * For non-empty non-glob strings, we use dtrace_match_string().
8150 static dtrace_probekey_f
*
8151 dtrace_probekey_func(const char *p
)
8155 if (p
== NULL
|| *p
== '\0')
8156 return (&dtrace_match_nul
);
8158 while ((c
= *p
++) != '\0') {
8159 if (c
== '[' || c
== '?' || c
== '*' || c
== '\\')
8160 return (&dtrace_match_glob
);
8163 return (&dtrace_match_string
);
8167 * Build a probe comparison key for use with dtrace_match_probe() from the
8168 * given probe description. By convention, a null key only matches anchored
8169 * probes: if each field is the empty string, reset dtpk_fmatch to
8170 * dtrace_match_nonzero().
8173 dtrace_probekey(const dtrace_probedesc_t
*pdp
, dtrace_probekey_t
*pkp
)
8175 pkp
->dtpk_prov
= pdp
->dtpd_provider
;
8176 pkp
->dtpk_pmatch
= dtrace_probekey_func(pdp
->dtpd_provider
);
8178 pkp
->dtpk_mod
= pdp
->dtpd_mod
;
8179 pkp
->dtpk_mmatch
= dtrace_probekey_func(pdp
->dtpd_mod
);
8181 pkp
->dtpk_func
= pdp
->dtpd_func
;
8182 pkp
->dtpk_fmatch
= dtrace_probekey_func(pdp
->dtpd_func
);
8184 pkp
->dtpk_name
= pdp
->dtpd_name
;
8185 pkp
->dtpk_nmatch
= dtrace_probekey_func(pdp
->dtpd_name
);
8187 pkp
->dtpk_id
= pdp
->dtpd_id
;
8189 if (pkp
->dtpk_id
== DTRACE_IDNONE
&&
8190 pkp
->dtpk_pmatch
== &dtrace_match_nul
&&
8191 pkp
->dtpk_mmatch
== &dtrace_match_nul
&&
8192 pkp
->dtpk_fmatch
== &dtrace_match_nul
&&
8193 pkp
->dtpk_nmatch
== &dtrace_match_nul
)
8194 pkp
->dtpk_fmatch
= &dtrace_match_nonzero
;
8198 * DTrace Provider-to-Framework API Functions
8200 * These functions implement much of the Provider-to-Framework API, as
8201 * described in <sys/dtrace.h>. The parts of the API not in this section are
8202 * the functions in the API for probe management (found below), and
8203 * dtrace_probe() itself (found above).
8207 * Register the calling provider with the DTrace framework. This should
8208 * generally be called by DTrace providers in their attach(9E) entry point.
8211 dtrace_register(const char *name
, const dtrace_pattr_t
*pap
, uint32_t priv
,
8212 cred_t
*cr
, const dtrace_pops_t
*pops
, void *arg
, dtrace_provider_id_t
*idp
)
8214 dtrace_provider_t
*provider
;
8216 if (name
== NULL
|| pap
== NULL
|| pops
== NULL
|| idp
== NULL
) {
8217 cmn_err(CE_WARN
, "failed to register provider '%s': invalid "
8218 "arguments", name
? name
: "<NULL>");
8222 if (name
[0] == '\0' || dtrace_badname(name
)) {
8223 cmn_err(CE_WARN
, "failed to register provider '%s': invalid "
8224 "provider name", name
);
8228 if ((pops
->dtps_provide
== NULL
&& pops
->dtps_provide_module
== NULL
) ||
8229 pops
->dtps_enable
== NULL
|| pops
->dtps_disable
== NULL
||
8230 pops
->dtps_destroy
== NULL
||
8231 ((pops
->dtps_resume
== NULL
) != (pops
->dtps_suspend
== NULL
))) {
8232 cmn_err(CE_WARN
, "failed to register provider '%s': invalid "
8233 "provider ops", name
);
8237 if (dtrace_badattr(&pap
->dtpa_provider
) ||
8238 dtrace_badattr(&pap
->dtpa_mod
) ||
8239 dtrace_badattr(&pap
->dtpa_func
) ||
8240 dtrace_badattr(&pap
->dtpa_name
) ||
8241 dtrace_badattr(&pap
->dtpa_args
)) {
8242 cmn_err(CE_WARN
, "failed to register provider '%s': invalid "
8243 "provider attributes", name
);
8247 if (priv
& ~DTRACE_PRIV_ALL
) {
8248 cmn_err(CE_WARN
, "failed to register provider '%s': invalid "
8249 "privilege attributes", name
);
8253 if ((priv
& DTRACE_PRIV_KERNEL
) &&
8254 (priv
& (DTRACE_PRIV_USER
| DTRACE_PRIV_OWNER
)) &&
8255 pops
->dtps_mode
== NULL
) {
8256 cmn_err(CE_WARN
, "failed to register provider '%s': need "
8257 "dtps_mode() op for given privilege attributes", name
);
8261 provider
= kmem_zalloc(sizeof (dtrace_provider_t
), KM_SLEEP
);
8262 provider
->dtpv_name
= kmem_alloc(strlen(name
) + 1, KM_SLEEP
);
8263 (void) strcpy(provider
->dtpv_name
, name
);
8265 provider
->dtpv_attr
= *pap
;
8266 provider
->dtpv_priv
.dtpp_flags
= priv
;
8268 provider
->dtpv_priv
.dtpp_uid
= crgetuid(cr
);
8269 provider
->dtpv_priv
.dtpp_zoneid
= crgetzoneid(cr
);
8271 provider
->dtpv_pops
= *pops
;
8273 if (pops
->dtps_provide
== NULL
) {
8274 ASSERT(pops
->dtps_provide_module
!= NULL
);
8275 provider
->dtpv_pops
.dtps_provide
=
8276 (void (*)(void *, const dtrace_probedesc_t
*))dtrace_nullop
;
8279 if (pops
->dtps_provide_module
== NULL
) {
8280 ASSERT(pops
->dtps_provide
!= NULL
);
8281 provider
->dtpv_pops
.dtps_provide_module
=
8282 (void (*)(void *, struct modctl
*))dtrace_nullop
;
8285 if (pops
->dtps_suspend
== NULL
) {
8286 ASSERT(pops
->dtps_resume
== NULL
);
8287 provider
->dtpv_pops
.dtps_suspend
=
8288 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
;
8289 provider
->dtpv_pops
.dtps_resume
=
8290 (void (*)(void *, dtrace_id_t
, void *))dtrace_nullop
;
8293 provider
->dtpv_arg
= arg
;
8294 *idp
= (dtrace_provider_id_t
)provider
;
8296 if (pops
== &dtrace_provider_ops
) {
8297 ASSERT(MUTEX_HELD(&dtrace_provider_lock
));
8298 ASSERT(MUTEX_HELD(&dtrace_lock
));
8299 ASSERT(dtrace_anon
.dta_enabling
== NULL
);
8302 * We make sure that the DTrace provider is at the head of
8303 * the provider chain.
8305 provider
->dtpv_next
= dtrace_provider
;
8306 dtrace_provider
= provider
;
8310 mutex_enter(&dtrace_provider_lock
);
8311 mutex_enter(&dtrace_lock
);
8314 * If there is at least one provider registered, we'll add this
8315 * provider after the first provider.
8317 if (dtrace_provider
!= NULL
) {
8318 provider
->dtpv_next
= dtrace_provider
->dtpv_next
;
8319 dtrace_provider
->dtpv_next
= provider
;
8321 dtrace_provider
= provider
;
8324 if (dtrace_retained
!= NULL
) {
8325 dtrace_enabling_provide(provider
);
8328 * Now we need to call dtrace_enabling_matchall() -- which
8329 * will acquire cpu_lock and dtrace_lock. We therefore need
8330 * to drop all of our locks before calling into it...
8332 mutex_exit(&dtrace_lock
);
8333 mutex_exit(&dtrace_provider_lock
);
8334 dtrace_enabling_matchall();
8339 mutex_exit(&dtrace_lock
);
8340 mutex_exit(&dtrace_provider_lock
);
8346 * Unregister the specified provider from the DTrace framework. This should
8347 * generally be called by DTrace providers in their detach(9E) entry point.
8350 dtrace_unregister(dtrace_provider_id_t id
)
8352 dtrace_provider_t
*old
= (dtrace_provider_t
*)id
;
8353 dtrace_provider_t
*prev
= NULL
;
8354 int i
, self
= 0, noreap
= 0;
8355 dtrace_probe_t
*probe
, *first
= NULL
;
8357 if (old
->dtpv_pops
.dtps_enable
==
8358 (int (*)(void *, dtrace_id_t
, void *))dtrace_enable_nullop
) {
8360 * If DTrace itself is the provider, we're called with locks
8363 ASSERT(old
== dtrace_provider
);
8364 ASSERT(dtrace_devi
!= NULL
);
8365 ASSERT(MUTEX_HELD(&dtrace_provider_lock
));
8366 ASSERT(MUTEX_HELD(&dtrace_lock
));
8369 if (dtrace_provider
->dtpv_next
!= NULL
) {
8371 * There's another provider here; return failure.
8376 mutex_enter(&dtrace_provider_lock
);
8377 mutex_enter(&mod_lock
);
8378 mutex_enter(&dtrace_lock
);
8382 * If anyone has /dev/dtrace open, or if there are anonymous enabled
8383 * probes, we refuse to let providers slither away, unless this
8384 * provider has already been explicitly invalidated.
8386 if (!old
->dtpv_defunct
&&
8387 (dtrace_opens
|| (dtrace_anon
.dta_state
!= NULL
&&
8388 dtrace_anon
.dta_state
->dts_necbs
> 0))) {
8390 mutex_exit(&dtrace_lock
);
8391 mutex_exit(&mod_lock
);
8392 mutex_exit(&dtrace_provider_lock
);
8398 * Attempt to destroy the probes associated with this provider.
8400 for (i
= 0; i
< dtrace_nprobes
; i
++) {
8401 if ((probe
= dtrace_probes
[i
]) == NULL
)
8404 if (probe
->dtpr_provider
!= old
)
8407 if (probe
->dtpr_ecb
== NULL
)
8411 * If we are trying to unregister a defunct provider, and the
8412 * provider was made defunct within the interval dictated by
8413 * dtrace_unregister_defunct_reap, we'll (asynchronously)
8414 * attempt to reap our enablings. To denote that the provider
8415 * should reattempt to unregister itself at some point in the
8416 * future, we will return a differentiable error code (EAGAIN
8417 * instead of EBUSY) in this case.
8419 if (dtrace_gethrtime() - old
->dtpv_defunct
>
8420 dtrace_unregister_defunct_reap
)
8424 mutex_exit(&dtrace_lock
);
8425 mutex_exit(&mod_lock
);
8426 mutex_exit(&dtrace_provider_lock
);
8432 (void) taskq_dispatch(dtrace_taskq
,
8433 (task_func_t
*)dtrace_enabling_reap
, NULL
, TQ_SLEEP
);
8439 * All of the probes for this provider are disabled; we can safely
8440 * remove all of them from their hash chains and from the probe array.
8442 for (i
= 0; i
< dtrace_nprobes
; i
++) {
8443 if ((probe
= dtrace_probes
[i
]) == NULL
)
8446 if (probe
->dtpr_provider
!= old
)
8449 dtrace_probes
[i
] = NULL
;
8451 dtrace_hash_remove(dtrace_bymod
, probe
);
8452 dtrace_hash_remove(dtrace_byfunc
, probe
);
8453 dtrace_hash_remove(dtrace_byname
, probe
);
8455 if (first
== NULL
) {
8457 probe
->dtpr_nextmod
= NULL
;
8459 probe
->dtpr_nextmod
= first
;
8465 * The provider's probes have been removed from the hash chains and
8466 * from the probe array. Now issue a dtrace_sync() to be sure that
8467 * everyone has cleared out from any probe array processing.
8471 for (probe
= first
; probe
!= NULL
; probe
= first
) {
8472 first
= probe
->dtpr_nextmod
;
8474 old
->dtpv_pops
.dtps_destroy(old
->dtpv_arg
, probe
->dtpr_id
,
8476 kmem_free(probe
->dtpr_mod
, strlen(probe
->dtpr_mod
) + 1);
8477 kmem_free(probe
->dtpr_func
, strlen(probe
->dtpr_func
) + 1);
8478 kmem_free(probe
->dtpr_name
, strlen(probe
->dtpr_name
) + 1);
8479 vmem_free(dtrace_arena
, (void *)(uintptr_t)(probe
->dtpr_id
), 1);
8480 kmem_free(probe
, sizeof (dtrace_probe_t
));
8483 if ((prev
= dtrace_provider
) == old
) {
8484 ASSERT(self
|| dtrace_devi
== NULL
);
8485 ASSERT(old
->dtpv_next
== NULL
|| dtrace_devi
== NULL
);
8486 dtrace_provider
= old
->dtpv_next
;
8488 while (prev
!= NULL
&& prev
->dtpv_next
!= old
)
8489 prev
= prev
->dtpv_next
;
8492 panic("attempt to unregister non-existent "
8493 "dtrace provider %p\n", (void *)id
);
8496 prev
->dtpv_next
= old
->dtpv_next
;
8500 mutex_exit(&dtrace_lock
);
8501 mutex_exit(&mod_lock
);
8502 mutex_exit(&dtrace_provider_lock
);
8505 kmem_free(old
->dtpv_name
, strlen(old
->dtpv_name
) + 1);
8506 kmem_free(old
, sizeof (dtrace_provider_t
));
8512 * Invalidate the specified provider. All subsequent probe lookups for the
8513 * specified provider will fail, but its probes will not be removed.
8516 dtrace_invalidate(dtrace_provider_id_t id
)
8518 dtrace_provider_t
*pvp
= (dtrace_provider_t
*)id
;
8520 ASSERT(pvp
->dtpv_pops
.dtps_enable
!=
8521 (int (*)(void *, dtrace_id_t
, void *))dtrace_enable_nullop
);
8523 mutex_enter(&dtrace_provider_lock
);
8524 mutex_enter(&dtrace_lock
);
8526 pvp
->dtpv_defunct
= dtrace_gethrtime();
8528 mutex_exit(&dtrace_lock
);
8529 mutex_exit(&dtrace_provider_lock
);
8533 * Indicate whether or not DTrace has attached.
8536 dtrace_attached(void)
8539 * dtrace_provider will be non-NULL iff the DTrace driver has
8540 * attached. (It's non-NULL because DTrace is always itself a
8543 return (dtrace_provider
!= NULL
);
8547 * Remove all the unenabled probes for the given provider. This function is
8548 * not unlike dtrace_unregister(), except that it doesn't remove the provider
8549 * -- just as many of its associated probes as it can.
8552 dtrace_condense(dtrace_provider_id_t id
)
8554 dtrace_provider_t
*prov
= (dtrace_provider_t
*)id
;
8556 dtrace_probe_t
*probe
;
8559 * Make sure this isn't the dtrace provider itself.
8561 ASSERT(prov
->dtpv_pops
.dtps_enable
!=
8562 (int (*)(void *, dtrace_id_t
, void *))dtrace_enable_nullop
);
8564 mutex_enter(&dtrace_provider_lock
);
8565 mutex_enter(&dtrace_lock
);
8568 * Attempt to destroy the probes associated with this provider.
8570 for (i
= 0; i
< dtrace_nprobes
; i
++) {
8571 if ((probe
= dtrace_probes
[i
]) == NULL
)
8574 if (probe
->dtpr_provider
!= prov
)
8577 if (probe
->dtpr_ecb
!= NULL
)
8580 dtrace_probes
[i
] = NULL
;
8582 dtrace_hash_remove(dtrace_bymod
, probe
);
8583 dtrace_hash_remove(dtrace_byfunc
, probe
);
8584 dtrace_hash_remove(dtrace_byname
, probe
);
8586 prov
->dtpv_pops
.dtps_destroy(prov
->dtpv_arg
, i
+ 1,
8588 kmem_free(probe
->dtpr_mod
, strlen(probe
->dtpr_mod
) + 1);
8589 kmem_free(probe
->dtpr_func
, strlen(probe
->dtpr_func
) + 1);
8590 kmem_free(probe
->dtpr_name
, strlen(probe
->dtpr_name
) + 1);
8591 kmem_free(probe
, sizeof (dtrace_probe_t
));
8592 vmem_free(dtrace_arena
, (void *)((uintptr_t)i
+ 1), 1);
8595 mutex_exit(&dtrace_lock
);
8596 mutex_exit(&dtrace_provider_lock
);
8602 * DTrace Probe Management Functions
8604 * The functions in this section perform the DTrace probe management,
8605 * including functions to create probes, look-up probes, and call into the
8606 * providers to request that probes be provided. Some of these functions are
8607 * in the Provider-to-Framework API; these functions can be identified by the
8608 * fact that they are not declared "static".
8612 * Create a probe with the specified module name, function name, and name.
8615 dtrace_probe_create(dtrace_provider_id_t prov
, const char *mod
,
8616 const char *func
, const char *name
, int aframes
, void *arg
)
8618 dtrace_probe_t
*probe
, **probes
;
8619 dtrace_provider_t
*provider
= (dtrace_provider_t
*)prov
;
8622 if (provider
== dtrace_provider
) {
8623 ASSERT(MUTEX_HELD(&dtrace_lock
));
8625 mutex_enter(&dtrace_lock
);
8628 id
= (dtrace_id_t
)(uintptr_t)vmem_alloc(dtrace_arena
, 1,
8629 VM_BESTFIT
| VM_SLEEP
);
8630 probe
= kmem_zalloc(sizeof (dtrace_probe_t
), KM_SLEEP
);
8632 probe
->dtpr_id
= id
;
8633 probe
->dtpr_gen
= dtrace_probegen
++;
8634 probe
->dtpr_mod
= dtrace_strdup(mod
);
8635 probe
->dtpr_func
= dtrace_strdup(func
);
8636 probe
->dtpr_name
= dtrace_strdup(name
);
8637 probe
->dtpr_arg
= arg
;
8638 probe
->dtpr_aframes
= aframes
;
8639 probe
->dtpr_provider
= provider
;
8641 dtrace_hash_add(dtrace_bymod
, probe
);
8642 dtrace_hash_add(dtrace_byfunc
, probe
);
8643 dtrace_hash_add(dtrace_byname
, probe
);
8645 if (id
- 1 >= dtrace_nprobes
) {
8646 size_t osize
= dtrace_nprobes
* sizeof (dtrace_probe_t
*);
8647 size_t nsize
= osize
<< 1;
8651 ASSERT(dtrace_probes
== NULL
);
8652 nsize
= sizeof (dtrace_probe_t
*);
8655 probes
= kmem_zalloc(nsize
, KM_SLEEP
);
8657 if (dtrace_probes
== NULL
) {
8659 dtrace_probes
= probes
;
8662 dtrace_probe_t
**oprobes
= dtrace_probes
;
8664 bcopy(oprobes
, probes
, osize
);
8665 dtrace_membar_producer();
8666 dtrace_probes
= probes
;
8671 * All CPUs are now seeing the new probes array; we can
8672 * safely free the old array.
8674 kmem_free(oprobes
, osize
);
8675 dtrace_nprobes
<<= 1;
8678 ASSERT(id
- 1 < dtrace_nprobes
);
8681 ASSERT(dtrace_probes
[id
- 1] == NULL
);
8682 dtrace_probes
[id
- 1] = probe
;
8684 if (provider
!= dtrace_provider
)
8685 mutex_exit(&dtrace_lock
);
8690 static dtrace_probe_t
*
8691 dtrace_probe_lookup_id(dtrace_id_t id
)
8693 ASSERT(MUTEX_HELD(&dtrace_lock
));
8695 if (id
== 0 || id
> dtrace_nprobes
)
8698 return (dtrace_probes
[id
- 1]);
8702 dtrace_probe_lookup_match(dtrace_probe_t
*probe
, void *arg
)
8704 *((dtrace_id_t
*)arg
) = probe
->dtpr_id
;
8706 return (DTRACE_MATCH_DONE
);
8710 * Look up a probe based on provider and one or more of module name, function
8711 * name and probe name.
8714 dtrace_probe_lookup(dtrace_provider_id_t prid
, const char *mod
,
8715 const char *func
, const char *name
)
8717 dtrace_probekey_t pkey
;
8721 pkey
.dtpk_prov
= ((dtrace_provider_t
*)prid
)->dtpv_name
;
8722 pkey
.dtpk_pmatch
= &dtrace_match_string
;
8723 pkey
.dtpk_mod
= mod
;
8724 pkey
.dtpk_mmatch
= mod
? &dtrace_match_string
: &dtrace_match_nul
;
8725 pkey
.dtpk_func
= func
;
8726 pkey
.dtpk_fmatch
= func
? &dtrace_match_string
: &dtrace_match_nul
;
8727 pkey
.dtpk_name
= name
;
8728 pkey
.dtpk_nmatch
= name
? &dtrace_match_string
: &dtrace_match_nul
;
8729 pkey
.dtpk_id
= DTRACE_IDNONE
;
8731 mutex_enter(&dtrace_lock
);
8732 match
= dtrace_match(&pkey
, DTRACE_PRIV_ALL
, 0, 0,
8733 dtrace_probe_lookup_match
, &id
);
8734 mutex_exit(&dtrace_lock
);
8736 ASSERT(match
== 1 || match
== 0);
8737 return (match
? id
: 0);
8741 * Returns the probe argument associated with the specified probe.
8744 dtrace_probe_arg(dtrace_provider_id_t id
, dtrace_id_t pid
)
8746 dtrace_probe_t
*probe
;
8749 mutex_enter(&dtrace_lock
);
8751 if ((probe
= dtrace_probe_lookup_id(pid
)) != NULL
&&
8752 probe
->dtpr_provider
== (dtrace_provider_t
*)id
)
8753 rval
= probe
->dtpr_arg
;
8755 mutex_exit(&dtrace_lock
);
8761 * Copy a probe into a probe description.
8764 dtrace_probe_description(const dtrace_probe_t
*prp
, dtrace_probedesc_t
*pdp
)
8766 bzero(pdp
, sizeof (dtrace_probedesc_t
));
8767 pdp
->dtpd_id
= prp
->dtpr_id
;
8769 (void) strncpy(pdp
->dtpd_provider
,
8770 prp
->dtpr_provider
->dtpv_name
, DTRACE_PROVNAMELEN
- 1);
8772 (void) strncpy(pdp
->dtpd_mod
, prp
->dtpr_mod
, DTRACE_MODNAMELEN
- 1);
8773 (void) strncpy(pdp
->dtpd_func
, prp
->dtpr_func
, DTRACE_FUNCNAMELEN
- 1);
8774 (void) strncpy(pdp
->dtpd_name
, prp
->dtpr_name
, DTRACE_NAMELEN
- 1);
8778 * Called to indicate that a probe -- or probes -- should be provided by a
8779 * specfied provider. If the specified description is NULL, the provider will
8780 * be told to provide all of its probes. (This is done whenever a new
8781 * consumer comes along, or whenever a retained enabling is to be matched.) If
8782 * the specified description is non-NULL, the provider is given the
8783 * opportunity to dynamically provide the specified probe, allowing providers
8784 * to support the creation of probes on-the-fly. (So-called _autocreated_
8785 * probes.) If the provider is NULL, the operations will be applied to all
8786 * providers; if the provider is non-NULL the operations will only be applied
8787 * to the specified provider. The dtrace_provider_lock must be held, and the
8788 * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
8789 * will need to grab the dtrace_lock when it reenters the framework through
8790 * dtrace_probe_lookup(), dtrace_probe_create(), etc.
8793 dtrace_probe_provide(dtrace_probedesc_t
*desc
, dtrace_provider_t
*prv
)
8798 ASSERT(MUTEX_HELD(&dtrace_provider_lock
));
8802 prv
= dtrace_provider
;
8807 * First, call the blanket provide operation.
8809 prv
->dtpv_pops
.dtps_provide(prv
->dtpv_arg
, desc
);
8812 * Now call the per-module provide operation. We will grab
8813 * mod_lock to prevent the list from being modified. Note
8814 * that this also prevents the mod_busy bits from changing.
8815 * (mod_busy can only be changed with mod_lock held.)
8817 mutex_enter(&mod_lock
);
8821 if (ctl
->mod_busy
|| ctl
->mod_mp
== NULL
)
8824 prv
->dtpv_pops
.dtps_provide_module(prv
->dtpv_arg
, ctl
);
8826 } while ((ctl
= ctl
->mod_next
) != &modules
);
8828 mutex_exit(&mod_lock
);
8829 } while (all
&& (prv
= prv
->dtpv_next
) != NULL
);
8833 * Iterate over each probe, and call the Framework-to-Provider API function
8837 dtrace_probe_foreach(uintptr_t offs
)
8839 dtrace_provider_t
*prov
;
8840 void (*func
)(void *, dtrace_id_t
, void *);
8841 dtrace_probe_t
*probe
;
8842 dtrace_icookie_t cookie
;
8846 * We disable interrupts to walk through the probe array. This is
8847 * safe -- the dtrace_sync() in dtrace_unregister() assures that we
8848 * won't see stale data.
8850 cookie
= dtrace_interrupt_disable();
8852 for (i
= 0; i
< dtrace_nprobes
; i
++) {
8853 if ((probe
= dtrace_probes
[i
]) == NULL
)
8856 if (probe
->dtpr_ecb
== NULL
) {
8858 * This probe isn't enabled -- don't call the function.
8863 prov
= probe
->dtpr_provider
;
8864 func
= *((void(**)(void *, dtrace_id_t
, void *))
8865 ((uintptr_t)&prov
->dtpv_pops
+ offs
));
8867 func(prov
->dtpv_arg
, i
+ 1, probe
->dtpr_arg
);
8870 dtrace_interrupt_enable(cookie
);
8874 dtrace_probe_enable(const dtrace_probedesc_t
*desc
, dtrace_enabling_t
*enab
)
8876 dtrace_probekey_t pkey
;
8881 ASSERT(MUTEX_HELD(&dtrace_lock
));
8882 dtrace_ecb_create_cache
= NULL
;
8886 * If we're passed a NULL description, we're being asked to
8887 * create an ECB with a NULL probe.
8889 (void) dtrace_ecb_create_enable(NULL
, enab
);
8893 dtrace_probekey(desc
, &pkey
);
8894 dtrace_cred2priv(enab
->dten_vstate
->dtvs_state
->dts_cred
.dcr_cred
,
8895 &priv
, &uid
, &zoneid
);
8897 return (dtrace_match(&pkey
, priv
, uid
, zoneid
, dtrace_ecb_create_enable
,
8902 * DTrace Helper Provider Functions
8905 dtrace_dofattr2attr(dtrace_attribute_t
*attr
, const dof_attr_t dofattr
)
8907 attr
->dtat_name
= DOF_ATTR_NAME(dofattr
);
8908 attr
->dtat_data
= DOF_ATTR_DATA(dofattr
);
8909 attr
->dtat_class
= DOF_ATTR_CLASS(dofattr
);
8913 dtrace_dofprov2hprov(dtrace_helper_provdesc_t
*hprov
,
8914 const dof_provider_t
*dofprov
, char *strtab
)
8916 hprov
->dthpv_provname
= strtab
+ dofprov
->dofpv_name
;
8917 dtrace_dofattr2attr(&hprov
->dthpv_pattr
.dtpa_provider
,
8918 dofprov
->dofpv_provattr
);
8919 dtrace_dofattr2attr(&hprov
->dthpv_pattr
.dtpa_mod
,
8920 dofprov
->dofpv_modattr
);
8921 dtrace_dofattr2attr(&hprov
->dthpv_pattr
.dtpa_func
,
8922 dofprov
->dofpv_funcattr
);
8923 dtrace_dofattr2attr(&hprov
->dthpv_pattr
.dtpa_name
,
8924 dofprov
->dofpv_nameattr
);
8925 dtrace_dofattr2attr(&hprov
->dthpv_pattr
.dtpa_args
,
8926 dofprov
->dofpv_argsattr
);
8930 dtrace_helper_provide_one(dof_helper_t
*dhp
, dof_sec_t
*sec
, pid_t pid
)
8932 uintptr_t daddr
= (uintptr_t)dhp
->dofhp_dof
;
8933 dof_hdr_t
*dof
= (dof_hdr_t
*)daddr
;
8934 dof_sec_t
*str_sec
, *prb_sec
, *arg_sec
, *off_sec
, *enoff_sec
;
8935 dof_provider_t
*provider
;
8937 uint32_t *off
, *enoff
;
8941 dtrace_helper_provdesc_t dhpv
;
8942 dtrace_helper_probedesc_t dhpb
;
8943 dtrace_meta_t
*meta
= dtrace_meta_pid
;
8944 dtrace_mops_t
*mops
= &meta
->dtm_mops
;
8947 provider
= (dof_provider_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
8948 str_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
8949 provider
->dofpv_strtab
* dof
->dofh_secsize
);
8950 prb_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
8951 provider
->dofpv_probes
* dof
->dofh_secsize
);
8952 arg_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
8953 provider
->dofpv_prargs
* dof
->dofh_secsize
);
8954 off_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
8955 provider
->dofpv_proffs
* dof
->dofh_secsize
);
8957 strtab
= (char *)(uintptr_t)(daddr
+ str_sec
->dofs_offset
);
8958 off
= (uint32_t *)(uintptr_t)(daddr
+ off_sec
->dofs_offset
);
8959 arg
= (uint8_t *)(uintptr_t)(daddr
+ arg_sec
->dofs_offset
);
8963 * See dtrace_helper_provider_validate().
8965 if (dof
->dofh_ident
[DOF_ID_VERSION
] != DOF_VERSION_1
&&
8966 provider
->dofpv_prenoffs
!= DOF_SECT_NONE
) {
8967 enoff_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
8968 provider
->dofpv_prenoffs
* dof
->dofh_secsize
);
8969 enoff
= (uint32_t *)(uintptr_t)(daddr
+ enoff_sec
->dofs_offset
);
8972 nprobes
= prb_sec
->dofs_size
/ prb_sec
->dofs_entsize
;
8975 * Create the provider.
8977 dtrace_dofprov2hprov(&dhpv
, provider
, strtab
);
8979 if ((parg
= mops
->dtms_provide_pid(meta
->dtm_arg
, &dhpv
, pid
)) == NULL
)
8985 * Create the probes.
8987 for (i
= 0; i
< nprobes
; i
++) {
8988 probe
= (dof_probe_t
*)(uintptr_t)(daddr
+
8989 prb_sec
->dofs_offset
+ i
* prb_sec
->dofs_entsize
);
8991 dhpb
.dthpb_mod
= dhp
->dofhp_mod
;
8992 dhpb
.dthpb_func
= strtab
+ probe
->dofpr_func
;
8993 dhpb
.dthpb_name
= strtab
+ probe
->dofpr_name
;
8994 dhpb
.dthpb_base
= probe
->dofpr_addr
;
8995 dhpb
.dthpb_offs
= off
+ probe
->dofpr_offidx
;
8996 dhpb
.dthpb_noffs
= probe
->dofpr_noffs
;
8997 if (enoff
!= NULL
) {
8998 dhpb
.dthpb_enoffs
= enoff
+ probe
->dofpr_enoffidx
;
8999 dhpb
.dthpb_nenoffs
= probe
->dofpr_nenoffs
;
9001 dhpb
.dthpb_enoffs
= NULL
;
9002 dhpb
.dthpb_nenoffs
= 0;
9004 dhpb
.dthpb_args
= arg
+ probe
->dofpr_argidx
;
9005 dhpb
.dthpb_nargc
= probe
->dofpr_nargc
;
9006 dhpb
.dthpb_xargc
= probe
->dofpr_xargc
;
9007 dhpb
.dthpb_ntypes
= strtab
+ probe
->dofpr_nargv
;
9008 dhpb
.dthpb_xtypes
= strtab
+ probe
->dofpr_xargv
;
9010 mops
->dtms_create_probe(meta
->dtm_arg
, parg
, &dhpb
);
9015 dtrace_helper_provide(dof_helper_t
*dhp
, pid_t pid
)
9017 uintptr_t daddr
= (uintptr_t)dhp
->dofhp_dof
;
9018 dof_hdr_t
*dof
= (dof_hdr_t
*)daddr
;
9021 ASSERT(MUTEX_HELD(&dtrace_meta_lock
));
9023 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
9024 dof_sec_t
*sec
= (dof_sec_t
*)(uintptr_t)(daddr
+
9025 dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
9027 if (sec
->dofs_type
!= DOF_SECT_PROVIDER
)
9030 dtrace_helper_provide_one(dhp
, sec
, pid
);
9034 * We may have just created probes, so we must now rematch against
9035 * any retained enablings. Note that this call will acquire both
9036 * cpu_lock and dtrace_lock; the fact that we are holding
9037 * dtrace_meta_lock now is what defines the ordering with respect to
9038 * these three locks.
9040 dtrace_enabling_matchall();
9044 dtrace_helper_provider_remove_one(dof_helper_t
*dhp
, dof_sec_t
*sec
, pid_t pid
)
9046 uintptr_t daddr
= (uintptr_t)dhp
->dofhp_dof
;
9047 dof_hdr_t
*dof
= (dof_hdr_t
*)daddr
;
9049 dof_provider_t
*provider
;
9051 dtrace_helper_provdesc_t dhpv
;
9052 dtrace_meta_t
*meta
= dtrace_meta_pid
;
9053 dtrace_mops_t
*mops
= &meta
->dtm_mops
;
9055 provider
= (dof_provider_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
9056 str_sec
= (dof_sec_t
*)(uintptr_t)(daddr
+ dof
->dofh_secoff
+
9057 provider
->dofpv_strtab
* dof
->dofh_secsize
);
9059 strtab
= (char *)(uintptr_t)(daddr
+ str_sec
->dofs_offset
);
9062 * Create the provider.
9064 dtrace_dofprov2hprov(&dhpv
, provider
, strtab
);
9066 mops
->dtms_remove_pid(meta
->dtm_arg
, &dhpv
, pid
);
9072 dtrace_helper_provider_remove(dof_helper_t
*dhp
, pid_t pid
)
9074 uintptr_t daddr
= (uintptr_t)dhp
->dofhp_dof
;
9075 dof_hdr_t
*dof
= (dof_hdr_t
*)daddr
;
9078 ASSERT(MUTEX_HELD(&dtrace_meta_lock
));
9080 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
9081 dof_sec_t
*sec
= (dof_sec_t
*)(uintptr_t)(daddr
+
9082 dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
9084 if (sec
->dofs_type
!= DOF_SECT_PROVIDER
)
9087 dtrace_helper_provider_remove_one(dhp
, sec
, pid
);
9092 * DTrace Meta Provider-to-Framework API Functions
9094 * These functions implement the Meta Provider-to-Framework API, as described
9095 * in <sys/dtrace.h>.
9098 dtrace_meta_register(const char *name
, const dtrace_mops_t
*mops
, void *arg
,
9099 dtrace_meta_provider_id_t
*idp
)
9101 dtrace_meta_t
*meta
;
9102 dtrace_helpers_t
*help
, *next
;
9105 *idp
= DTRACE_METAPROVNONE
;
9108 * We strictly don't need the name, but we hold onto it for
9109 * debuggability. All hail error queues!
9112 cmn_err(CE_WARN
, "failed to register meta-provider: "
9118 mops
->dtms_create_probe
== NULL
||
9119 mops
->dtms_provide_pid
== NULL
||
9120 mops
->dtms_remove_pid
== NULL
) {
9121 cmn_err(CE_WARN
, "failed to register meta-register %s: "
9122 "invalid ops", name
);
9126 meta
= kmem_zalloc(sizeof (dtrace_meta_t
), KM_SLEEP
);
9127 meta
->dtm_mops
= *mops
;
9128 meta
->dtm_name
= kmem_alloc(strlen(name
) + 1, KM_SLEEP
);
9129 (void) strcpy(meta
->dtm_name
, name
);
9130 meta
->dtm_arg
= arg
;
9132 mutex_enter(&dtrace_meta_lock
);
9133 mutex_enter(&dtrace_lock
);
9135 if (dtrace_meta_pid
!= NULL
) {
9136 mutex_exit(&dtrace_lock
);
9137 mutex_exit(&dtrace_meta_lock
);
9138 cmn_err(CE_WARN
, "failed to register meta-register %s: "
9139 "user-land meta-provider exists", name
);
9140 kmem_free(meta
->dtm_name
, strlen(meta
->dtm_name
) + 1);
9141 kmem_free(meta
, sizeof (dtrace_meta_t
));
9145 dtrace_meta_pid
= meta
;
9146 *idp
= (dtrace_meta_provider_id_t
)meta
;
9149 * If there are providers and probes ready to go, pass them
9150 * off to the new meta provider now.
9153 help
= dtrace_deferred_pid
;
9154 dtrace_deferred_pid
= NULL
;
9156 mutex_exit(&dtrace_lock
);
9158 while (help
!= NULL
) {
9159 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
9160 dtrace_helper_provide(&help
->dthps_provs
[i
]->dthp_prov
,
9164 next
= help
->dthps_next
;
9165 help
->dthps_next
= NULL
;
9166 help
->dthps_prev
= NULL
;
9167 help
->dthps_deferred
= 0;
9171 mutex_exit(&dtrace_meta_lock
);
9177 dtrace_meta_unregister(dtrace_meta_provider_id_t id
)
9179 dtrace_meta_t
**pp
, *old
= (dtrace_meta_t
*)id
;
9181 mutex_enter(&dtrace_meta_lock
);
9182 mutex_enter(&dtrace_lock
);
9184 if (old
== dtrace_meta_pid
) {
9185 pp
= &dtrace_meta_pid
;
9187 panic("attempt to unregister non-existent "
9188 "dtrace meta-provider %p\n", (void *)old
);
9191 if (old
->dtm_count
!= 0) {
9192 mutex_exit(&dtrace_lock
);
9193 mutex_exit(&dtrace_meta_lock
);
9199 mutex_exit(&dtrace_lock
);
9200 mutex_exit(&dtrace_meta_lock
);
9202 kmem_free(old
->dtm_name
, strlen(old
->dtm_name
) + 1);
9203 kmem_free(old
, sizeof (dtrace_meta_t
));
9210 * DTrace DIF Object Functions
9213 dtrace_difo_err(uint_t pc
, const char *format
, ...)
9215 if (dtrace_err_verbose
) {
9218 (void) uprintf("dtrace DIF object error: [%u]: ", pc
);
9219 va_start(alist
, format
);
9220 (void) vuprintf(format
, alist
);
9224 #ifdef DTRACE_ERRDEBUG
9225 dtrace_errdebug(format
);
9231 * Validate a DTrace DIF object by checking the IR instructions. The following
9232 * rules are currently enforced by dtrace_difo_validate():
9234 * 1. Each instruction must have a valid opcode
9235 * 2. Each register, string, variable, or subroutine reference must be valid
9236 * 3. No instruction can modify register %r0 (must be zero)
9237 * 4. All instruction reserved bits must be set to zero
9238 * 5. The last instruction must be a "ret" instruction
9239 * 6. All branch targets must reference a valid instruction _after_ the branch
9242 dtrace_difo_validate(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
, uint_t nregs
,
9246 int (*efunc
)(uint_t pc
, const char *, ...) = dtrace_difo_err
;
9249 int maxglobal
= -1, maxlocal
= -1, maxtlocal
= -1;
9251 kcheckload
= cr
== NULL
||
9252 (vstate
->dtvs_state
->dts_cred
.dcr_visible
& DTRACE_CRV_KERNEL
) == 0;
9254 dp
->dtdo_destructive
= 0;
9256 for (pc
= 0; pc
< dp
->dtdo_len
&& err
== 0; pc
++) {
9257 dif_instr_t instr
= dp
->dtdo_buf
[pc
];
9259 uint_t r1
= DIF_INSTR_R1(instr
);
9260 uint_t r2
= DIF_INSTR_R2(instr
);
9261 uint_t rd
= DIF_INSTR_RD(instr
);
9262 uint_t rs
= DIF_INSTR_RS(instr
);
9263 uint_t label
= DIF_INSTR_LABEL(instr
);
9264 uint_t v
= DIF_INSTR_VAR(instr
);
9265 uint_t subr
= DIF_INSTR_SUBR(instr
);
9266 uint_t type
= DIF_INSTR_TYPE(instr
);
9267 uint_t op
= DIF_INSTR_OP(instr
);
9285 err
+= efunc(pc
, "invalid register %u\n", r1
);
9287 err
+= efunc(pc
, "invalid register %u\n", r2
);
9289 err
+= efunc(pc
, "invalid register %u\n", rd
);
9291 err
+= efunc(pc
, "cannot write to %r0\n");
9297 err
+= efunc(pc
, "invalid register %u\n", r1
);
9299 err
+= efunc(pc
, "non-zero reserved bits\n");
9301 err
+= efunc(pc
, "invalid register %u\n", rd
);
9303 err
+= efunc(pc
, "cannot write to %r0\n");
9313 err
+= efunc(pc
, "invalid register %u\n", r1
);
9315 err
+= efunc(pc
, "non-zero reserved bits\n");
9317 err
+= efunc(pc
, "invalid register %u\n", rd
);
9319 err
+= efunc(pc
, "cannot write to %r0\n");
9321 dp
->dtdo_buf
[pc
] = DIF_INSTR_LOAD(op
+
9322 DIF_OP_RLDSB
- DIF_OP_LDSB
, r1
, rd
);
9332 err
+= efunc(pc
, "invalid register %u\n", r1
);
9334 err
+= efunc(pc
, "non-zero reserved bits\n");
9336 err
+= efunc(pc
, "invalid register %u\n", rd
);
9338 err
+= efunc(pc
, "cannot write to %r0\n");
9348 err
+= efunc(pc
, "invalid register %u\n", r1
);
9350 err
+= efunc(pc
, "non-zero reserved bits\n");
9352 err
+= efunc(pc
, "invalid register %u\n", rd
);
9354 err
+= efunc(pc
, "cannot write to %r0\n");
9361 err
+= efunc(pc
, "invalid register %u\n", r1
);
9363 err
+= efunc(pc
, "non-zero reserved bits\n");
9365 err
+= efunc(pc
, "invalid register %u\n", rd
);
9367 err
+= efunc(pc
, "cannot write to 0 address\n");
9372 err
+= efunc(pc
, "invalid register %u\n", r1
);
9374 err
+= efunc(pc
, "invalid register %u\n", r2
);
9376 err
+= efunc(pc
, "non-zero reserved bits\n");
9380 err
+= efunc(pc
, "invalid register %u\n", r1
);
9381 if (r2
!= 0 || rd
!= 0)
9382 err
+= efunc(pc
, "non-zero reserved bits\n");
9395 if (label
>= dp
->dtdo_len
) {
9396 err
+= efunc(pc
, "invalid branch target %u\n",
9400 err
+= efunc(pc
, "backward branch to %u\n",
9405 if (r1
!= 0 || r2
!= 0)
9406 err
+= efunc(pc
, "non-zero reserved bits\n");
9408 err
+= efunc(pc
, "invalid register %u\n", rd
);
9412 case DIF_OP_FLUSHTS
:
9413 if (r1
!= 0 || r2
!= 0 || rd
!= 0)
9414 err
+= efunc(pc
, "non-zero reserved bits\n");
9417 if (DIF_INSTR_INTEGER(instr
) >= dp
->dtdo_intlen
) {
9418 err
+= efunc(pc
, "invalid integer ref %u\n",
9419 DIF_INSTR_INTEGER(instr
));
9422 err
+= efunc(pc
, "invalid register %u\n", rd
);
9424 err
+= efunc(pc
, "cannot write to %r0\n");
9427 if (DIF_INSTR_STRING(instr
) >= dp
->dtdo_strlen
) {
9428 err
+= efunc(pc
, "invalid string ref %u\n",
9429 DIF_INSTR_STRING(instr
));
9432 err
+= efunc(pc
, "invalid register %u\n", rd
);
9434 err
+= efunc(pc
, "cannot write to %r0\n");
9438 if (r1
> DIF_VAR_ARRAY_MAX
)
9439 err
+= efunc(pc
, "invalid array %u\n", r1
);
9441 err
+= efunc(pc
, "invalid register %u\n", r2
);
9443 err
+= efunc(pc
, "invalid register %u\n", rd
);
9445 err
+= efunc(pc
, "cannot write to %r0\n");
9448 if (r1
> DIF_VAR_ARRAY_MAX
)
9449 err
+= efunc(pc
, "invalid array %u\n", r1
);
9451 err
+= efunc(pc
, "invalid register %u\n", r2
);
9453 err
+= efunc(pc
, "invalid register %u\n", rd
);
9454 dp
->dtdo_destructive
= 1;
9461 if (v
< DIF_VAR_OTHER_MIN
|| v
> DIF_VAR_OTHER_MAX
)
9462 err
+= efunc(pc
, "invalid variable %u\n", v
);
9464 err
+= efunc(pc
, "invalid register %u\n", rd
);
9466 err
+= efunc(pc
, "cannot write to %r0\n");
9473 if (v
< DIF_VAR_OTHER_UBASE
|| v
> DIF_VAR_OTHER_MAX
)
9474 err
+= efunc(pc
, "invalid variable %u\n", v
);
9476 err
+= efunc(pc
, "invalid register %u\n", rd
);
9479 if (subr
> DIF_SUBR_MAX
)
9480 err
+= efunc(pc
, "invalid subr %u\n", subr
);
9482 err
+= efunc(pc
, "invalid register %u\n", rd
);
9484 err
+= efunc(pc
, "cannot write to %r0\n");
9486 if (subr
== DIF_SUBR_COPYOUT
||
9487 subr
== DIF_SUBR_COPYOUTSTR
) {
9488 dp
->dtdo_destructive
= 1;
9491 if (subr
== DIF_SUBR_GETF
) {
9493 * If we have a getf() we need to record that
9494 * in our state. Note that our state can be
9495 * NULL if this is a helper -- but in that
9496 * case, the call to getf() is itself illegal,
9497 * and will be caught (slightly later) when
9498 * the helper is validated.
9500 if (vstate
->dtvs_state
!= NULL
)
9501 vstate
->dtvs_state
->dts_getf
++;
9506 if (type
!= DIF_TYPE_STRING
&& type
!= DIF_TYPE_CTF
)
9507 err
+= efunc(pc
, "invalid ref type %u\n", type
);
9509 err
+= efunc(pc
, "invalid register %u\n", r2
);
9511 err
+= efunc(pc
, "invalid register %u\n", rs
);
9514 if (type
!= DIF_TYPE_CTF
)
9515 err
+= efunc(pc
, "invalid val type %u\n", type
);
9517 err
+= efunc(pc
, "invalid register %u\n", r2
);
9519 err
+= efunc(pc
, "invalid register %u\n", rs
);
9522 err
+= efunc(pc
, "invalid opcode %u\n",
9523 DIF_INSTR_OP(instr
));
9527 if (dp
->dtdo_len
!= 0 &&
9528 DIF_INSTR_OP(dp
->dtdo_buf
[dp
->dtdo_len
- 1]) != DIF_OP_RET
) {
9529 err
+= efunc(dp
->dtdo_len
- 1,
9530 "expected 'ret' as last DIF instruction\n");
9533 if (!(dp
->dtdo_rtype
.dtdt_flags
& (DIF_TF_BYREF
| DIF_TF_BYUREF
))) {
9535 * If we're not returning by reference, the size must be either
9536 * 0 or the size of one of the base types.
9538 switch (dp
->dtdo_rtype
.dtdt_size
) {
9540 case sizeof (uint8_t):
9541 case sizeof (uint16_t):
9542 case sizeof (uint32_t):
9543 case sizeof (uint64_t):
9547 err
+= efunc(dp
->dtdo_len
- 1, "bad return size\n");
9551 for (i
= 0; i
< dp
->dtdo_varlen
&& err
== 0; i
++) {
9552 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
], *existing
= NULL
;
9553 dtrace_diftype_t
*vt
, *et
;
9556 if (v
->dtdv_scope
!= DIFV_SCOPE_GLOBAL
&&
9557 v
->dtdv_scope
!= DIFV_SCOPE_THREAD
&&
9558 v
->dtdv_scope
!= DIFV_SCOPE_LOCAL
) {
9559 err
+= efunc(i
, "unrecognized variable scope %d\n",
9564 if (v
->dtdv_kind
!= DIFV_KIND_ARRAY
&&
9565 v
->dtdv_kind
!= DIFV_KIND_SCALAR
) {
9566 err
+= efunc(i
, "unrecognized variable type %d\n",
9571 if ((id
= v
->dtdv_id
) > DIF_VARIABLE_MAX
) {
9572 err
+= efunc(i
, "%d exceeds variable id limit\n", id
);
9576 if (id
< DIF_VAR_OTHER_UBASE
)
9580 * For user-defined variables, we need to check that this
9581 * definition is identical to any previous definition that we
9584 ndx
= id
- DIF_VAR_OTHER_UBASE
;
9586 switch (v
->dtdv_scope
) {
9587 case DIFV_SCOPE_GLOBAL
:
9588 if (maxglobal
== -1 || ndx
> maxglobal
)
9591 if (ndx
< vstate
->dtvs_nglobals
) {
9592 dtrace_statvar_t
*svar
;
9594 if ((svar
= vstate
->dtvs_globals
[ndx
]) != NULL
)
9595 existing
= &svar
->dtsv_var
;
9600 case DIFV_SCOPE_THREAD
:
9601 if (maxtlocal
== -1 || ndx
> maxtlocal
)
9604 if (ndx
< vstate
->dtvs_ntlocals
)
9605 existing
= &vstate
->dtvs_tlocals
[ndx
];
9608 case DIFV_SCOPE_LOCAL
:
9609 if (maxlocal
== -1 || ndx
> maxlocal
)
9612 if (ndx
< vstate
->dtvs_nlocals
) {
9613 dtrace_statvar_t
*svar
;
9615 if ((svar
= vstate
->dtvs_locals
[ndx
]) != NULL
)
9616 existing
= &svar
->dtsv_var
;
9624 if (vt
->dtdt_flags
& DIF_TF_BYREF
) {
9625 if (vt
->dtdt_size
== 0) {
9626 err
+= efunc(i
, "zero-sized variable\n");
9630 if ((v
->dtdv_scope
== DIFV_SCOPE_GLOBAL
||
9631 v
->dtdv_scope
== DIFV_SCOPE_LOCAL
) &&
9632 vt
->dtdt_size
> dtrace_statvar_maxsize
) {
9633 err
+= efunc(i
, "oversized by-ref static\n");
9638 if (existing
== NULL
|| existing
->dtdv_id
== 0)
9641 ASSERT(existing
->dtdv_id
== v
->dtdv_id
);
9642 ASSERT(existing
->dtdv_scope
== v
->dtdv_scope
);
9644 if (existing
->dtdv_kind
!= v
->dtdv_kind
)
9645 err
+= efunc(i
, "%d changed variable kind\n", id
);
9647 et
= &existing
->dtdv_type
;
9649 if (vt
->dtdt_flags
!= et
->dtdt_flags
) {
9650 err
+= efunc(i
, "%d changed variable type flags\n", id
);
9654 if (vt
->dtdt_size
!= 0 && vt
->dtdt_size
!= et
->dtdt_size
) {
9655 err
+= efunc(i
, "%d changed variable type size\n", id
);
9660 for (pc
= 0; pc
< dp
->dtdo_len
&& err
== 0; pc
++) {
9661 dif_instr_t instr
= dp
->dtdo_buf
[pc
];
9663 uint_t v
= DIF_INSTR_VAR(instr
);
9664 uint_t op
= DIF_INSTR_OP(instr
);
9671 if (v
> DIF_VAR_OTHER_UBASE
+ maxglobal
)
9672 err
+= efunc(pc
, "invalid variable %u\n", v
);
9678 if (v
> DIF_VAR_OTHER_UBASE
+ maxtlocal
)
9679 err
+= efunc(pc
, "invalid variable %u\n", v
);
9683 if (v
> DIF_VAR_OTHER_UBASE
+ maxlocal
)
9684 err
+= efunc(pc
, "invalid variable %u\n", v
);
9695 * Validate a DTrace DIF object that it is to be used as a helper. Helpers
9696 * are much more constrained than normal DIFOs. Specifically, they may
9699 * 1. Make calls to subroutines other than copyin(), copyinstr() or
9700 * miscellaneous string routines
9701 * 2. Access DTrace variables other than the args[] array, and the
9702 * curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
9703 * 3. Have thread-local variables.
9704 * 4. Have dynamic variables.
9707 dtrace_difo_validate_helper(dtrace_difo_t
*dp
)
9709 int (*efunc
)(uint_t pc
, const char *, ...) = dtrace_difo_err
;
9713 for (pc
= 0; pc
< dp
->dtdo_len
; pc
++) {
9714 dif_instr_t instr
= dp
->dtdo_buf
[pc
];
9716 uint_t v
= DIF_INSTR_VAR(instr
);
9717 uint_t subr
= DIF_INSTR_SUBR(instr
);
9718 uint_t op
= DIF_INSTR_OP(instr
);
9773 case DIF_OP_FLUSHTS
:
9785 if (v
>= DIF_VAR_OTHER_UBASE
)
9788 if (v
>= DIF_VAR_ARG0
&& v
<= DIF_VAR_ARG9
)
9791 if (v
== DIF_VAR_CURTHREAD
|| v
== DIF_VAR_PID
||
9792 v
== DIF_VAR_PPID
|| v
== DIF_VAR_TID
||
9793 v
== DIF_VAR_EXECNAME
|| v
== DIF_VAR_ZONENAME
||
9794 v
== DIF_VAR_UID
|| v
== DIF_VAR_GID
)
9797 err
+= efunc(pc
, "illegal variable %u\n", v
);
9801 if (v
< DIF_VAR_OTHER_UBASE
) {
9802 err
+= efunc(pc
, "illegal variable load\n");
9809 err
+= efunc(pc
, "illegal dynamic variable load\n");
9813 if (v
< DIF_VAR_OTHER_UBASE
) {
9814 err
+= efunc(pc
, "illegal variable store\n");
9821 err
+= efunc(pc
, "illegal dynamic variable store\n");
9825 if (subr
== DIF_SUBR_ALLOCA
||
9826 subr
== DIF_SUBR_BCOPY
||
9827 subr
== DIF_SUBR_COPYIN
||
9828 subr
== DIF_SUBR_COPYINTO
||
9829 subr
== DIF_SUBR_COPYINSTR
||
9830 subr
== DIF_SUBR_INDEX
||
9831 subr
== DIF_SUBR_INET_NTOA
||
9832 subr
== DIF_SUBR_INET_NTOA6
||
9833 subr
== DIF_SUBR_INET_NTOP
||
9834 subr
== DIF_SUBR_JSON
||
9835 subr
== DIF_SUBR_LLTOSTR
||
9836 subr
== DIF_SUBR_STRTOLL
||
9837 subr
== DIF_SUBR_RINDEX
||
9838 subr
== DIF_SUBR_STRCHR
||
9839 subr
== DIF_SUBR_STRJOIN
||
9840 subr
== DIF_SUBR_STRRCHR
||
9841 subr
== DIF_SUBR_STRSTR
||
9842 subr
== DIF_SUBR_HTONS
||
9843 subr
== DIF_SUBR_HTONL
||
9844 subr
== DIF_SUBR_HTONLL
||
9845 subr
== DIF_SUBR_NTOHS
||
9846 subr
== DIF_SUBR_NTOHL
||
9847 subr
== DIF_SUBR_NTOHLL
)
9850 err
+= efunc(pc
, "invalid subr %u\n", subr
);
9854 err
+= efunc(pc
, "invalid opcode %u\n",
9855 DIF_INSTR_OP(instr
));
9863 * Returns 1 if the expression in the DIF object can be cached on a per-thread
9867 dtrace_difo_cacheable(dtrace_difo_t
*dp
)
9874 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
9875 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
9877 if (v
->dtdv_scope
!= DIFV_SCOPE_GLOBAL
)
9880 switch (v
->dtdv_id
) {
9881 case DIF_VAR_CURTHREAD
:
9884 case DIF_VAR_EXECNAME
:
9885 case DIF_VAR_ZONENAME
:
9894 * This DIF object may be cacheable. Now we need to look for any
9895 * array loading instructions, any memory loading instructions, or
9896 * any stores to thread-local variables.
9898 for (i
= 0; i
< dp
->dtdo_len
; i
++) {
9899 uint_t op
= DIF_INSTR_OP(dp
->dtdo_buf
[i
]);
9901 if ((op
>= DIF_OP_LDSB
&& op
<= DIF_OP_LDX
) ||
9902 (op
>= DIF_OP_ULDSB
&& op
<= DIF_OP_ULDX
) ||
9903 (op
>= DIF_OP_RLDSB
&& op
<= DIF_OP_RLDX
) ||
9904 op
== DIF_OP_LDGA
|| op
== DIF_OP_STTS
)
9912 dtrace_difo_hold(dtrace_difo_t
*dp
)
9916 ASSERT(MUTEX_HELD(&dtrace_lock
));
9919 ASSERT(dp
->dtdo_refcnt
!= 0);
9922 * We need to check this DIF object for references to the variable
9923 * DIF_VAR_VTIMESTAMP.
9925 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
9926 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
9928 if (v
->dtdv_id
!= DIF_VAR_VTIMESTAMP
)
9931 if (dtrace_vtime_references
++ == 0)
9932 dtrace_vtime_enable();
9937 * This routine calculates the dynamic variable chunksize for a given DIF
9938 * object. The calculation is not fool-proof, and can probably be tricked by
9939 * malicious DIF -- but it works for all compiler-generated DIF. Because this
9940 * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
9941 * if a dynamic variable size exceeds the chunksize.
9944 dtrace_difo_chunksize(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
)
9947 dtrace_key_t tupregs
[DIF_DTR_NREGS
+ 2]; /* +2 for thread and id */
9948 const dif_instr_t
*text
= dp
->dtdo_buf
;
9954 for (pc
= 0; pc
< dp
->dtdo_len
; pc
++) {
9955 dif_instr_t instr
= text
[pc
];
9956 uint_t op
= DIF_INSTR_OP(instr
);
9957 uint_t rd
= DIF_INSTR_RD(instr
);
9958 uint_t r1
= DIF_INSTR_R1(instr
);
9962 dtrace_key_t
*key
= tupregs
;
9966 sval
= dp
->dtdo_inttab
[DIF_INSTR_INTEGER(instr
)];
9971 key
= &tupregs
[DIF_DTR_NREGS
];
9972 key
[0].dttk_size
= 0;
9973 key
[1].dttk_size
= 0;
9975 scope
= DIFV_SCOPE_THREAD
;
9982 if (DIF_INSTR_OP(instr
) == DIF_OP_STTAA
)
9983 key
[nkeys
++].dttk_size
= 0;
9985 key
[nkeys
++].dttk_size
= 0;
9987 if (op
== DIF_OP_STTAA
) {
9988 scope
= DIFV_SCOPE_THREAD
;
9990 scope
= DIFV_SCOPE_GLOBAL
;
9996 if (ttop
== DIF_DTR_NREGS
)
9999 if ((srd
== 0 || sval
== 0) && r1
== DIF_TYPE_STRING
) {
10001 * If the register for the size of the "pushtr"
10002 * is %r0 (or the value is 0) and the type is
10003 * a string, we'll use the system-wide default
10006 tupregs
[ttop
++].dttk_size
=
10007 dtrace_strsize_default
;
10012 if (sval
> LONG_MAX
)
10015 tupregs
[ttop
++].dttk_size
= sval
;
10020 case DIF_OP_PUSHTV
:
10021 if (ttop
== DIF_DTR_NREGS
)
10024 tupregs
[ttop
++].dttk_size
= 0;
10027 case DIF_OP_FLUSHTS
:
10044 * We have a dynamic variable allocation; calculate its size.
10046 for (ksize
= 0, i
= 0; i
< nkeys
; i
++)
10047 ksize
+= P2ROUNDUP(key
[i
].dttk_size
, sizeof (uint64_t));
10049 size
= sizeof (dtrace_dynvar_t
);
10050 size
+= sizeof (dtrace_key_t
) * (nkeys
- 1);
10054 * Now we need to determine the size of the stored data.
10056 id
= DIF_INSTR_VAR(instr
);
10058 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
10059 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
10061 if (v
->dtdv_id
== id
&& v
->dtdv_scope
== scope
) {
10062 size
+= v
->dtdv_type
.dtdt_size
;
10067 if (i
== dp
->dtdo_varlen
)
10071 * We have the size. If this is larger than the chunk size
10072 * for our dynamic variable state, reset the chunk size.
10074 size
= P2ROUNDUP(size
, sizeof (uint64_t));
10077 * Before setting the chunk size, check that we're not going
10078 * to set it to a negative value...
10080 if (size
> LONG_MAX
)
10084 * ...and make certain that we didn't badly overflow.
10086 if (size
< ksize
|| size
< sizeof (dtrace_dynvar_t
))
10089 if (size
> vstate
->dtvs_dynvars
.dtds_chunksize
)
10090 vstate
->dtvs_dynvars
.dtds_chunksize
= size
;
10095 dtrace_difo_init(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
)
10097 int i
, oldsvars
, osz
, nsz
, otlocals
, ntlocals
;
10100 ASSERT(MUTEX_HELD(&dtrace_lock
));
10101 ASSERT(dp
->dtdo_buf
!= NULL
&& dp
->dtdo_len
!= 0);
10103 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
10104 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
10105 dtrace_statvar_t
*svar
, ***svarp
;
10107 uint8_t scope
= v
->dtdv_scope
;
10110 if ((id
= v
->dtdv_id
) < DIF_VAR_OTHER_UBASE
)
10113 id
-= DIF_VAR_OTHER_UBASE
;
10116 case DIFV_SCOPE_THREAD
:
10117 while (id
>= (otlocals
= vstate
->dtvs_ntlocals
)) {
10118 dtrace_difv_t
*tlocals
;
10120 if ((ntlocals
= (otlocals
<< 1)) == 0)
10123 osz
= otlocals
* sizeof (dtrace_difv_t
);
10124 nsz
= ntlocals
* sizeof (dtrace_difv_t
);
10126 tlocals
= kmem_zalloc(nsz
, KM_SLEEP
);
10129 bcopy(vstate
->dtvs_tlocals
,
10131 kmem_free(vstate
->dtvs_tlocals
, osz
);
10134 vstate
->dtvs_tlocals
= tlocals
;
10135 vstate
->dtvs_ntlocals
= ntlocals
;
10138 vstate
->dtvs_tlocals
[id
] = *v
;
10141 case DIFV_SCOPE_LOCAL
:
10142 np
= &vstate
->dtvs_nlocals
;
10143 svarp
= &vstate
->dtvs_locals
;
10145 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
)
10146 dsize
= NCPU
* (v
->dtdv_type
.dtdt_size
+
10147 sizeof (uint64_t));
10149 dsize
= NCPU
* sizeof (uint64_t);
10153 case DIFV_SCOPE_GLOBAL
:
10154 np
= &vstate
->dtvs_nglobals
;
10155 svarp
= &vstate
->dtvs_globals
;
10157 if (v
->dtdv_type
.dtdt_flags
& DIF_TF_BYREF
)
10158 dsize
= v
->dtdv_type
.dtdt_size
+
10167 while (id
>= (oldsvars
= *np
)) {
10168 dtrace_statvar_t
**statics
;
10169 int newsvars
, oldsize
, newsize
;
10171 if ((newsvars
= (oldsvars
<< 1)) == 0)
10174 oldsize
= oldsvars
* sizeof (dtrace_statvar_t
*);
10175 newsize
= newsvars
* sizeof (dtrace_statvar_t
*);
10177 statics
= kmem_zalloc(newsize
, KM_SLEEP
);
10179 if (oldsize
!= 0) {
10180 bcopy(*svarp
, statics
, oldsize
);
10181 kmem_free(*svarp
, oldsize
);
10188 if ((svar
= (*svarp
)[id
]) == NULL
) {
10189 svar
= kmem_zalloc(sizeof (dtrace_statvar_t
), KM_SLEEP
);
10190 svar
->dtsv_var
= *v
;
10192 if ((svar
->dtsv_size
= dsize
) != 0) {
10193 svar
->dtsv_data
= (uint64_t)(uintptr_t)
10194 kmem_zalloc(dsize
, KM_SLEEP
);
10197 (*svarp
)[id
] = svar
;
10200 svar
->dtsv_refcnt
++;
10203 dtrace_difo_chunksize(dp
, vstate
);
10204 dtrace_difo_hold(dp
);
10207 static dtrace_difo_t
*
10208 dtrace_difo_duplicate(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
)
10210 dtrace_difo_t
*new;
10213 ASSERT(dp
->dtdo_buf
!= NULL
);
10214 ASSERT(dp
->dtdo_refcnt
!= 0);
10216 new = kmem_zalloc(sizeof (dtrace_difo_t
), KM_SLEEP
);
10218 ASSERT(dp
->dtdo_buf
!= NULL
);
10219 sz
= dp
->dtdo_len
* sizeof (dif_instr_t
);
10220 new->dtdo_buf
= kmem_alloc(sz
, KM_SLEEP
);
10221 bcopy(dp
->dtdo_buf
, new->dtdo_buf
, sz
);
10222 new->dtdo_len
= dp
->dtdo_len
;
10224 if (dp
->dtdo_strtab
!= NULL
) {
10225 ASSERT(dp
->dtdo_strlen
!= 0);
10226 new->dtdo_strtab
= kmem_alloc(dp
->dtdo_strlen
, KM_SLEEP
);
10227 bcopy(dp
->dtdo_strtab
, new->dtdo_strtab
, dp
->dtdo_strlen
);
10228 new->dtdo_strlen
= dp
->dtdo_strlen
;
10231 if (dp
->dtdo_inttab
!= NULL
) {
10232 ASSERT(dp
->dtdo_intlen
!= 0);
10233 sz
= dp
->dtdo_intlen
* sizeof (uint64_t);
10234 new->dtdo_inttab
= kmem_alloc(sz
, KM_SLEEP
);
10235 bcopy(dp
->dtdo_inttab
, new->dtdo_inttab
, sz
);
10236 new->dtdo_intlen
= dp
->dtdo_intlen
;
10239 if (dp
->dtdo_vartab
!= NULL
) {
10240 ASSERT(dp
->dtdo_varlen
!= 0);
10241 sz
= dp
->dtdo_varlen
* sizeof (dtrace_difv_t
);
10242 new->dtdo_vartab
= kmem_alloc(sz
, KM_SLEEP
);
10243 bcopy(dp
->dtdo_vartab
, new->dtdo_vartab
, sz
);
10244 new->dtdo_varlen
= dp
->dtdo_varlen
;
10247 dtrace_difo_init(new, vstate
);
10252 dtrace_difo_destroy(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
)
10256 ASSERT(dp
->dtdo_refcnt
== 0);
10258 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
10259 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
10260 dtrace_statvar_t
*svar
, **svarp
;
10262 uint8_t scope
= v
->dtdv_scope
;
10266 case DIFV_SCOPE_THREAD
:
10269 case DIFV_SCOPE_LOCAL
:
10270 np
= &vstate
->dtvs_nlocals
;
10271 svarp
= vstate
->dtvs_locals
;
10274 case DIFV_SCOPE_GLOBAL
:
10275 np
= &vstate
->dtvs_nglobals
;
10276 svarp
= vstate
->dtvs_globals
;
10283 if ((id
= v
->dtdv_id
) < DIF_VAR_OTHER_UBASE
)
10286 id
-= DIF_VAR_OTHER_UBASE
;
10290 ASSERT(svar
!= NULL
);
10291 ASSERT(svar
->dtsv_refcnt
> 0);
10293 if (--svar
->dtsv_refcnt
> 0)
10296 if (svar
->dtsv_size
!= 0) {
10297 ASSERT(svar
->dtsv_data
!= NULL
);
10298 kmem_free((void *)(uintptr_t)svar
->dtsv_data
,
10302 kmem_free(svar
, sizeof (dtrace_statvar_t
));
10306 kmem_free(dp
->dtdo_buf
, dp
->dtdo_len
* sizeof (dif_instr_t
));
10307 kmem_free(dp
->dtdo_inttab
, dp
->dtdo_intlen
* sizeof (uint64_t));
10308 kmem_free(dp
->dtdo_strtab
, dp
->dtdo_strlen
);
10309 kmem_free(dp
->dtdo_vartab
, dp
->dtdo_varlen
* sizeof (dtrace_difv_t
));
10311 kmem_free(dp
, sizeof (dtrace_difo_t
));
10315 dtrace_difo_release(dtrace_difo_t
*dp
, dtrace_vstate_t
*vstate
)
10319 ASSERT(MUTEX_HELD(&dtrace_lock
));
10320 ASSERT(dp
->dtdo_refcnt
!= 0);
10322 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
10323 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
10325 if (v
->dtdv_id
!= DIF_VAR_VTIMESTAMP
)
10328 ASSERT(dtrace_vtime_references
> 0);
10329 if (--dtrace_vtime_references
== 0)
10330 dtrace_vtime_disable();
10333 if (--dp
->dtdo_refcnt
== 0)
10334 dtrace_difo_destroy(dp
, vstate
);
10338 * DTrace Format Functions
10341 dtrace_format_add(dtrace_state_t
*state
, char *str
)
10344 uint16_t ndx
, len
= strlen(str
) + 1;
10346 fmt
= kmem_zalloc(len
, KM_SLEEP
);
10347 bcopy(str
, fmt
, len
);
10349 for (ndx
= 0; ndx
< state
->dts_nformats
; ndx
++) {
10350 if (state
->dts_formats
[ndx
] == NULL
) {
10351 state
->dts_formats
[ndx
] = fmt
;
10356 if (state
->dts_nformats
== USHRT_MAX
) {
10358 * This is only likely if a denial-of-service attack is being
10359 * attempted. As such, it's okay to fail silently here.
10361 kmem_free(fmt
, len
);
10366 * For simplicity, we always resize the formats array to be exactly the
10367 * number of formats.
10369 ndx
= state
->dts_nformats
++;
10370 new = kmem_alloc((ndx
+ 1) * sizeof (char *), KM_SLEEP
);
10372 if (state
->dts_formats
!= NULL
) {
10374 bcopy(state
->dts_formats
, new, ndx
* sizeof (char *));
10375 kmem_free(state
->dts_formats
, ndx
* sizeof (char *));
10378 state
->dts_formats
= new;
10379 state
->dts_formats
[ndx
] = fmt
;
10385 dtrace_format_remove(dtrace_state_t
*state
, uint16_t format
)
10389 ASSERT(state
->dts_formats
!= NULL
);
10390 ASSERT(format
<= state
->dts_nformats
);
10391 ASSERT(state
->dts_formats
[format
- 1] != NULL
);
10393 fmt
= state
->dts_formats
[format
- 1];
10394 kmem_free(fmt
, strlen(fmt
) + 1);
10395 state
->dts_formats
[format
- 1] = NULL
;
10399 dtrace_format_destroy(dtrace_state_t
*state
)
10403 if (state
->dts_nformats
== 0) {
10404 ASSERT(state
->dts_formats
== NULL
);
10408 ASSERT(state
->dts_formats
!= NULL
);
10410 for (i
= 0; i
< state
->dts_nformats
; i
++) {
10411 char *fmt
= state
->dts_formats
[i
];
10416 kmem_free(fmt
, strlen(fmt
) + 1);
10419 kmem_free(state
->dts_formats
, state
->dts_nformats
* sizeof (char *));
10420 state
->dts_nformats
= 0;
10421 state
->dts_formats
= NULL
;
10425 * DTrace Predicate Functions
10427 static dtrace_predicate_t
*
10428 dtrace_predicate_create(dtrace_difo_t
*dp
)
10430 dtrace_predicate_t
*pred
;
10432 ASSERT(MUTEX_HELD(&dtrace_lock
));
10433 ASSERT(dp
->dtdo_refcnt
!= 0);
10435 pred
= kmem_zalloc(sizeof (dtrace_predicate_t
), KM_SLEEP
);
10436 pred
->dtp_difo
= dp
;
10437 pred
->dtp_refcnt
= 1;
10439 if (!dtrace_difo_cacheable(dp
))
10442 if (dtrace_predcache_id
== DTRACE_CACHEIDNONE
) {
10444 * This is only theoretically possible -- we have had 2^32
10445 * cacheable predicates on this machine. We cannot allow any
10446 * more predicates to become cacheable: as unlikely as it is,
10447 * there may be a thread caching a (now stale) predicate cache
10448 * ID. (N.B.: the temptation is being successfully resisted to
10449 * have this cmn_err() "Holy shit -- we executed this code!")
10454 pred
->dtp_cacheid
= dtrace_predcache_id
++;
10460 dtrace_predicate_hold(dtrace_predicate_t
*pred
)
10462 ASSERT(MUTEX_HELD(&dtrace_lock
));
10463 ASSERT(pred
->dtp_difo
!= NULL
&& pred
->dtp_difo
->dtdo_refcnt
!= 0);
10464 ASSERT(pred
->dtp_refcnt
> 0);
10466 pred
->dtp_refcnt
++;
10470 dtrace_predicate_release(dtrace_predicate_t
*pred
, dtrace_vstate_t
*vstate
)
10472 dtrace_difo_t
*dp
= pred
->dtp_difo
;
10474 ASSERT(MUTEX_HELD(&dtrace_lock
));
10475 ASSERT(dp
!= NULL
&& dp
->dtdo_refcnt
!= 0);
10476 ASSERT(pred
->dtp_refcnt
> 0);
10478 if (--pred
->dtp_refcnt
== 0) {
10479 dtrace_difo_release(pred
->dtp_difo
, vstate
);
10480 kmem_free(pred
, sizeof (dtrace_predicate_t
));
10485 * DTrace Action Description Functions
10487 static dtrace_actdesc_t
*
10488 dtrace_actdesc_create(dtrace_actkind_t kind
, uint32_t ntuple
,
10489 uint64_t uarg
, uint64_t arg
)
10491 dtrace_actdesc_t
*act
;
10493 ASSERT(!DTRACEACT_ISPRINTFLIKE(kind
) || (arg
!= NULL
&&
10494 arg
>= KERNELBASE
) || (arg
== NULL
&& kind
== DTRACEACT_PRINTA
));
10496 act
= kmem_zalloc(sizeof (dtrace_actdesc_t
), KM_SLEEP
);
10497 act
->dtad_kind
= kind
;
10498 act
->dtad_ntuple
= ntuple
;
10499 act
->dtad_uarg
= uarg
;
10500 act
->dtad_arg
= arg
;
10501 act
->dtad_refcnt
= 1;
10507 dtrace_actdesc_hold(dtrace_actdesc_t
*act
)
10509 ASSERT(act
->dtad_refcnt
>= 1);
10510 act
->dtad_refcnt
++;
10514 dtrace_actdesc_release(dtrace_actdesc_t
*act
, dtrace_vstate_t
*vstate
)
10516 dtrace_actkind_t kind
= act
->dtad_kind
;
10519 ASSERT(act
->dtad_refcnt
>= 1);
10521 if (--act
->dtad_refcnt
!= 0)
10524 if ((dp
= act
->dtad_difo
) != NULL
)
10525 dtrace_difo_release(dp
, vstate
);
10527 if (DTRACEACT_ISPRINTFLIKE(kind
)) {
10528 char *str
= (char *)(uintptr_t)act
->dtad_arg
;
10530 ASSERT((str
!= NULL
&& (uintptr_t)str
>= KERNELBASE
) ||
10531 (str
== NULL
&& act
->dtad_kind
== DTRACEACT_PRINTA
));
10534 kmem_free(str
, strlen(str
) + 1);
10537 kmem_free(act
, sizeof (dtrace_actdesc_t
));
10541 * DTrace ECB Functions
10543 static dtrace_ecb_t
*
10544 dtrace_ecb_add(dtrace_state_t
*state
, dtrace_probe_t
*probe
)
10547 dtrace_epid_t epid
;
10549 ASSERT(MUTEX_HELD(&dtrace_lock
));
10551 ecb
= kmem_zalloc(sizeof (dtrace_ecb_t
), KM_SLEEP
);
10552 ecb
->dte_predicate
= NULL
;
10553 ecb
->dte_probe
= probe
;
10556 * The default size is the size of the default action: recording
10559 ecb
->dte_size
= ecb
->dte_needed
= sizeof (dtrace_rechdr_t
);
10560 ecb
->dte_alignment
= sizeof (dtrace_epid_t
);
10562 epid
= state
->dts_epid
++;
10564 if (epid
- 1 >= state
->dts_necbs
) {
10565 dtrace_ecb_t
**oecbs
= state
->dts_ecbs
, **ecbs
;
10566 int necbs
= state
->dts_necbs
<< 1;
10568 ASSERT(epid
== state
->dts_necbs
+ 1);
10571 ASSERT(oecbs
== NULL
);
10575 ecbs
= kmem_zalloc(necbs
* sizeof (*ecbs
), KM_SLEEP
);
10578 bcopy(oecbs
, ecbs
, state
->dts_necbs
* sizeof (*ecbs
));
10580 dtrace_membar_producer();
10581 state
->dts_ecbs
= ecbs
;
10583 if (oecbs
!= NULL
) {
10585 * If this state is active, we must dtrace_sync()
10586 * before we can free the old dts_ecbs array: we're
10587 * coming in hot, and there may be active ring
10588 * buffer processing (which indexes into the dts_ecbs
10589 * array) on another CPU.
10591 if (state
->dts_activity
!= DTRACE_ACTIVITY_INACTIVE
)
10594 kmem_free(oecbs
, state
->dts_necbs
* sizeof (*ecbs
));
10597 dtrace_membar_producer();
10598 state
->dts_necbs
= necbs
;
10601 ecb
->dte_state
= state
;
10603 ASSERT(state
->dts_ecbs
[epid
- 1] == NULL
);
10604 dtrace_membar_producer();
10605 state
->dts_ecbs
[(ecb
->dte_epid
= epid
) - 1] = ecb
;
10611 dtrace_ecb_enable(dtrace_ecb_t
*ecb
)
10613 dtrace_probe_t
*probe
= ecb
->dte_probe
;
10615 ASSERT(MUTEX_HELD(&cpu_lock
));
10616 ASSERT(MUTEX_HELD(&dtrace_lock
));
10617 ASSERT(ecb
->dte_next
== NULL
);
10619 if (probe
== NULL
) {
10621 * This is the NULL probe -- there's nothing to do.
10626 if (probe
->dtpr_ecb
== NULL
) {
10627 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
10630 * We're the first ECB on this probe.
10632 probe
->dtpr_ecb
= probe
->dtpr_ecb_last
= ecb
;
10634 if (ecb
->dte_predicate
!= NULL
)
10635 probe
->dtpr_predcache
= ecb
->dte_predicate
->dtp_cacheid
;
10637 return (prov
->dtpv_pops
.dtps_enable(prov
->dtpv_arg
,
10638 probe
->dtpr_id
, probe
->dtpr_arg
));
10641 * This probe is already active. Swing the last pointer to
10642 * point to the new ECB, and issue a dtrace_sync() to assure
10643 * that all CPUs have seen the change.
10645 ASSERT(probe
->dtpr_ecb_last
!= NULL
);
10646 probe
->dtpr_ecb_last
->dte_next
= ecb
;
10647 probe
->dtpr_ecb_last
= ecb
;
10648 probe
->dtpr_predcache
= 0;
10656 dtrace_ecb_resize(dtrace_ecb_t
*ecb
)
10658 dtrace_action_t
*act
;
10659 uint32_t curneeded
= UINT32_MAX
;
10660 uint32_t aggbase
= UINT32_MAX
;
10663 * If we record anything, we always record the dtrace_rechdr_t. (And
10664 * we always record it first.)
10666 ecb
->dte_size
= sizeof (dtrace_rechdr_t
);
10667 ecb
->dte_alignment
= sizeof (dtrace_epid_t
);
10669 for (act
= ecb
->dte_action
; act
!= NULL
; act
= act
->dta_next
) {
10670 dtrace_recdesc_t
*rec
= &act
->dta_rec
;
10671 ASSERT(rec
->dtrd_size
> 0 || rec
->dtrd_alignment
== 1);
10673 ecb
->dte_alignment
= MAX(ecb
->dte_alignment
,
10674 rec
->dtrd_alignment
);
10676 if (DTRACEACT_ISAGG(act
->dta_kind
)) {
10677 dtrace_aggregation_t
*agg
= (dtrace_aggregation_t
*)act
;
10679 ASSERT(rec
->dtrd_size
!= 0);
10680 ASSERT(agg
->dtag_first
!= NULL
);
10681 ASSERT(act
->dta_prev
->dta_intuple
);
10682 ASSERT(aggbase
!= UINT32_MAX
);
10683 ASSERT(curneeded
!= UINT32_MAX
);
10685 agg
->dtag_base
= aggbase
;
10687 curneeded
= P2ROUNDUP(curneeded
, rec
->dtrd_alignment
);
10688 rec
->dtrd_offset
= curneeded
;
10689 if (curneeded
+ rec
->dtrd_size
< curneeded
)
10691 curneeded
+= rec
->dtrd_size
;
10692 ecb
->dte_needed
= MAX(ecb
->dte_needed
, curneeded
);
10694 aggbase
= UINT32_MAX
;
10695 curneeded
= UINT32_MAX
;
10696 } else if (act
->dta_intuple
) {
10697 if (curneeded
== UINT32_MAX
) {
10699 * This is the first record in a tuple. Align
10700 * curneeded to be at offset 4 in an 8-byte
10703 ASSERT(act
->dta_prev
== NULL
||
10704 !act
->dta_prev
->dta_intuple
);
10705 ASSERT3U(aggbase
, ==, UINT32_MAX
);
10706 curneeded
= P2PHASEUP(ecb
->dte_size
,
10707 sizeof (uint64_t), sizeof (dtrace_aggid_t
));
10709 aggbase
= curneeded
- sizeof (dtrace_aggid_t
);
10710 ASSERT(IS_P2ALIGNED(aggbase
,
10711 sizeof (uint64_t)));
10713 curneeded
= P2ROUNDUP(curneeded
, rec
->dtrd_alignment
);
10714 rec
->dtrd_offset
= curneeded
;
10715 if (curneeded
+ rec
->dtrd_size
< curneeded
)
10717 curneeded
+= rec
->dtrd_size
;
10719 /* tuples must be followed by an aggregation */
10720 ASSERT(act
->dta_prev
== NULL
||
10721 !act
->dta_prev
->dta_intuple
);
10723 ecb
->dte_size
= P2ROUNDUP(ecb
->dte_size
,
10724 rec
->dtrd_alignment
);
10725 rec
->dtrd_offset
= ecb
->dte_size
;
10726 if (ecb
->dte_size
+ rec
->dtrd_size
< ecb
->dte_size
)
10728 ecb
->dte_size
+= rec
->dtrd_size
;
10729 ecb
->dte_needed
= MAX(ecb
->dte_needed
, ecb
->dte_size
);
10733 if ((act
= ecb
->dte_action
) != NULL
&&
10734 !(act
->dta_kind
== DTRACEACT_SPECULATE
&& act
->dta_next
== NULL
) &&
10735 ecb
->dte_size
== sizeof (dtrace_rechdr_t
)) {
10737 * If the size is still sizeof (dtrace_rechdr_t), then all
10738 * actions store no data; set the size to 0.
10743 ecb
->dte_size
= P2ROUNDUP(ecb
->dte_size
, sizeof (dtrace_epid_t
));
10744 ecb
->dte_needed
= P2ROUNDUP(ecb
->dte_needed
, (sizeof (dtrace_epid_t
)));
10745 ecb
->dte_state
->dts_needed
= MAX(ecb
->dte_state
->dts_needed
,
10750 static dtrace_action_t
*
10751 dtrace_ecb_aggregation_create(dtrace_ecb_t
*ecb
, dtrace_actdesc_t
*desc
)
10753 dtrace_aggregation_t
*agg
;
10754 size_t size
= sizeof (uint64_t);
10755 int ntuple
= desc
->dtad_ntuple
;
10756 dtrace_action_t
*act
;
10757 dtrace_recdesc_t
*frec
;
10758 dtrace_aggid_t aggid
;
10759 dtrace_state_t
*state
= ecb
->dte_state
;
10761 agg
= kmem_zalloc(sizeof (dtrace_aggregation_t
), KM_SLEEP
);
10762 agg
->dtag_ecb
= ecb
;
10764 ASSERT(DTRACEACT_ISAGG(desc
->dtad_kind
));
10766 switch (desc
->dtad_kind
) {
10767 case DTRACEAGG_MIN
:
10768 agg
->dtag_initial
= INT64_MAX
;
10769 agg
->dtag_aggregate
= dtrace_aggregate_min
;
10772 case DTRACEAGG_MAX
:
10773 agg
->dtag_initial
= INT64_MIN
;
10774 agg
->dtag_aggregate
= dtrace_aggregate_max
;
10777 case DTRACEAGG_COUNT
:
10778 agg
->dtag_aggregate
= dtrace_aggregate_count
;
10781 case DTRACEAGG_QUANTIZE
:
10782 agg
->dtag_aggregate
= dtrace_aggregate_quantize
;
10783 size
= (((sizeof (uint64_t) * NBBY
) - 1) * 2 + 1) *
10787 case DTRACEAGG_LQUANTIZE
: {
10788 uint16_t step
= DTRACE_LQUANTIZE_STEP(desc
->dtad_arg
);
10789 uint16_t levels
= DTRACE_LQUANTIZE_LEVELS(desc
->dtad_arg
);
10791 agg
->dtag_initial
= desc
->dtad_arg
;
10792 agg
->dtag_aggregate
= dtrace_aggregate_lquantize
;
10794 if (step
== 0 || levels
== 0)
10797 size
= levels
* sizeof (uint64_t) + 3 * sizeof (uint64_t);
10801 case DTRACEAGG_LLQUANTIZE
: {
10802 uint16_t factor
= DTRACE_LLQUANTIZE_FACTOR(desc
->dtad_arg
);
10803 uint16_t low
= DTRACE_LLQUANTIZE_LOW(desc
->dtad_arg
);
10804 uint16_t high
= DTRACE_LLQUANTIZE_HIGH(desc
->dtad_arg
);
10805 uint16_t nsteps
= DTRACE_LLQUANTIZE_NSTEP(desc
->dtad_arg
);
10808 agg
->dtag_initial
= desc
->dtad_arg
;
10809 agg
->dtag_aggregate
= dtrace_aggregate_llquantize
;
10811 if (factor
< 2 || low
>= high
|| nsteps
< factor
)
10815 * Now check that the number of steps evenly divides a power
10816 * of the factor. (This assures both integer bucket size and
10817 * linearity within each magnitude.)
10819 for (v
= factor
; v
< nsteps
; v
*= factor
)
10822 if ((v
% nsteps
) || (nsteps
% factor
))
10825 size
= (dtrace_aggregate_llquantize_bucket(factor
,
10826 low
, high
, nsteps
, INT64_MAX
) + 2) * sizeof (uint64_t);
10830 case DTRACEAGG_AVG
:
10831 agg
->dtag_aggregate
= dtrace_aggregate_avg
;
10832 size
= sizeof (uint64_t) * 2;
10835 case DTRACEAGG_STDDEV
:
10836 agg
->dtag_aggregate
= dtrace_aggregate_stddev
;
10837 size
= sizeof (uint64_t) * 4;
10840 case DTRACEAGG_SUM
:
10841 agg
->dtag_aggregate
= dtrace_aggregate_sum
;
10848 agg
->dtag_action
.dta_rec
.dtrd_size
= size
;
10854 * We must make sure that we have enough actions for the n-tuple.
10856 for (act
= ecb
->dte_action_last
; act
!= NULL
; act
= act
->dta_prev
) {
10857 if (DTRACEACT_ISAGG(act
->dta_kind
))
10860 if (--ntuple
== 0) {
10862 * This is the action with which our n-tuple begins.
10864 agg
->dtag_first
= act
;
10870 * This n-tuple is short by ntuple elements. Return failure.
10872 ASSERT(ntuple
!= 0);
10874 kmem_free(agg
, sizeof (dtrace_aggregation_t
));
10879 * If the last action in the tuple has a size of zero, it's actually
10880 * an expression argument for the aggregating action.
10882 ASSERT(ecb
->dte_action_last
!= NULL
);
10883 act
= ecb
->dte_action_last
;
10885 if (act
->dta_kind
== DTRACEACT_DIFEXPR
) {
10886 ASSERT(act
->dta_difo
!= NULL
);
10888 if (act
->dta_difo
->dtdo_rtype
.dtdt_size
== 0)
10889 agg
->dtag_hasarg
= 1;
10893 * We need to allocate an id for this aggregation.
10895 aggid
= (dtrace_aggid_t
)(uintptr_t)vmem_alloc(state
->dts_aggid_arena
, 1,
10896 VM_BESTFIT
| VM_SLEEP
);
10898 if (aggid
- 1 >= state
->dts_naggregations
) {
10899 dtrace_aggregation_t
**oaggs
= state
->dts_aggregations
;
10900 dtrace_aggregation_t
**aggs
;
10901 int naggs
= state
->dts_naggregations
<< 1;
10902 int onaggs
= state
->dts_naggregations
;
10904 ASSERT(aggid
== state
->dts_naggregations
+ 1);
10907 ASSERT(oaggs
== NULL
);
10911 aggs
= kmem_zalloc(naggs
* sizeof (*aggs
), KM_SLEEP
);
10913 if (oaggs
!= NULL
) {
10914 bcopy(oaggs
, aggs
, onaggs
* sizeof (*aggs
));
10915 kmem_free(oaggs
, onaggs
* sizeof (*aggs
));
10918 state
->dts_aggregations
= aggs
;
10919 state
->dts_naggregations
= naggs
;
10922 ASSERT(state
->dts_aggregations
[aggid
- 1] == NULL
);
10923 state
->dts_aggregations
[(agg
->dtag_id
= aggid
) - 1] = agg
;
10925 frec
= &agg
->dtag_first
->dta_rec
;
10926 if (frec
->dtrd_alignment
< sizeof (dtrace_aggid_t
))
10927 frec
->dtrd_alignment
= sizeof (dtrace_aggid_t
);
10929 for (act
= agg
->dtag_first
; act
!= NULL
; act
= act
->dta_next
) {
10930 ASSERT(!act
->dta_intuple
);
10931 act
->dta_intuple
= 1;
10934 return (&agg
->dtag_action
);
10938 dtrace_ecb_aggregation_destroy(dtrace_ecb_t
*ecb
, dtrace_action_t
*act
)
10940 dtrace_aggregation_t
*agg
= (dtrace_aggregation_t
*)act
;
10941 dtrace_state_t
*state
= ecb
->dte_state
;
10942 dtrace_aggid_t aggid
= agg
->dtag_id
;
10944 ASSERT(DTRACEACT_ISAGG(act
->dta_kind
));
10945 vmem_free(state
->dts_aggid_arena
, (void *)(uintptr_t)aggid
, 1);
10947 ASSERT(state
->dts_aggregations
[aggid
- 1] == agg
);
10948 state
->dts_aggregations
[aggid
- 1] = NULL
;
10950 kmem_free(agg
, sizeof (dtrace_aggregation_t
));
10954 dtrace_ecb_action_add(dtrace_ecb_t
*ecb
, dtrace_actdesc_t
*desc
)
10956 dtrace_action_t
*action
, *last
;
10957 dtrace_difo_t
*dp
= desc
->dtad_difo
;
10958 uint32_t size
= 0, align
= sizeof (uint8_t), mask
;
10959 uint16_t format
= 0;
10960 dtrace_recdesc_t
*rec
;
10961 dtrace_state_t
*state
= ecb
->dte_state
;
10962 dtrace_optval_t
*opt
= state
->dts_options
, nframes
, strsize
;
10963 uint64_t arg
= desc
->dtad_arg
;
10965 ASSERT(MUTEX_HELD(&dtrace_lock
));
10966 ASSERT(ecb
->dte_action
== NULL
|| ecb
->dte_action
->dta_refcnt
== 1);
10968 if (DTRACEACT_ISAGG(desc
->dtad_kind
)) {
10970 * If this is an aggregating action, there must be neither
10971 * a speculate nor a commit on the action chain.
10973 dtrace_action_t
*act
;
10975 for (act
= ecb
->dte_action
; act
!= NULL
; act
= act
->dta_next
) {
10976 if (act
->dta_kind
== DTRACEACT_COMMIT
)
10979 if (act
->dta_kind
== DTRACEACT_SPECULATE
)
10983 action
= dtrace_ecb_aggregation_create(ecb
, desc
);
10985 if (action
== NULL
)
10988 if (DTRACEACT_ISDESTRUCTIVE(desc
->dtad_kind
) ||
10989 (desc
->dtad_kind
== DTRACEACT_DIFEXPR
&&
10990 dp
!= NULL
&& dp
->dtdo_destructive
)) {
10991 state
->dts_destructive
= 1;
10994 switch (desc
->dtad_kind
) {
10995 case DTRACEACT_PRINTF
:
10996 case DTRACEACT_PRINTA
:
10997 case DTRACEACT_SYSTEM
:
10998 case DTRACEACT_FREOPEN
:
10999 case DTRACEACT_DIFEXPR
:
11001 * We know that our arg is a string -- turn it into a
11005 ASSERT(desc
->dtad_kind
== DTRACEACT_PRINTA
||
11006 desc
->dtad_kind
== DTRACEACT_DIFEXPR
);
11009 ASSERT(arg
!= NULL
);
11010 ASSERT(arg
> KERNELBASE
);
11011 format
= dtrace_format_add(state
,
11012 (char *)(uintptr_t)arg
);
11016 case DTRACEACT_LIBACT
:
11017 case DTRACEACT_TRACEMEM
:
11018 case DTRACEACT_TRACEMEM_DYNSIZE
:
11022 if ((size
= dp
->dtdo_rtype
.dtdt_size
) != 0)
11025 if (dp
->dtdo_rtype
.dtdt_kind
== DIF_TYPE_STRING
) {
11026 if (!(dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
))
11029 size
= opt
[DTRACEOPT_STRSIZE
];
11034 case DTRACEACT_STACK
:
11035 if ((nframes
= arg
) == 0) {
11036 nframes
= opt
[DTRACEOPT_STACKFRAMES
];
11037 ASSERT(nframes
> 0);
11041 size
= nframes
* sizeof (pc_t
);
11044 case DTRACEACT_JSTACK
:
11045 if ((strsize
= DTRACE_USTACK_STRSIZE(arg
)) == 0)
11046 strsize
= opt
[DTRACEOPT_JSTACKSTRSIZE
];
11048 if ((nframes
= DTRACE_USTACK_NFRAMES(arg
)) == 0)
11049 nframes
= opt
[DTRACEOPT_JSTACKFRAMES
];
11051 arg
= DTRACE_USTACK_ARG(nframes
, strsize
);
11054 case DTRACEACT_USTACK
:
11055 if (desc
->dtad_kind
!= DTRACEACT_JSTACK
&&
11056 (nframes
= DTRACE_USTACK_NFRAMES(arg
)) == 0) {
11057 strsize
= DTRACE_USTACK_STRSIZE(arg
);
11058 nframes
= opt
[DTRACEOPT_USTACKFRAMES
];
11059 ASSERT(nframes
> 0);
11060 arg
= DTRACE_USTACK_ARG(nframes
, strsize
);
11064 * Save a slot for the pid.
11066 size
= (nframes
+ 1) * sizeof (uint64_t);
11067 size
+= DTRACE_USTACK_STRSIZE(arg
);
11068 size
= P2ROUNDUP(size
, (uint32_t)(sizeof (uintptr_t)));
11072 case DTRACEACT_SYM
:
11073 case DTRACEACT_MOD
:
11074 if (dp
== NULL
|| ((size
= dp
->dtdo_rtype
.dtdt_size
) !=
11075 sizeof (uint64_t)) ||
11076 (dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
))
11080 case DTRACEACT_USYM
:
11081 case DTRACEACT_UMOD
:
11082 case DTRACEACT_UADDR
:
11084 (dp
->dtdo_rtype
.dtdt_size
!= sizeof (uint64_t)) ||
11085 (dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
))
11089 * We have a slot for the pid, plus a slot for the
11090 * argument. To keep things simple (aligned with
11091 * bitness-neutral sizing), we store each as a 64-bit
11094 size
= 2 * sizeof (uint64_t);
11097 case DTRACEACT_STOP
:
11098 case DTRACEACT_BREAKPOINT
:
11099 case DTRACEACT_PANIC
:
11102 case DTRACEACT_CHILL
:
11103 case DTRACEACT_DISCARD
:
11104 case DTRACEACT_RAISE
:
11109 case DTRACEACT_EXIT
:
11111 (size
= dp
->dtdo_rtype
.dtdt_size
) != sizeof (int) ||
11112 (dp
->dtdo_rtype
.dtdt_flags
& DIF_TF_BYREF
))
11116 case DTRACEACT_SPECULATE
:
11117 if (ecb
->dte_size
> sizeof (dtrace_rechdr_t
))
11123 state
->dts_speculates
= 1;
11126 case DTRACEACT_COMMIT
: {
11127 dtrace_action_t
*act
= ecb
->dte_action
;
11129 for (; act
!= NULL
; act
= act
->dta_next
) {
11130 if (act
->dta_kind
== DTRACEACT_COMMIT
)
11143 if (size
!= 0 || desc
->dtad_kind
== DTRACEACT_SPECULATE
) {
11145 * If this is a data-storing action or a speculate,
11146 * we must be sure that there isn't a commit on the
11149 dtrace_action_t
*act
= ecb
->dte_action
;
11151 for (; act
!= NULL
; act
= act
->dta_next
) {
11152 if (act
->dta_kind
== DTRACEACT_COMMIT
)
11157 action
= kmem_zalloc(sizeof (dtrace_action_t
), KM_SLEEP
);
11158 action
->dta_rec
.dtrd_size
= size
;
11161 action
->dta_refcnt
= 1;
11162 rec
= &action
->dta_rec
;
11163 size
= rec
->dtrd_size
;
11165 for (mask
= sizeof (uint64_t) - 1; size
!= 0 && mask
> 0; mask
>>= 1) {
11166 if (!(size
& mask
)) {
11172 action
->dta_kind
= desc
->dtad_kind
;
11174 if ((action
->dta_difo
= dp
) != NULL
)
11175 dtrace_difo_hold(dp
);
11177 rec
->dtrd_action
= action
->dta_kind
;
11178 rec
->dtrd_arg
= arg
;
11179 rec
->dtrd_uarg
= desc
->dtad_uarg
;
11180 rec
->dtrd_alignment
= (uint16_t)align
;
11181 rec
->dtrd_format
= format
;
11183 if ((last
= ecb
->dte_action_last
) != NULL
) {
11184 ASSERT(ecb
->dte_action
!= NULL
);
11185 action
->dta_prev
= last
;
11186 last
->dta_next
= action
;
11188 ASSERT(ecb
->dte_action
== NULL
);
11189 ecb
->dte_action
= action
;
11192 ecb
->dte_action_last
= action
;
11198 dtrace_ecb_action_remove(dtrace_ecb_t
*ecb
)
11200 dtrace_action_t
*act
= ecb
->dte_action
, *next
;
11201 dtrace_vstate_t
*vstate
= &ecb
->dte_state
->dts_vstate
;
11205 if (act
!= NULL
&& act
->dta_refcnt
> 1) {
11206 ASSERT(act
->dta_next
== NULL
|| act
->dta_next
->dta_refcnt
== 1);
11209 for (; act
!= NULL
; act
= next
) {
11210 next
= act
->dta_next
;
11211 ASSERT(next
!= NULL
|| act
== ecb
->dte_action_last
);
11212 ASSERT(act
->dta_refcnt
== 1);
11214 if ((format
= act
->dta_rec
.dtrd_format
) != 0)
11215 dtrace_format_remove(ecb
->dte_state
, format
);
11217 if ((dp
= act
->dta_difo
) != NULL
)
11218 dtrace_difo_release(dp
, vstate
);
11220 if (DTRACEACT_ISAGG(act
->dta_kind
)) {
11221 dtrace_ecb_aggregation_destroy(ecb
, act
);
11223 kmem_free(act
, sizeof (dtrace_action_t
));
11228 ecb
->dte_action
= NULL
;
11229 ecb
->dte_action_last
= NULL
;
11234 dtrace_ecb_disable(dtrace_ecb_t
*ecb
)
11237 * We disable the ECB by removing it from its probe.
11239 dtrace_ecb_t
*pecb
, *prev
= NULL
;
11240 dtrace_probe_t
*probe
= ecb
->dte_probe
;
11242 ASSERT(MUTEX_HELD(&dtrace_lock
));
11244 if (probe
== NULL
) {
11246 * This is the NULL probe; there is nothing to disable.
11251 for (pecb
= probe
->dtpr_ecb
; pecb
!= NULL
; pecb
= pecb
->dte_next
) {
11257 ASSERT(pecb
!= NULL
);
11259 if (prev
== NULL
) {
11260 probe
->dtpr_ecb
= ecb
->dte_next
;
11262 prev
->dte_next
= ecb
->dte_next
;
11265 if (ecb
== probe
->dtpr_ecb_last
) {
11266 ASSERT(ecb
->dte_next
== NULL
);
11267 probe
->dtpr_ecb_last
= prev
;
11271 * The ECB has been disconnected from the probe; now sync to assure
11272 * that all CPUs have seen the change before returning.
11276 if (probe
->dtpr_ecb
== NULL
) {
11278 * That was the last ECB on the probe; clear the predicate
11279 * cache ID for the probe, disable it and sync one more time
11280 * to assure that we'll never hit it again.
11282 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
11284 ASSERT(ecb
->dte_next
== NULL
);
11285 ASSERT(probe
->dtpr_ecb_last
== NULL
);
11286 probe
->dtpr_predcache
= DTRACE_CACHEIDNONE
;
11287 prov
->dtpv_pops
.dtps_disable(prov
->dtpv_arg
,
11288 probe
->dtpr_id
, probe
->dtpr_arg
);
11292 * There is at least one ECB remaining on the probe. If there
11293 * is _exactly_ one, set the probe's predicate cache ID to be
11294 * the predicate cache ID of the remaining ECB.
11296 ASSERT(probe
->dtpr_ecb_last
!= NULL
);
11297 ASSERT(probe
->dtpr_predcache
== DTRACE_CACHEIDNONE
);
11299 if (probe
->dtpr_ecb
== probe
->dtpr_ecb_last
) {
11300 dtrace_predicate_t
*p
= probe
->dtpr_ecb
->dte_predicate
;
11302 ASSERT(probe
->dtpr_ecb
->dte_next
== NULL
);
11305 probe
->dtpr_predcache
= p
->dtp_cacheid
;
11308 ecb
->dte_next
= NULL
;
11313 dtrace_ecb_destroy(dtrace_ecb_t
*ecb
)
11315 dtrace_state_t
*state
= ecb
->dte_state
;
11316 dtrace_vstate_t
*vstate
= &state
->dts_vstate
;
11317 dtrace_predicate_t
*pred
;
11318 dtrace_epid_t epid
= ecb
->dte_epid
;
11320 ASSERT(MUTEX_HELD(&dtrace_lock
));
11321 ASSERT(ecb
->dte_next
== NULL
);
11322 ASSERT(ecb
->dte_probe
== NULL
|| ecb
->dte_probe
->dtpr_ecb
!= ecb
);
11324 if ((pred
= ecb
->dte_predicate
) != NULL
)
11325 dtrace_predicate_release(pred
, vstate
);
11327 dtrace_ecb_action_remove(ecb
);
11329 ASSERT(state
->dts_ecbs
[epid
- 1] == ecb
);
11330 state
->dts_ecbs
[epid
- 1] = NULL
;
11332 kmem_free(ecb
, sizeof (dtrace_ecb_t
));
11335 static dtrace_ecb_t
*
11336 dtrace_ecb_create(dtrace_state_t
*state
, dtrace_probe_t
*probe
,
11337 dtrace_enabling_t
*enab
)
11340 dtrace_predicate_t
*pred
;
11341 dtrace_actdesc_t
*act
;
11342 dtrace_provider_t
*prov
;
11343 dtrace_ecbdesc_t
*desc
= enab
->dten_current
;
11345 ASSERT(MUTEX_HELD(&dtrace_lock
));
11346 ASSERT(state
!= NULL
);
11348 ecb
= dtrace_ecb_add(state
, probe
);
11349 ecb
->dte_uarg
= desc
->dted_uarg
;
11351 if ((pred
= desc
->dted_pred
.dtpdd_predicate
) != NULL
) {
11352 dtrace_predicate_hold(pred
);
11353 ecb
->dte_predicate
= pred
;
11356 if (probe
!= NULL
) {
11358 * If the provider shows more leg than the consumer is old
11359 * enough to see, we need to enable the appropriate implicit
11360 * predicate bits to prevent the ecb from activating at
11363 * Providers specifying DTRACE_PRIV_USER at register time
11364 * are stating that they need the /proc-style privilege
11365 * model to be enforced, and this is what DTRACE_COND_OWNER
11366 * and DTRACE_COND_ZONEOWNER will then do at probe time.
11368 prov
= probe
->dtpr_provider
;
11369 if (!(state
->dts_cred
.dcr_visible
& DTRACE_CRV_ALLPROC
) &&
11370 (prov
->dtpv_priv
.dtpp_flags
& DTRACE_PRIV_USER
))
11371 ecb
->dte_cond
|= DTRACE_COND_OWNER
;
11373 if (!(state
->dts_cred
.dcr_visible
& DTRACE_CRV_ALLZONE
) &&
11374 (prov
->dtpv_priv
.dtpp_flags
& DTRACE_PRIV_USER
))
11375 ecb
->dte_cond
|= DTRACE_COND_ZONEOWNER
;
11378 * If the provider shows us kernel innards and the user
11379 * is lacking sufficient privilege, enable the
11380 * DTRACE_COND_USERMODE implicit predicate.
11382 if (!(state
->dts_cred
.dcr_visible
& DTRACE_CRV_KERNEL
) &&
11383 (prov
->dtpv_priv
.dtpp_flags
& DTRACE_PRIV_KERNEL
))
11384 ecb
->dte_cond
|= DTRACE_COND_USERMODE
;
11387 if (dtrace_ecb_create_cache
!= NULL
) {
11389 * If we have a cached ecb, we'll use its action list instead
11390 * of creating our own (saving both time and space).
11392 dtrace_ecb_t
*cached
= dtrace_ecb_create_cache
;
11393 dtrace_action_t
*act
= cached
->dte_action
;
11396 ASSERT(act
->dta_refcnt
> 0);
11398 ecb
->dte_action
= act
;
11399 ecb
->dte_action_last
= cached
->dte_action_last
;
11400 ecb
->dte_needed
= cached
->dte_needed
;
11401 ecb
->dte_size
= cached
->dte_size
;
11402 ecb
->dte_alignment
= cached
->dte_alignment
;
11408 for (act
= desc
->dted_action
; act
!= NULL
; act
= act
->dtad_next
) {
11409 if ((enab
->dten_error
= dtrace_ecb_action_add(ecb
, act
)) != 0) {
11410 dtrace_ecb_destroy(ecb
);
11415 if ((enab
->dten_error
= dtrace_ecb_resize(ecb
)) != 0) {
11416 dtrace_ecb_destroy(ecb
);
11420 return (dtrace_ecb_create_cache
= ecb
);
11424 dtrace_ecb_create_enable(dtrace_probe_t
*probe
, void *arg
)
11427 dtrace_enabling_t
*enab
= arg
;
11428 dtrace_state_t
*state
= enab
->dten_vstate
->dtvs_state
;
11430 ASSERT(state
!= NULL
);
11432 if (probe
!= NULL
&& probe
->dtpr_gen
< enab
->dten_probegen
) {
11434 * This probe was created in a generation for which this
11435 * enabling has previously created ECBs; we don't want to
11436 * enable it again, so just kick out.
11438 return (DTRACE_MATCH_NEXT
);
11441 if ((ecb
= dtrace_ecb_create(state
, probe
, enab
)) == NULL
)
11442 return (DTRACE_MATCH_DONE
);
11444 if (dtrace_ecb_enable(ecb
) < 0)
11445 return (DTRACE_MATCH_FAIL
);
11447 return (DTRACE_MATCH_NEXT
);
11450 static dtrace_ecb_t
*
11451 dtrace_epid2ecb(dtrace_state_t
*state
, dtrace_epid_t id
)
11455 ASSERT(MUTEX_HELD(&dtrace_lock
));
11457 if (id
== 0 || id
> state
->dts_necbs
)
11460 ASSERT(state
->dts_necbs
> 0 && state
->dts_ecbs
!= NULL
);
11461 ASSERT((ecb
= state
->dts_ecbs
[id
- 1]) == NULL
|| ecb
->dte_epid
== id
);
11463 return (state
->dts_ecbs
[id
- 1]);
11466 static dtrace_aggregation_t
*
11467 dtrace_aggid2agg(dtrace_state_t
*state
, dtrace_aggid_t id
)
11469 dtrace_aggregation_t
*agg
;
11471 ASSERT(MUTEX_HELD(&dtrace_lock
));
11473 if (id
== 0 || id
> state
->dts_naggregations
)
11476 ASSERT(state
->dts_naggregations
> 0 && state
->dts_aggregations
!= NULL
);
11477 ASSERT((agg
= state
->dts_aggregations
[id
- 1]) == NULL
||
11478 agg
->dtag_id
== id
);
11480 return (state
->dts_aggregations
[id
- 1]);
11484 * DTrace Buffer Functions
11486 * The following functions manipulate DTrace buffers. Most of these functions
11487 * are called in the context of establishing or processing consumer state;
11488 * exceptions are explicitly noted.
11492 * Note: called from cross call context. This function switches the two
11493 * buffers on a given CPU. The atomicity of this operation is assured by
11494 * disabling interrupts while the actual switch takes place; the disabling of
11495 * interrupts serializes the execution with any execution of dtrace_probe() on
11499 dtrace_buffer_switch(dtrace_buffer_t
*buf
)
11501 caddr_t tomax
= buf
->dtb_tomax
;
11502 caddr_t xamot
= buf
->dtb_xamot
;
11503 dtrace_icookie_t cookie
;
11506 ASSERT(!(buf
->dtb_flags
& DTRACEBUF_NOSWITCH
));
11507 ASSERT(!(buf
->dtb_flags
& DTRACEBUF_RING
));
11509 cookie
= dtrace_interrupt_disable();
11510 now
= dtrace_gethrtime();
11511 buf
->dtb_tomax
= xamot
;
11512 buf
->dtb_xamot
= tomax
;
11513 buf
->dtb_xamot_drops
= buf
->dtb_drops
;
11514 buf
->dtb_xamot_offset
= buf
->dtb_offset
;
11515 buf
->dtb_xamot_errors
= buf
->dtb_errors
;
11516 buf
->dtb_xamot_flags
= buf
->dtb_flags
;
11517 buf
->dtb_offset
= 0;
11518 buf
->dtb_drops
= 0;
11519 buf
->dtb_errors
= 0;
11520 buf
->dtb_flags
&= ~(DTRACEBUF_ERROR
| DTRACEBUF_DROPPED
);
11521 buf
->dtb_interval
= now
- buf
->dtb_switched
;
11522 buf
->dtb_switched
= now
;
11523 dtrace_interrupt_enable(cookie
);
11527 * Note: called from cross call context. This function activates a buffer
11528 * on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation
11529 * is guaranteed by the disabling of interrupts.
11532 dtrace_buffer_activate(dtrace_state_t
*state
)
11534 dtrace_buffer_t
*buf
;
11535 dtrace_icookie_t cookie
= dtrace_interrupt_disable();
11537 buf
= &state
->dts_buffer
[CPU
->cpu_id
];
11539 if (buf
->dtb_tomax
!= NULL
) {
11541 * We might like to assert that the buffer is marked inactive,
11542 * but this isn't necessarily true: the buffer for the CPU
11543 * that processes the BEGIN probe has its buffer activated
11544 * manually. In this case, we take the (harmless) action
11545 * re-clearing the bit INACTIVE bit.
11547 buf
->dtb_flags
&= ~DTRACEBUF_INACTIVE
;
11550 dtrace_interrupt_enable(cookie
);
11554 dtrace_buffer_alloc(dtrace_buffer_t
*bufs
, size_t size
, int flags
,
11555 processorid_t cpu
, int *factor
)
11558 dtrace_buffer_t
*buf
;
11559 int allocated
= 0, desired
= 0;
11561 ASSERT(MUTEX_HELD(&cpu_lock
));
11562 ASSERT(MUTEX_HELD(&dtrace_lock
));
11566 if (size
> dtrace_nonroot_maxsize
&&
11567 !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL
, B_FALSE
))
11573 if (cpu
!= DTRACE_CPUALL
&& cpu
!= cp
->cpu_id
)
11576 buf
= &bufs
[cp
->cpu_id
];
11579 * If there is already a buffer allocated for this CPU, it
11580 * is only possible that this is a DR event. In this case,
11581 * the buffer size must match our specified size.
11583 if (buf
->dtb_tomax
!= NULL
) {
11584 ASSERT(buf
->dtb_size
== size
);
11588 ASSERT(buf
->dtb_xamot
== NULL
);
11590 if ((buf
->dtb_tomax
= kmem_zalloc(size
,
11591 KM_NOSLEEP
| KM_NORMALPRI
)) == NULL
)
11594 buf
->dtb_size
= size
;
11595 buf
->dtb_flags
= flags
;
11596 buf
->dtb_offset
= 0;
11597 buf
->dtb_drops
= 0;
11599 if (flags
& DTRACEBUF_NOSWITCH
)
11602 if ((buf
->dtb_xamot
= kmem_zalloc(size
,
11603 KM_NOSLEEP
| KM_NORMALPRI
)) == NULL
)
11605 } while ((cp
= cp
->cpu_next
) != cpu_list
);
11613 if (cpu
!= DTRACE_CPUALL
&& cpu
!= cp
->cpu_id
)
11616 buf
= &bufs
[cp
->cpu_id
];
11619 if (buf
->dtb_xamot
!= NULL
) {
11620 ASSERT(buf
->dtb_tomax
!= NULL
);
11621 ASSERT(buf
->dtb_size
== size
);
11622 kmem_free(buf
->dtb_xamot
, size
);
11626 if (buf
->dtb_tomax
!= NULL
) {
11627 ASSERT(buf
->dtb_size
== size
);
11628 kmem_free(buf
->dtb_tomax
, size
);
11632 buf
->dtb_tomax
= NULL
;
11633 buf
->dtb_xamot
= NULL
;
11635 } while ((cp
= cp
->cpu_next
) != cpu_list
);
11637 *factor
= desired
/ (allocated
> 0 ? allocated
: 1);
11643 * Note: called from probe context. This function just increments the drop
11644 * count on a buffer. It has been made a function to allow for the
11645 * possibility of understanding the source of mysterious drop counts. (A
11646 * problem for which one may be particularly disappointed that DTrace cannot
11647 * be used to understand DTrace.)
11650 dtrace_buffer_drop(dtrace_buffer_t
*buf
)
11656 * Note: called from probe context. This function is called to reserve space
11657 * in a buffer. If mstate is non-NULL, sets the scratch base and size in the
11658 * mstate. Returns the new offset in the buffer, or a negative value if an
11659 * error has occurred.
11662 dtrace_buffer_reserve(dtrace_buffer_t
*buf
, size_t needed
, size_t align
,
11663 dtrace_state_t
*state
, dtrace_mstate_t
*mstate
)
11665 intptr_t offs
= buf
->dtb_offset
, soffs
;
11670 if (buf
->dtb_flags
& DTRACEBUF_INACTIVE
)
11673 if ((tomax
= buf
->dtb_tomax
) == NULL
) {
11674 dtrace_buffer_drop(buf
);
11678 if (!(buf
->dtb_flags
& (DTRACEBUF_RING
| DTRACEBUF_FILL
))) {
11679 while (offs
& (align
- 1)) {
11681 * Assert that our alignment is off by a number which
11682 * is itself sizeof (uint32_t) aligned.
11684 ASSERT(!((align
- (offs
& (align
- 1))) &
11685 (sizeof (uint32_t) - 1)));
11686 DTRACE_STORE(uint32_t, tomax
, offs
, DTRACE_EPIDNONE
);
11687 offs
+= sizeof (uint32_t);
11690 if ((soffs
= offs
+ needed
) > buf
->dtb_size
) {
11691 dtrace_buffer_drop(buf
);
11695 if (mstate
== NULL
)
11698 mstate
->dtms_scratch_base
= (uintptr_t)tomax
+ soffs
;
11699 mstate
->dtms_scratch_size
= buf
->dtb_size
- soffs
;
11700 mstate
->dtms_scratch_ptr
= mstate
->dtms_scratch_base
;
11705 if (buf
->dtb_flags
& DTRACEBUF_FILL
) {
11706 if (state
->dts_activity
!= DTRACE_ACTIVITY_COOLDOWN
&&
11707 (buf
->dtb_flags
& DTRACEBUF_FULL
))
11712 total
= needed
+ (offs
& (align
- 1));
11715 * For a ring buffer, life is quite a bit more complicated. Before
11716 * we can store any padding, we need to adjust our wrapping offset.
11717 * (If we've never before wrapped or we're not about to, no adjustment
11720 if ((buf
->dtb_flags
& DTRACEBUF_WRAPPED
) ||
11721 offs
+ total
> buf
->dtb_size
) {
11722 woffs
= buf
->dtb_xamot_offset
;
11724 if (offs
+ total
> buf
->dtb_size
) {
11726 * We can't fit in the end of the buffer. First, a
11727 * sanity check that we can fit in the buffer at all.
11729 if (total
> buf
->dtb_size
) {
11730 dtrace_buffer_drop(buf
);
11735 * We're going to be storing at the top of the buffer,
11736 * so now we need to deal with the wrapped offset. We
11737 * only reset our wrapped offset to 0 if it is
11738 * currently greater than the current offset. If it
11739 * is less than the current offset, it is because a
11740 * previous allocation induced a wrap -- but the
11741 * allocation didn't subsequently take the space due
11742 * to an error or false predicate evaluation. In this
11743 * case, we'll just leave the wrapped offset alone: if
11744 * the wrapped offset hasn't been advanced far enough
11745 * for this allocation, it will be adjusted in the
11748 if (buf
->dtb_flags
& DTRACEBUF_WRAPPED
) {
11756 * Now we know that we're going to be storing to the
11757 * top of the buffer and that there is room for us
11758 * there. We need to clear the buffer from the current
11759 * offset to the end (there may be old gunk there).
11761 while (offs
< buf
->dtb_size
)
11765 * We need to set our offset to zero. And because we
11766 * are wrapping, we need to set the bit indicating as
11767 * much. We can also adjust our needed space back
11768 * down to the space required by the ECB -- we know
11769 * that the top of the buffer is aligned.
11773 buf
->dtb_flags
|= DTRACEBUF_WRAPPED
;
11776 * There is room for us in the buffer, so we simply
11777 * need to check the wrapped offset.
11779 if (woffs
< offs
) {
11781 * The wrapped offset is less than the offset.
11782 * This can happen if we allocated buffer space
11783 * that induced a wrap, but then we didn't
11784 * subsequently take the space due to an error
11785 * or false predicate evaluation. This is
11786 * okay; we know that _this_ allocation isn't
11787 * going to induce a wrap. We still can't
11788 * reset the wrapped offset to be zero,
11789 * however: the space may have been trashed in
11790 * the previous failed probe attempt. But at
11791 * least the wrapped offset doesn't need to
11792 * be adjusted at all...
11798 while (offs
+ total
> woffs
) {
11799 dtrace_epid_t epid
= *(uint32_t *)(tomax
+ woffs
);
11802 if (epid
== DTRACE_EPIDNONE
) {
11803 size
= sizeof (uint32_t);
11805 ASSERT3U(epid
, <=, state
->dts_necbs
);
11806 ASSERT(state
->dts_ecbs
[epid
- 1] != NULL
);
11808 size
= state
->dts_ecbs
[epid
- 1]->dte_size
;
11811 ASSERT(woffs
+ size
<= buf
->dtb_size
);
11814 if (woffs
+ size
== buf
->dtb_size
) {
11816 * We've reached the end of the buffer; we want
11817 * to set the wrapped offset to 0 and break
11818 * out. However, if the offs is 0, then we're
11819 * in a strange edge-condition: the amount of
11820 * space that we want to reserve plus the size
11821 * of the record that we're overwriting is
11822 * greater than the size of the buffer. This
11823 * is problematic because if we reserve the
11824 * space but subsequently don't consume it (due
11825 * to a failed predicate or error) the wrapped
11826 * offset will be 0 -- yet the EPID at offset 0
11827 * will not be committed. This situation is
11828 * relatively easy to deal with: if we're in
11829 * this case, the buffer is indistinguishable
11830 * from one that hasn't wrapped; we need only
11831 * finish the job by clearing the wrapped bit,
11832 * explicitly setting the offset to be 0, and
11833 * zero'ing out the old data in the buffer.
11836 buf
->dtb_flags
&= ~DTRACEBUF_WRAPPED
;
11837 buf
->dtb_offset
= 0;
11840 while (woffs
< buf
->dtb_size
)
11841 tomax
[woffs
++] = 0;
11852 * We have a wrapped offset. It may be that the wrapped offset
11853 * has become zero -- that's okay.
11855 buf
->dtb_xamot_offset
= woffs
;
11860 * Now we can plow the buffer with any necessary padding.
11862 while (offs
& (align
- 1)) {
11864 * Assert that our alignment is off by a number which
11865 * is itself sizeof (uint32_t) aligned.
11867 ASSERT(!((align
- (offs
& (align
- 1))) &
11868 (sizeof (uint32_t) - 1)));
11869 DTRACE_STORE(uint32_t, tomax
, offs
, DTRACE_EPIDNONE
);
11870 offs
+= sizeof (uint32_t);
11873 if (buf
->dtb_flags
& DTRACEBUF_FILL
) {
11874 if (offs
+ needed
> buf
->dtb_size
- state
->dts_reserve
) {
11875 buf
->dtb_flags
|= DTRACEBUF_FULL
;
11880 if (mstate
== NULL
)
11884 * For ring buffers and fill buffers, the scratch space is always
11885 * the inactive buffer.
11887 mstate
->dtms_scratch_base
= (uintptr_t)buf
->dtb_xamot
;
11888 mstate
->dtms_scratch_size
= buf
->dtb_size
;
11889 mstate
->dtms_scratch_ptr
= mstate
->dtms_scratch_base
;
11895 dtrace_buffer_polish(dtrace_buffer_t
*buf
)
11897 ASSERT(buf
->dtb_flags
& DTRACEBUF_RING
);
11898 ASSERT(MUTEX_HELD(&dtrace_lock
));
11900 if (!(buf
->dtb_flags
& DTRACEBUF_WRAPPED
))
11904 * We need to polish the ring buffer. There are three cases:
11906 * - The first (and presumably most common) is that there is no gap
11907 * between the buffer offset and the wrapped offset. In this case,
11908 * there is nothing in the buffer that isn't valid data; we can
11909 * mark the buffer as polished and return.
11911 * - The second (less common than the first but still more common
11912 * than the third) is that there is a gap between the buffer offset
11913 * and the wrapped offset, and the wrapped offset is larger than the
11914 * buffer offset. This can happen because of an alignment issue, or
11915 * can happen because of a call to dtrace_buffer_reserve() that
11916 * didn't subsequently consume the buffer space. In this case,
11917 * we need to zero the data from the buffer offset to the wrapped
11920 * - The third (and least common) is that there is a gap between the
11921 * buffer offset and the wrapped offset, but the wrapped offset is
11922 * _less_ than the buffer offset. This can only happen because a
11923 * call to dtrace_buffer_reserve() induced a wrap, but the space
11924 * was not subsequently consumed. In this case, we need to zero the
11925 * space from the offset to the end of the buffer _and_ from the
11926 * top of the buffer to the wrapped offset.
11928 if (buf
->dtb_offset
< buf
->dtb_xamot_offset
) {
11929 bzero(buf
->dtb_tomax
+ buf
->dtb_offset
,
11930 buf
->dtb_xamot_offset
- buf
->dtb_offset
);
11933 if (buf
->dtb_offset
> buf
->dtb_xamot_offset
) {
11934 bzero(buf
->dtb_tomax
+ buf
->dtb_offset
,
11935 buf
->dtb_size
- buf
->dtb_offset
);
11936 bzero(buf
->dtb_tomax
, buf
->dtb_xamot_offset
);
11941 * This routine determines if data generated at the specified time has likely
11942 * been entirely consumed at user-level. This routine is called to determine
11943 * if an ECB on a defunct probe (but for an active enabling) can be safely
11944 * disabled and destroyed.
11947 dtrace_buffer_consumed(dtrace_buffer_t
*bufs
, hrtime_t when
)
11951 for (i
= 0; i
< NCPU
; i
++) {
11952 dtrace_buffer_t
*buf
= &bufs
[i
];
11954 if (buf
->dtb_size
== 0)
11957 if (buf
->dtb_flags
& DTRACEBUF_RING
)
11960 if (!buf
->dtb_switched
&& buf
->dtb_offset
!= 0)
11963 if (buf
->dtb_switched
- buf
->dtb_interval
< when
)
11971 dtrace_buffer_free(dtrace_buffer_t
*bufs
)
11975 for (i
= 0; i
< NCPU
; i
++) {
11976 dtrace_buffer_t
*buf
= &bufs
[i
];
11978 if (buf
->dtb_tomax
== NULL
) {
11979 ASSERT(buf
->dtb_xamot
== NULL
);
11980 ASSERT(buf
->dtb_size
== 0);
11984 if (buf
->dtb_xamot
!= NULL
) {
11985 ASSERT(!(buf
->dtb_flags
& DTRACEBUF_NOSWITCH
));
11986 kmem_free(buf
->dtb_xamot
, buf
->dtb_size
);
11989 kmem_free(buf
->dtb_tomax
, buf
->dtb_size
);
11991 buf
->dtb_tomax
= NULL
;
11992 buf
->dtb_xamot
= NULL
;
11997 * DTrace Enabling Functions
11999 static dtrace_enabling_t
*
12000 dtrace_enabling_create(dtrace_vstate_t
*vstate
)
12002 dtrace_enabling_t
*enab
;
12004 enab
= kmem_zalloc(sizeof (dtrace_enabling_t
), KM_SLEEP
);
12005 enab
->dten_vstate
= vstate
;
12011 dtrace_enabling_add(dtrace_enabling_t
*enab
, dtrace_ecbdesc_t
*ecb
)
12013 dtrace_ecbdesc_t
**ndesc
;
12014 size_t osize
, nsize
;
12017 * We can't add to enablings after we've enabled them, or after we've
12020 ASSERT(enab
->dten_probegen
== 0);
12021 ASSERT(enab
->dten_next
== NULL
&& enab
->dten_prev
== NULL
);
12023 if (enab
->dten_ndesc
< enab
->dten_maxdesc
) {
12024 enab
->dten_desc
[enab
->dten_ndesc
++] = ecb
;
12028 osize
= enab
->dten_maxdesc
* sizeof (dtrace_enabling_t
*);
12030 if (enab
->dten_maxdesc
== 0) {
12031 enab
->dten_maxdesc
= 1;
12033 enab
->dten_maxdesc
<<= 1;
12036 ASSERT(enab
->dten_ndesc
< enab
->dten_maxdesc
);
12038 nsize
= enab
->dten_maxdesc
* sizeof (dtrace_enabling_t
*);
12039 ndesc
= kmem_zalloc(nsize
, KM_SLEEP
);
12040 bcopy(enab
->dten_desc
, ndesc
, osize
);
12041 kmem_free(enab
->dten_desc
, osize
);
12043 enab
->dten_desc
= ndesc
;
12044 enab
->dten_desc
[enab
->dten_ndesc
++] = ecb
;
12048 dtrace_enabling_addlike(dtrace_enabling_t
*enab
, dtrace_ecbdesc_t
*ecb
,
12049 dtrace_probedesc_t
*pd
)
12051 dtrace_ecbdesc_t
*new;
12052 dtrace_predicate_t
*pred
;
12053 dtrace_actdesc_t
*act
;
12056 * We're going to create a new ECB description that matches the
12057 * specified ECB in every way, but has the specified probe description.
12059 new = kmem_zalloc(sizeof (dtrace_ecbdesc_t
), KM_SLEEP
);
12061 if ((pred
= ecb
->dted_pred
.dtpdd_predicate
) != NULL
)
12062 dtrace_predicate_hold(pred
);
12064 for (act
= ecb
->dted_action
; act
!= NULL
; act
= act
->dtad_next
)
12065 dtrace_actdesc_hold(act
);
12067 new->dted_action
= ecb
->dted_action
;
12068 new->dted_pred
= ecb
->dted_pred
;
12069 new->dted_probe
= *pd
;
12070 new->dted_uarg
= ecb
->dted_uarg
;
12072 dtrace_enabling_add(enab
, new);
12076 dtrace_enabling_dump(dtrace_enabling_t
*enab
)
12080 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12081 dtrace_probedesc_t
*desc
= &enab
->dten_desc
[i
]->dted_probe
;
12083 cmn_err(CE_NOTE
, "enabling probe %d (%s:%s:%s:%s)", i
,
12084 desc
->dtpd_provider
, desc
->dtpd_mod
,
12085 desc
->dtpd_func
, desc
->dtpd_name
);
12090 dtrace_enabling_destroy(dtrace_enabling_t
*enab
)
12093 dtrace_ecbdesc_t
*ep
;
12094 dtrace_vstate_t
*vstate
= enab
->dten_vstate
;
12096 ASSERT(MUTEX_HELD(&dtrace_lock
));
12098 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12099 dtrace_actdesc_t
*act
, *next
;
12100 dtrace_predicate_t
*pred
;
12102 ep
= enab
->dten_desc
[i
];
12104 if ((pred
= ep
->dted_pred
.dtpdd_predicate
) != NULL
)
12105 dtrace_predicate_release(pred
, vstate
);
12107 for (act
= ep
->dted_action
; act
!= NULL
; act
= next
) {
12108 next
= act
->dtad_next
;
12109 dtrace_actdesc_release(act
, vstate
);
12112 kmem_free(ep
, sizeof (dtrace_ecbdesc_t
));
12115 kmem_free(enab
->dten_desc
,
12116 enab
->dten_maxdesc
* sizeof (dtrace_enabling_t
*));
12119 * If this was a retained enabling, decrement the dts_nretained count
12120 * and take it off of the dtrace_retained list.
12122 if (enab
->dten_prev
!= NULL
|| enab
->dten_next
!= NULL
||
12123 dtrace_retained
== enab
) {
12124 ASSERT(enab
->dten_vstate
->dtvs_state
!= NULL
);
12125 ASSERT(enab
->dten_vstate
->dtvs_state
->dts_nretained
> 0);
12126 enab
->dten_vstate
->dtvs_state
->dts_nretained
--;
12127 dtrace_retained_gen
++;
12130 if (enab
->dten_prev
== NULL
) {
12131 if (dtrace_retained
== enab
) {
12132 dtrace_retained
= enab
->dten_next
;
12134 if (dtrace_retained
!= NULL
)
12135 dtrace_retained
->dten_prev
= NULL
;
12138 ASSERT(enab
!= dtrace_retained
);
12139 ASSERT(dtrace_retained
!= NULL
);
12140 enab
->dten_prev
->dten_next
= enab
->dten_next
;
12143 if (enab
->dten_next
!= NULL
) {
12144 ASSERT(dtrace_retained
!= NULL
);
12145 enab
->dten_next
->dten_prev
= enab
->dten_prev
;
12148 kmem_free(enab
, sizeof (dtrace_enabling_t
));
12152 dtrace_enabling_retain(dtrace_enabling_t
*enab
)
12154 dtrace_state_t
*state
;
12156 ASSERT(MUTEX_HELD(&dtrace_lock
));
12157 ASSERT(enab
->dten_next
== NULL
&& enab
->dten_prev
== NULL
);
12158 ASSERT(enab
->dten_vstate
!= NULL
);
12160 state
= enab
->dten_vstate
->dtvs_state
;
12161 ASSERT(state
!= NULL
);
12164 * We only allow each state to retain dtrace_retain_max enablings.
12166 if (state
->dts_nretained
>= dtrace_retain_max
)
12169 state
->dts_nretained
++;
12170 dtrace_retained_gen
++;
12172 if (dtrace_retained
== NULL
) {
12173 dtrace_retained
= enab
;
12177 enab
->dten_next
= dtrace_retained
;
12178 dtrace_retained
->dten_prev
= enab
;
12179 dtrace_retained
= enab
;
12185 dtrace_enabling_replicate(dtrace_state_t
*state
, dtrace_probedesc_t
*match
,
12186 dtrace_probedesc_t
*create
)
12188 dtrace_enabling_t
*new, *enab
;
12189 int found
= 0, err
= ENOENT
;
12191 ASSERT(MUTEX_HELD(&dtrace_lock
));
12192 ASSERT(strlen(match
->dtpd_provider
) < DTRACE_PROVNAMELEN
);
12193 ASSERT(strlen(match
->dtpd_mod
) < DTRACE_MODNAMELEN
);
12194 ASSERT(strlen(match
->dtpd_func
) < DTRACE_FUNCNAMELEN
);
12195 ASSERT(strlen(match
->dtpd_name
) < DTRACE_NAMELEN
);
12197 new = dtrace_enabling_create(&state
->dts_vstate
);
12200 * Iterate over all retained enablings, looking for enablings that
12201 * match the specified state.
12203 for (enab
= dtrace_retained
; enab
!= NULL
; enab
= enab
->dten_next
) {
12207 * dtvs_state can only be NULL for helper enablings -- and
12208 * helper enablings can't be retained.
12210 ASSERT(enab
->dten_vstate
->dtvs_state
!= NULL
);
12212 if (enab
->dten_vstate
->dtvs_state
!= state
)
12216 * Now iterate over each probe description; we're looking for
12217 * an exact match to the specified probe description.
12219 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12220 dtrace_ecbdesc_t
*ep
= enab
->dten_desc
[i
];
12221 dtrace_probedesc_t
*pd
= &ep
->dted_probe
;
12223 if (strcmp(pd
->dtpd_provider
, match
->dtpd_provider
))
12226 if (strcmp(pd
->dtpd_mod
, match
->dtpd_mod
))
12229 if (strcmp(pd
->dtpd_func
, match
->dtpd_func
))
12232 if (strcmp(pd
->dtpd_name
, match
->dtpd_name
))
12236 * We have a winning probe! Add it to our growing
12240 dtrace_enabling_addlike(new, ep
, create
);
12244 if (!found
|| (err
= dtrace_enabling_retain(new)) != 0) {
12245 dtrace_enabling_destroy(new);
12253 dtrace_enabling_retract(dtrace_state_t
*state
)
12255 dtrace_enabling_t
*enab
, *next
;
12257 ASSERT(MUTEX_HELD(&dtrace_lock
));
12260 * Iterate over all retained enablings, destroy the enablings retained
12261 * for the specified state.
12263 for (enab
= dtrace_retained
; enab
!= NULL
; enab
= next
) {
12264 next
= enab
->dten_next
;
12267 * dtvs_state can only be NULL for helper enablings -- and
12268 * helper enablings can't be retained.
12270 ASSERT(enab
->dten_vstate
->dtvs_state
!= NULL
);
12272 if (enab
->dten_vstate
->dtvs_state
== state
) {
12273 ASSERT(state
->dts_nretained
> 0);
12274 dtrace_enabling_destroy(enab
);
12278 ASSERT(state
->dts_nretained
== 0);
12282 dtrace_enabling_match(dtrace_enabling_t
*enab
, int *nmatched
)
12285 int total_matched
= 0, matched
= 0;
12287 ASSERT(MUTEX_HELD(&cpu_lock
));
12288 ASSERT(MUTEX_HELD(&dtrace_lock
));
12290 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12291 dtrace_ecbdesc_t
*ep
= enab
->dten_desc
[i
];
12293 enab
->dten_current
= ep
;
12294 enab
->dten_error
= 0;
12297 * If a provider failed to enable a probe then get out and
12298 * let the consumer know we failed.
12300 if ((matched
= dtrace_probe_enable(&ep
->dted_probe
, enab
)) < 0)
12303 total_matched
+= matched
;
12305 if (enab
->dten_error
!= 0) {
12307 * If we get an error half-way through enabling the
12308 * probes, we kick out -- perhaps with some number of
12309 * them enabled. Leaving enabled probes enabled may
12310 * be slightly confusing for user-level, but we expect
12311 * that no one will attempt to actually drive on in
12312 * the face of such errors. If this is an anonymous
12313 * enabling (indicated with a NULL nmatched pointer),
12314 * we cmn_err() a message. We aren't expecting to
12315 * get such an error -- such as it can exist at all,
12316 * it would be a result of corrupted DOF in the driver
12319 if (nmatched
== NULL
) {
12320 cmn_err(CE_WARN
, "dtrace_enabling_match() "
12321 "error on %p: %d", (void *)ep
,
12325 return (enab
->dten_error
);
12329 enab
->dten_probegen
= dtrace_probegen
;
12330 if (nmatched
!= NULL
)
12331 *nmatched
= total_matched
;
12337 dtrace_enabling_matchall(void)
12339 dtrace_enabling_t
*enab
;
12341 mutex_enter(&cpu_lock
);
12342 mutex_enter(&dtrace_lock
);
12345 * Iterate over all retained enablings to see if any probes match
12346 * against them. We only perform this operation on enablings for which
12347 * we have sufficient permissions by virtue of being in the global zone
12348 * or in the same zone as the DTrace client. Because we can be called
12349 * after dtrace_detach() has been called, we cannot assert that there
12350 * are retained enablings. We can safely load from dtrace_retained,
12351 * however: the taskq_destroy() at the end of dtrace_detach() will
12352 * block pending our completion.
12354 for (enab
= dtrace_retained
; enab
!= NULL
; enab
= enab
->dten_next
) {
12355 dtrace_cred_t
*dcr
= &enab
->dten_vstate
->dtvs_state
->dts_cred
;
12356 cred_t
*cr
= dcr
->dcr_cred
;
12357 zoneid_t zone
= cr
!= NULL
? crgetzoneid(cr
) : 0;
12359 if ((dcr
->dcr_visible
& DTRACE_CRV_ALLZONE
) || (cr
!= NULL
&&
12360 (zone
== GLOBAL_ZONEID
|| getzoneid() == zone
)))
12361 (void) dtrace_enabling_match(enab
, NULL
);
12364 mutex_exit(&dtrace_lock
);
12365 mutex_exit(&cpu_lock
);
12369 * If an enabling is to be enabled without having matched probes (that is, if
12370 * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
12371 * enabling must be _primed_ by creating an ECB for every ECB description.
12372 * This must be done to assure that we know the number of speculations, the
12373 * number of aggregations, the minimum buffer size needed, etc. before we
12374 * transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually
12375 * enabling any probes, we create ECBs for every ECB decription, but with a
12376 * NULL probe -- which is exactly what this function does.
12379 dtrace_enabling_prime(dtrace_state_t
*state
)
12381 dtrace_enabling_t
*enab
;
12384 for (enab
= dtrace_retained
; enab
!= NULL
; enab
= enab
->dten_next
) {
12385 ASSERT(enab
->dten_vstate
->dtvs_state
!= NULL
);
12387 if (enab
->dten_vstate
->dtvs_state
!= state
)
12391 * We don't want to prime an enabling more than once, lest
12392 * we allow a malicious user to induce resource exhaustion.
12393 * (The ECBs that result from priming an enabling aren't
12394 * leaked -- but they also aren't deallocated until the
12395 * consumer state is destroyed.)
12397 if (enab
->dten_primed
)
12400 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12401 enab
->dten_current
= enab
->dten_desc
[i
];
12402 (void) dtrace_probe_enable(NULL
, enab
);
12405 enab
->dten_primed
= 1;
12410 * Called to indicate that probes should be provided due to retained
12411 * enablings. This is implemented in terms of dtrace_probe_provide(), but it
12412 * must take an initial lap through the enabling calling the dtps_provide()
12413 * entry point explicitly to allow for autocreated probes.
12416 dtrace_enabling_provide(dtrace_provider_t
*prv
)
12419 dtrace_probedesc_t desc
;
12420 dtrace_genid_t gen
;
12422 ASSERT(MUTEX_HELD(&dtrace_lock
));
12423 ASSERT(MUTEX_HELD(&dtrace_provider_lock
));
12427 prv
= dtrace_provider
;
12431 dtrace_enabling_t
*enab
;
12432 void *parg
= prv
->dtpv_arg
;
12435 gen
= dtrace_retained_gen
;
12436 for (enab
= dtrace_retained
; enab
!= NULL
;
12437 enab
= enab
->dten_next
) {
12438 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
12439 desc
= enab
->dten_desc
[i
]->dted_probe
;
12440 mutex_exit(&dtrace_lock
);
12441 prv
->dtpv_pops
.dtps_provide(parg
, &desc
);
12442 mutex_enter(&dtrace_lock
);
12444 * Process the retained enablings again if
12445 * they have changed while we weren't holding
12448 if (gen
!= dtrace_retained_gen
)
12452 } while (all
&& (prv
= prv
->dtpv_next
) != NULL
);
12454 mutex_exit(&dtrace_lock
);
12455 dtrace_probe_provide(NULL
, all
? NULL
: prv
);
12456 mutex_enter(&dtrace_lock
);
12460 * Called to reap ECBs that are attached to probes from defunct providers.
12463 dtrace_enabling_reap(void)
12465 dtrace_provider_t
*prov
;
12466 dtrace_probe_t
*probe
;
12471 mutex_enter(&cpu_lock
);
12472 mutex_enter(&dtrace_lock
);
12474 for (i
= 0; i
< dtrace_nprobes
; i
++) {
12475 if ((probe
= dtrace_probes
[i
]) == NULL
)
12478 if (probe
->dtpr_ecb
== NULL
)
12481 prov
= probe
->dtpr_provider
;
12483 if ((when
= prov
->dtpv_defunct
) == 0)
12487 * We have ECBs on a defunct provider: we want to reap these
12488 * ECBs to allow the provider to unregister. The destruction
12489 * of these ECBs must be done carefully: if we destroy the ECB
12490 * and the consumer later wishes to consume an EPID that
12491 * corresponds to the destroyed ECB (and if the EPID metadata
12492 * has not been previously consumed), the consumer will abort
12493 * processing on the unknown EPID. To reduce (but not, sadly,
12494 * eliminate) the possibility of this, we will only destroy an
12495 * ECB for a defunct provider if, for the state that
12496 * corresponds to the ECB:
12498 * (a) There is no speculative tracing (which can effectively
12499 * cache an EPID for an arbitrary amount of time).
12501 * (b) The principal buffers have been switched twice since the
12502 * provider became defunct.
12504 * (c) The aggregation buffers are of zero size or have been
12505 * switched twice since the provider became defunct.
12507 * We use dts_speculates to determine (a) and call a function
12508 * (dtrace_buffer_consumed()) to determine (b) and (c). Note
12509 * that as soon as we've been unable to destroy one of the ECBs
12510 * associated with the probe, we quit trying -- reaping is only
12511 * fruitful in as much as we can destroy all ECBs associated
12512 * with the defunct provider's probes.
12514 while ((ecb
= probe
->dtpr_ecb
) != NULL
) {
12515 dtrace_state_t
*state
= ecb
->dte_state
;
12516 dtrace_buffer_t
*buf
= state
->dts_buffer
;
12517 dtrace_buffer_t
*aggbuf
= state
->dts_aggbuffer
;
12519 if (state
->dts_speculates
)
12522 if (!dtrace_buffer_consumed(buf
, when
))
12525 if (!dtrace_buffer_consumed(aggbuf
, when
))
12528 dtrace_ecb_disable(ecb
);
12529 ASSERT(probe
->dtpr_ecb
!= ecb
);
12530 dtrace_ecb_destroy(ecb
);
12534 mutex_exit(&dtrace_lock
);
12535 mutex_exit(&cpu_lock
);
12539 * DTrace DOF Functions
12543 dtrace_dof_error(dof_hdr_t
*dof
, const char *str
)
12545 if (dtrace_err_verbose
)
12546 cmn_err(CE_WARN
, "failed to process DOF: %s", str
);
12548 #ifdef DTRACE_ERRDEBUG
12549 dtrace_errdebug(str
);
12554 * Create DOF out of a currently enabled state. Right now, we only create
12555 * DOF containing the run-time options -- but this could be expanded to create
12556 * complete DOF representing the enabled state.
12559 dtrace_dof_create(dtrace_state_t
*state
)
12563 dof_optdesc_t
*opt
;
12564 int i
, len
= sizeof (dof_hdr_t
) +
12565 roundup(sizeof (dof_sec_t
), sizeof (uint64_t)) +
12566 sizeof (dof_optdesc_t
) * DTRACEOPT_MAX
;
12568 ASSERT(MUTEX_HELD(&dtrace_lock
));
12570 dof
= kmem_zalloc(len
, KM_SLEEP
);
12571 dof
->dofh_ident
[DOF_ID_MAG0
] = DOF_MAG_MAG0
;
12572 dof
->dofh_ident
[DOF_ID_MAG1
] = DOF_MAG_MAG1
;
12573 dof
->dofh_ident
[DOF_ID_MAG2
] = DOF_MAG_MAG2
;
12574 dof
->dofh_ident
[DOF_ID_MAG3
] = DOF_MAG_MAG3
;
12576 dof
->dofh_ident
[DOF_ID_MODEL
] = DOF_MODEL_NATIVE
;
12577 dof
->dofh_ident
[DOF_ID_ENCODING
] = DOF_ENCODE_NATIVE
;
12578 dof
->dofh_ident
[DOF_ID_VERSION
] = DOF_VERSION
;
12579 dof
->dofh_ident
[DOF_ID_DIFVERS
] = DIF_VERSION
;
12580 dof
->dofh_ident
[DOF_ID_DIFIREG
] = DIF_DIR_NREGS
;
12581 dof
->dofh_ident
[DOF_ID_DIFTREG
] = DIF_DTR_NREGS
;
12583 dof
->dofh_flags
= 0;
12584 dof
->dofh_hdrsize
= sizeof (dof_hdr_t
);
12585 dof
->dofh_secsize
= sizeof (dof_sec_t
);
12586 dof
->dofh_secnum
= 1; /* only DOF_SECT_OPTDESC */
12587 dof
->dofh_secoff
= sizeof (dof_hdr_t
);
12588 dof
->dofh_loadsz
= len
;
12589 dof
->dofh_filesz
= len
;
12593 * Fill in the option section header...
12595 sec
= (dof_sec_t
*)((uintptr_t)dof
+ sizeof (dof_hdr_t
));
12596 sec
->dofs_type
= DOF_SECT_OPTDESC
;
12597 sec
->dofs_align
= sizeof (uint64_t);
12598 sec
->dofs_flags
= DOF_SECF_LOAD
;
12599 sec
->dofs_entsize
= sizeof (dof_optdesc_t
);
12601 opt
= (dof_optdesc_t
*)((uintptr_t)sec
+
12602 roundup(sizeof (dof_sec_t
), sizeof (uint64_t)));
12604 sec
->dofs_offset
= (uintptr_t)opt
- (uintptr_t)dof
;
12605 sec
->dofs_size
= sizeof (dof_optdesc_t
) * DTRACEOPT_MAX
;
12607 for (i
= 0; i
< DTRACEOPT_MAX
; i
++) {
12608 opt
[i
].dofo_option
= i
;
12609 opt
[i
].dofo_strtab
= DOF_SECIDX_NONE
;
12610 opt
[i
].dofo_value
= state
->dts_options
[i
];
12617 dtrace_dof_copyin(uintptr_t uarg
, int *errp
)
12619 dof_hdr_t hdr
, *dof
;
12621 ASSERT(!MUTEX_HELD(&dtrace_lock
));
12624 * First, we're going to copyin() the sizeof (dof_hdr_t).
12626 if (copyin((void *)uarg
, &hdr
, sizeof (hdr
)) != 0) {
12627 dtrace_dof_error(NULL
, "failed to copyin DOF header");
12633 * Now we'll allocate the entire DOF and copy it in -- provided
12634 * that the length isn't outrageous.
12636 if (hdr
.dofh_loadsz
>= dtrace_dof_maxsize
) {
12637 dtrace_dof_error(&hdr
, "load size exceeds maximum");
12642 if (hdr
.dofh_loadsz
< sizeof (hdr
)) {
12643 dtrace_dof_error(&hdr
, "invalid load size");
12648 dof
= kmem_alloc(hdr
.dofh_loadsz
, KM_SLEEP
);
12650 if (copyin((void *)uarg
, dof
, hdr
.dofh_loadsz
) != 0 ||
12651 dof
->dofh_loadsz
!= hdr
.dofh_loadsz
) {
12652 kmem_free(dof
, hdr
.dofh_loadsz
);
12661 dtrace_dof_property(const char *name
)
12665 unsigned int len
, i
;
12669 * Unfortunately, array of values in .conf files are always (and
12670 * only) interpreted to be integer arrays. We must read our DOF
12671 * as an integer array, and then squeeze it into a byte array.
12673 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY
, dtrace_devi
, 0,
12674 (char *)name
, (int **)&buf
, &len
) != DDI_PROP_SUCCESS
)
12677 for (i
= 0; i
< len
; i
++)
12678 buf
[i
] = (uchar_t
)(((int *)buf
)[i
]);
12680 if (len
< sizeof (dof_hdr_t
)) {
12681 ddi_prop_free(buf
);
12682 dtrace_dof_error(NULL
, "truncated header");
12686 if (len
< (loadsz
= ((dof_hdr_t
*)buf
)->dofh_loadsz
)) {
12687 ddi_prop_free(buf
);
12688 dtrace_dof_error(NULL
, "truncated DOF");
12692 if (loadsz
>= dtrace_dof_maxsize
) {
12693 ddi_prop_free(buf
);
12694 dtrace_dof_error(NULL
, "oversized DOF");
12698 dof
= kmem_alloc(loadsz
, KM_SLEEP
);
12699 bcopy(buf
, dof
, loadsz
);
12700 ddi_prop_free(buf
);
12706 dtrace_dof_destroy(dof_hdr_t
*dof
)
12708 kmem_free(dof
, dof
->dofh_loadsz
);
12712 * Return the dof_sec_t pointer corresponding to a given section index. If the
12713 * index is not valid, dtrace_dof_error() is called and NULL is returned. If
12714 * a type other than DOF_SECT_NONE is specified, the header is checked against
12715 * this type and NULL is returned if the types do not match.
12718 dtrace_dof_sect(dof_hdr_t
*dof
, uint32_t type
, dof_secidx_t i
)
12720 dof_sec_t
*sec
= (dof_sec_t
*)(uintptr_t)
12721 ((uintptr_t)dof
+ dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
12723 if (i
>= dof
->dofh_secnum
) {
12724 dtrace_dof_error(dof
, "referenced section index is invalid");
12728 if (!(sec
->dofs_flags
& DOF_SECF_LOAD
)) {
12729 dtrace_dof_error(dof
, "referenced section is not loadable");
12733 if (type
!= DOF_SECT_NONE
&& type
!= sec
->dofs_type
) {
12734 dtrace_dof_error(dof
, "referenced section is the wrong type");
12741 static dtrace_probedesc_t
*
12742 dtrace_dof_probedesc(dof_hdr_t
*dof
, dof_sec_t
*sec
, dtrace_probedesc_t
*desc
)
12744 dof_probedesc_t
*probe
;
12746 uintptr_t daddr
= (uintptr_t)dof
;
12750 if (sec
->dofs_type
!= DOF_SECT_PROBEDESC
) {
12751 dtrace_dof_error(dof
, "invalid probe section");
12755 if (sec
->dofs_align
!= sizeof (dof_secidx_t
)) {
12756 dtrace_dof_error(dof
, "bad alignment in probe description");
12760 if (sec
->dofs_offset
+ sizeof (dof_probedesc_t
) > dof
->dofh_loadsz
) {
12761 dtrace_dof_error(dof
, "truncated probe description");
12765 probe
= (dof_probedesc_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
12766 strtab
= dtrace_dof_sect(dof
, DOF_SECT_STRTAB
, probe
->dofp_strtab
);
12768 if (strtab
== NULL
)
12771 str
= daddr
+ strtab
->dofs_offset
;
12772 size
= strtab
->dofs_size
;
12774 if (probe
->dofp_provider
>= strtab
->dofs_size
) {
12775 dtrace_dof_error(dof
, "corrupt probe provider");
12779 (void) strncpy(desc
->dtpd_provider
,
12780 (char *)(str
+ probe
->dofp_provider
),
12781 MIN(DTRACE_PROVNAMELEN
- 1, size
- probe
->dofp_provider
));
12783 if (probe
->dofp_mod
>= strtab
->dofs_size
) {
12784 dtrace_dof_error(dof
, "corrupt probe module");
12788 (void) strncpy(desc
->dtpd_mod
, (char *)(str
+ probe
->dofp_mod
),
12789 MIN(DTRACE_MODNAMELEN
- 1, size
- probe
->dofp_mod
));
12791 if (probe
->dofp_func
>= strtab
->dofs_size
) {
12792 dtrace_dof_error(dof
, "corrupt probe function");
12796 (void) strncpy(desc
->dtpd_func
, (char *)(str
+ probe
->dofp_func
),
12797 MIN(DTRACE_FUNCNAMELEN
- 1, size
- probe
->dofp_func
));
12799 if (probe
->dofp_name
>= strtab
->dofs_size
) {
12800 dtrace_dof_error(dof
, "corrupt probe name");
12804 (void) strncpy(desc
->dtpd_name
, (char *)(str
+ probe
->dofp_name
),
12805 MIN(DTRACE_NAMELEN
- 1, size
- probe
->dofp_name
));
12810 static dtrace_difo_t
*
12811 dtrace_dof_difo(dof_hdr_t
*dof
, dof_sec_t
*sec
, dtrace_vstate_t
*vstate
,
12816 dof_difohdr_t
*dofd
;
12817 uintptr_t daddr
= (uintptr_t)dof
;
12818 size_t max
= dtrace_difo_maxsize
;
12821 static const struct {
12829 { DOF_SECT_DIF
, offsetof(dtrace_difo_t
, dtdo_buf
),
12830 offsetof(dtrace_difo_t
, dtdo_len
), sizeof (dif_instr_t
),
12831 sizeof (dif_instr_t
), "multiple DIF sections" },
12833 { DOF_SECT_INTTAB
, offsetof(dtrace_difo_t
, dtdo_inttab
),
12834 offsetof(dtrace_difo_t
, dtdo_intlen
), sizeof (uint64_t),
12835 sizeof (uint64_t), "multiple integer tables" },
12837 { DOF_SECT_STRTAB
, offsetof(dtrace_difo_t
, dtdo_strtab
),
12838 offsetof(dtrace_difo_t
, dtdo_strlen
), 0,
12839 sizeof (char), "multiple string tables" },
12841 { DOF_SECT_VARTAB
, offsetof(dtrace_difo_t
, dtdo_vartab
),
12842 offsetof(dtrace_difo_t
, dtdo_varlen
), sizeof (dtrace_difv_t
),
12843 sizeof (uint_t
), "multiple variable tables" },
12845 { DOF_SECT_NONE
, 0, 0, 0, NULL
}
12848 if (sec
->dofs_type
!= DOF_SECT_DIFOHDR
) {
12849 dtrace_dof_error(dof
, "invalid DIFO header section");
12853 if (sec
->dofs_align
!= sizeof (dof_secidx_t
)) {
12854 dtrace_dof_error(dof
, "bad alignment in DIFO header");
12858 if (sec
->dofs_size
< sizeof (dof_difohdr_t
) ||
12859 sec
->dofs_size
% sizeof (dof_secidx_t
)) {
12860 dtrace_dof_error(dof
, "bad size in DIFO header");
12864 dofd
= (dof_difohdr_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
12865 n
= (sec
->dofs_size
- sizeof (*dofd
)) / sizeof (dof_secidx_t
) + 1;
12867 dp
= kmem_zalloc(sizeof (dtrace_difo_t
), KM_SLEEP
);
12868 dp
->dtdo_rtype
= dofd
->dofd_rtype
;
12870 for (l
= 0; l
< n
; l
++) {
12875 if ((subsec
= dtrace_dof_sect(dof
, DOF_SECT_NONE
,
12876 dofd
->dofd_links
[l
])) == NULL
)
12877 goto err
; /* invalid section link */
12879 if (ttl
+ subsec
->dofs_size
> max
) {
12880 dtrace_dof_error(dof
, "exceeds maximum size");
12884 ttl
+= subsec
->dofs_size
;
12886 for (i
= 0; difo
[i
].section
!= DOF_SECT_NONE
; i
++) {
12887 if (subsec
->dofs_type
!= difo
[i
].section
)
12890 if (!(subsec
->dofs_flags
& DOF_SECF_LOAD
)) {
12891 dtrace_dof_error(dof
, "section not loaded");
12895 if (subsec
->dofs_align
!= difo
[i
].align
) {
12896 dtrace_dof_error(dof
, "bad alignment");
12900 bufp
= (void **)((uintptr_t)dp
+ difo
[i
].bufoffs
);
12901 lenp
= (uint32_t *)((uintptr_t)dp
+ difo
[i
].lenoffs
);
12903 if (*bufp
!= NULL
) {
12904 dtrace_dof_error(dof
, difo
[i
].msg
);
12908 if (difo
[i
].entsize
!= subsec
->dofs_entsize
) {
12909 dtrace_dof_error(dof
, "entry size mismatch");
12913 if (subsec
->dofs_entsize
!= 0 &&
12914 (subsec
->dofs_size
% subsec
->dofs_entsize
) != 0) {
12915 dtrace_dof_error(dof
, "corrupt entry size");
12919 *lenp
= subsec
->dofs_size
;
12920 *bufp
= kmem_alloc(subsec
->dofs_size
, KM_SLEEP
);
12921 bcopy((char *)(uintptr_t)(daddr
+ subsec
->dofs_offset
),
12922 *bufp
, subsec
->dofs_size
);
12924 if (subsec
->dofs_entsize
!= 0)
12925 *lenp
/= subsec
->dofs_entsize
;
12931 * If we encounter a loadable DIFO sub-section that is not
12932 * known to us, assume this is a broken program and fail.
12934 if (difo
[i
].section
== DOF_SECT_NONE
&&
12935 (subsec
->dofs_flags
& DOF_SECF_LOAD
)) {
12936 dtrace_dof_error(dof
, "unrecognized DIFO subsection");
12941 if (dp
->dtdo_buf
== NULL
) {
12943 * We can't have a DIF object without DIF text.
12945 dtrace_dof_error(dof
, "missing DIF text");
12950 * Before we validate the DIF object, run through the variable table
12951 * looking for the strings -- if any of their size are under, we'll set
12952 * their size to be the system-wide default string size. Note that
12953 * this should _not_ happen if the "strsize" option has been set --
12954 * in this case, the compiler should have set the size to reflect the
12955 * setting of the option.
12957 for (i
= 0; i
< dp
->dtdo_varlen
; i
++) {
12958 dtrace_difv_t
*v
= &dp
->dtdo_vartab
[i
];
12959 dtrace_diftype_t
*t
= &v
->dtdv_type
;
12961 if (v
->dtdv_id
< DIF_VAR_OTHER_UBASE
)
12964 if (t
->dtdt_kind
== DIF_TYPE_STRING
&& t
->dtdt_size
== 0)
12965 t
->dtdt_size
= dtrace_strsize_default
;
12968 if (dtrace_difo_validate(dp
, vstate
, DIF_DIR_NREGS
, cr
) != 0)
12971 dtrace_difo_init(dp
, vstate
);
12975 kmem_free(dp
->dtdo_buf
, dp
->dtdo_len
* sizeof (dif_instr_t
));
12976 kmem_free(dp
->dtdo_inttab
, dp
->dtdo_intlen
* sizeof (uint64_t));
12977 kmem_free(dp
->dtdo_strtab
, dp
->dtdo_strlen
);
12978 kmem_free(dp
->dtdo_vartab
, dp
->dtdo_varlen
* sizeof (dtrace_difv_t
));
12980 kmem_free(dp
, sizeof (dtrace_difo_t
));
12984 static dtrace_predicate_t
*
12985 dtrace_dof_predicate(dof_hdr_t
*dof
, dof_sec_t
*sec
, dtrace_vstate_t
*vstate
,
12990 if ((dp
= dtrace_dof_difo(dof
, sec
, vstate
, cr
)) == NULL
)
12993 return (dtrace_predicate_create(dp
));
12996 static dtrace_actdesc_t
*
12997 dtrace_dof_actdesc(dof_hdr_t
*dof
, dof_sec_t
*sec
, dtrace_vstate_t
*vstate
,
13000 dtrace_actdesc_t
*act
, *first
= NULL
, *last
= NULL
, *next
;
13001 dof_actdesc_t
*desc
;
13002 dof_sec_t
*difosec
;
13004 uintptr_t daddr
= (uintptr_t)dof
;
13006 dtrace_actkind_t kind
;
13008 if (sec
->dofs_type
!= DOF_SECT_ACTDESC
) {
13009 dtrace_dof_error(dof
, "invalid action section");
13013 if (sec
->dofs_offset
+ sizeof (dof_actdesc_t
) > dof
->dofh_loadsz
) {
13014 dtrace_dof_error(dof
, "truncated action description");
13018 if (sec
->dofs_align
!= sizeof (uint64_t)) {
13019 dtrace_dof_error(dof
, "bad alignment in action description");
13023 if (sec
->dofs_size
< sec
->dofs_entsize
) {
13024 dtrace_dof_error(dof
, "section entry size exceeds total size");
13028 if (sec
->dofs_entsize
!= sizeof (dof_actdesc_t
)) {
13029 dtrace_dof_error(dof
, "bad entry size in action description");
13033 if (sec
->dofs_size
/ sec
->dofs_entsize
> dtrace_actions_max
) {
13034 dtrace_dof_error(dof
, "actions exceed dtrace_actions_max");
13038 for (offs
= 0; offs
< sec
->dofs_size
; offs
+= sec
->dofs_entsize
) {
13039 desc
= (dof_actdesc_t
*)(daddr
+
13040 (uintptr_t)sec
->dofs_offset
+ offs
);
13041 kind
= (dtrace_actkind_t
)desc
->dofa_kind
;
13043 if ((DTRACEACT_ISPRINTFLIKE(kind
) &&
13044 (kind
!= DTRACEACT_PRINTA
||
13045 desc
->dofa_strtab
!= DOF_SECIDX_NONE
)) ||
13046 (kind
== DTRACEACT_DIFEXPR
&&
13047 desc
->dofa_strtab
!= DOF_SECIDX_NONE
)) {
13053 * The argument to these actions is an index into the
13054 * DOF string table. For printf()-like actions, this
13055 * is the format string. For print(), this is the
13056 * CTF type of the expression result.
13058 if ((strtab
= dtrace_dof_sect(dof
,
13059 DOF_SECT_STRTAB
, desc
->dofa_strtab
)) == NULL
)
13062 str
= (char *)((uintptr_t)dof
+
13063 (uintptr_t)strtab
->dofs_offset
);
13065 for (i
= desc
->dofa_arg
; i
< strtab
->dofs_size
; i
++) {
13066 if (str
[i
] == '\0')
13070 if (i
>= strtab
->dofs_size
) {
13071 dtrace_dof_error(dof
, "bogus format string");
13075 if (i
== desc
->dofa_arg
) {
13076 dtrace_dof_error(dof
, "empty format string");
13080 i
-= desc
->dofa_arg
;
13081 fmt
= kmem_alloc(i
+ 1, KM_SLEEP
);
13082 bcopy(&str
[desc
->dofa_arg
], fmt
, i
+ 1);
13083 arg
= (uint64_t)(uintptr_t)fmt
;
13085 if (kind
== DTRACEACT_PRINTA
) {
13086 ASSERT(desc
->dofa_strtab
== DOF_SECIDX_NONE
);
13089 arg
= desc
->dofa_arg
;
13093 act
= dtrace_actdesc_create(kind
, desc
->dofa_ntuple
,
13094 desc
->dofa_uarg
, arg
);
13096 if (last
!= NULL
) {
13097 last
->dtad_next
= act
;
13104 if (desc
->dofa_difo
== DOF_SECIDX_NONE
)
13107 if ((difosec
= dtrace_dof_sect(dof
,
13108 DOF_SECT_DIFOHDR
, desc
->dofa_difo
)) == NULL
)
13111 act
->dtad_difo
= dtrace_dof_difo(dof
, difosec
, vstate
, cr
);
13113 if (act
->dtad_difo
== NULL
)
13117 ASSERT(first
!= NULL
);
13121 for (act
= first
; act
!= NULL
; act
= next
) {
13122 next
= act
->dtad_next
;
13123 dtrace_actdesc_release(act
, vstate
);
13129 static dtrace_ecbdesc_t
*
13130 dtrace_dof_ecbdesc(dof_hdr_t
*dof
, dof_sec_t
*sec
, dtrace_vstate_t
*vstate
,
13133 dtrace_ecbdesc_t
*ep
;
13134 dof_ecbdesc_t
*ecb
;
13135 dtrace_probedesc_t
*desc
;
13136 dtrace_predicate_t
*pred
= NULL
;
13138 if (sec
->dofs_size
< sizeof (dof_ecbdesc_t
)) {
13139 dtrace_dof_error(dof
, "truncated ECB description");
13143 if (sec
->dofs_align
!= sizeof (uint64_t)) {
13144 dtrace_dof_error(dof
, "bad alignment in ECB description");
13148 ecb
= (dof_ecbdesc_t
*)((uintptr_t)dof
+ (uintptr_t)sec
->dofs_offset
);
13149 sec
= dtrace_dof_sect(dof
, DOF_SECT_PROBEDESC
, ecb
->dofe_probes
);
13154 ep
= kmem_zalloc(sizeof (dtrace_ecbdesc_t
), KM_SLEEP
);
13155 ep
->dted_uarg
= ecb
->dofe_uarg
;
13156 desc
= &ep
->dted_probe
;
13158 if (dtrace_dof_probedesc(dof
, sec
, desc
) == NULL
)
13161 if (ecb
->dofe_pred
!= DOF_SECIDX_NONE
) {
13162 if ((sec
= dtrace_dof_sect(dof
,
13163 DOF_SECT_DIFOHDR
, ecb
->dofe_pred
)) == NULL
)
13166 if ((pred
= dtrace_dof_predicate(dof
, sec
, vstate
, cr
)) == NULL
)
13169 ep
->dted_pred
.dtpdd_predicate
= pred
;
13172 if (ecb
->dofe_actions
!= DOF_SECIDX_NONE
) {
13173 if ((sec
= dtrace_dof_sect(dof
,
13174 DOF_SECT_ACTDESC
, ecb
->dofe_actions
)) == NULL
)
13177 ep
->dted_action
= dtrace_dof_actdesc(dof
, sec
, vstate
, cr
);
13179 if (ep
->dted_action
== NULL
)
13187 dtrace_predicate_release(pred
, vstate
);
13188 kmem_free(ep
, sizeof (dtrace_ecbdesc_t
));
13193 * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
13194 * specified DOF. At present, this amounts to simply adding 'ubase' to the
13195 * site of any user SETX relocations to account for load object base address.
13196 * In the future, if we need other relocations, this function can be extended.
13199 dtrace_dof_relocate(dof_hdr_t
*dof
, dof_sec_t
*sec
, uint64_t ubase
)
13201 uintptr_t daddr
= (uintptr_t)dof
;
13203 dof_relohdr_t
*dofr
=
13204 (dof_relohdr_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
13205 dof_sec_t
*ss
, *rs
, *ts
;
13209 if (sec
->dofs_size
< sizeof (dof_relohdr_t
) ||
13210 sec
->dofs_align
!= sizeof (dof_secidx_t
)) {
13211 dtrace_dof_error(dof
, "invalid relocation header");
13215 ss
= dtrace_dof_sect(dof
, DOF_SECT_STRTAB
, dofr
->dofr_strtab
);
13216 rs
= dtrace_dof_sect(dof
, DOF_SECT_RELTAB
, dofr
->dofr_relsec
);
13217 ts
= dtrace_dof_sect(dof
, DOF_SECT_NONE
, dofr
->dofr_tgtsec
);
13218 ts_end
= (uintptr_t)ts
+ sizeof (dof_sec_t
);
13220 if (ss
== NULL
|| rs
== NULL
|| ts
== NULL
)
13221 return (-1); /* dtrace_dof_error() has been called already */
13223 if (rs
->dofs_entsize
< sizeof (dof_relodesc_t
) ||
13224 rs
->dofs_align
!= sizeof (uint64_t)) {
13225 dtrace_dof_error(dof
, "invalid relocation section");
13229 r
= (dof_relodesc_t
*)(uintptr_t)(daddr
+ rs
->dofs_offset
);
13230 n
= rs
->dofs_size
/ rs
->dofs_entsize
;
13232 for (i
= 0; i
< n
; i
++) {
13233 uintptr_t taddr
= daddr
+ ts
->dofs_offset
+ r
->dofr_offset
;
13235 switch (r
->dofr_type
) {
13236 case DOF_RELO_NONE
:
13238 case DOF_RELO_SETX
:
13239 if (r
->dofr_offset
>= ts
->dofs_size
|| r
->dofr_offset
+
13240 sizeof (uint64_t) > ts
->dofs_size
) {
13241 dtrace_dof_error(dof
, "bad relocation offset");
13245 if (taddr
>= (uintptr_t)ts
&& taddr
< ts_end
) {
13246 dtrace_dof_error(dof
, "bad relocation offset");
13250 if (!IS_P2ALIGNED(taddr
, sizeof (uint64_t))) {
13251 dtrace_dof_error(dof
, "misaligned setx relo");
13255 *(uint64_t *)taddr
+= ubase
;
13258 dtrace_dof_error(dof
, "invalid relocation type");
13262 r
= (dof_relodesc_t
*)((uintptr_t)r
+ rs
->dofs_entsize
);
13269 * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
13270 * header: it should be at the front of a memory region that is at least
13271 * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
13272 * size. It need not be validated in any other way.
13275 dtrace_dof_slurp(dof_hdr_t
*dof
, dtrace_vstate_t
*vstate
, cred_t
*cr
,
13276 dtrace_enabling_t
**enabp
, uint64_t ubase
, int noprobes
)
13278 uint64_t len
= dof
->dofh_loadsz
, seclen
;
13279 uintptr_t daddr
= (uintptr_t)dof
;
13280 dtrace_ecbdesc_t
*ep
;
13281 dtrace_enabling_t
*enab
;
13284 ASSERT(MUTEX_HELD(&dtrace_lock
));
13285 ASSERT(dof
->dofh_loadsz
>= sizeof (dof_hdr_t
));
13288 * Check the DOF header identification bytes. In addition to checking
13289 * valid settings, we also verify that unused bits/bytes are zeroed so
13290 * we can use them later without fear of regressing existing binaries.
13292 if (bcmp(&dof
->dofh_ident
[DOF_ID_MAG0
],
13293 DOF_MAG_STRING
, DOF_MAG_STRLEN
) != 0) {
13294 dtrace_dof_error(dof
, "DOF magic string mismatch");
13298 if (dof
->dofh_ident
[DOF_ID_MODEL
] != DOF_MODEL_ILP32
&&
13299 dof
->dofh_ident
[DOF_ID_MODEL
] != DOF_MODEL_LP64
) {
13300 dtrace_dof_error(dof
, "DOF has invalid data model");
13304 if (dof
->dofh_ident
[DOF_ID_ENCODING
] != DOF_ENCODE_NATIVE
) {
13305 dtrace_dof_error(dof
, "DOF encoding mismatch");
13309 if (dof
->dofh_ident
[DOF_ID_VERSION
] != DOF_VERSION_1
&&
13310 dof
->dofh_ident
[DOF_ID_VERSION
] != DOF_VERSION_2
) {
13311 dtrace_dof_error(dof
, "DOF version mismatch");
13315 if (dof
->dofh_ident
[DOF_ID_DIFVERS
] != DIF_VERSION_2
) {
13316 dtrace_dof_error(dof
, "DOF uses unsupported instruction set");
13320 if (dof
->dofh_ident
[DOF_ID_DIFIREG
] > DIF_DIR_NREGS
) {
13321 dtrace_dof_error(dof
, "DOF uses too many integer registers");
13325 if (dof
->dofh_ident
[DOF_ID_DIFTREG
] > DIF_DTR_NREGS
) {
13326 dtrace_dof_error(dof
, "DOF uses too many tuple registers");
13330 for (i
= DOF_ID_PAD
; i
< DOF_ID_SIZE
; i
++) {
13331 if (dof
->dofh_ident
[i
] != 0) {
13332 dtrace_dof_error(dof
, "DOF has invalid ident byte set");
13337 if (dof
->dofh_flags
& ~DOF_FL_VALID
) {
13338 dtrace_dof_error(dof
, "DOF has invalid flag bits set");
13342 if (dof
->dofh_secsize
== 0) {
13343 dtrace_dof_error(dof
, "zero section header size");
13348 * Check that the section headers don't exceed the amount of DOF
13349 * data. Note that we cast the section size and number of sections
13350 * to uint64_t's to prevent possible overflow in the multiplication.
13352 seclen
= (uint64_t)dof
->dofh_secnum
* (uint64_t)dof
->dofh_secsize
;
13354 if (dof
->dofh_secoff
> len
|| seclen
> len
||
13355 dof
->dofh_secoff
+ seclen
> len
) {
13356 dtrace_dof_error(dof
, "truncated section headers");
13360 if (!IS_P2ALIGNED(dof
->dofh_secoff
, sizeof (uint64_t))) {
13361 dtrace_dof_error(dof
, "misaligned section headers");
13365 if (!IS_P2ALIGNED(dof
->dofh_secsize
, sizeof (uint64_t))) {
13366 dtrace_dof_error(dof
, "misaligned section size");
13371 * Take an initial pass through the section headers to be sure that
13372 * the headers don't have stray offsets. If the 'noprobes' flag is
13373 * set, do not permit sections relating to providers, probes, or args.
13375 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
13376 dof_sec_t
*sec
= (dof_sec_t
*)(daddr
+
13377 (uintptr_t)dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
13380 switch (sec
->dofs_type
) {
13381 case DOF_SECT_PROVIDER
:
13382 case DOF_SECT_PROBES
:
13383 case DOF_SECT_PRARGS
:
13384 case DOF_SECT_PROFFS
:
13385 dtrace_dof_error(dof
, "illegal sections "
13391 if (DOF_SEC_ISLOADABLE(sec
->dofs_type
) &&
13392 !(sec
->dofs_flags
& DOF_SECF_LOAD
)) {
13393 dtrace_dof_error(dof
, "loadable section with load "
13398 if (!(sec
->dofs_flags
& DOF_SECF_LOAD
))
13399 continue; /* just ignore non-loadable sections */
13401 if (!ISP2(sec
->dofs_align
)) {
13402 dtrace_dof_error(dof
, "bad section alignment");
13406 if (sec
->dofs_offset
& (sec
->dofs_align
- 1)) {
13407 dtrace_dof_error(dof
, "misaligned section");
13411 if (sec
->dofs_offset
> len
|| sec
->dofs_size
> len
||
13412 sec
->dofs_offset
+ sec
->dofs_size
> len
) {
13413 dtrace_dof_error(dof
, "corrupt section header");
13417 if (sec
->dofs_type
== DOF_SECT_STRTAB
&& *((char *)daddr
+
13418 sec
->dofs_offset
+ sec
->dofs_size
- 1) != '\0') {
13419 dtrace_dof_error(dof
, "non-terminating string table");
13425 * Take a second pass through the sections and locate and perform any
13426 * relocations that are present. We do this after the first pass to
13427 * be sure that all sections have had their headers validated.
13429 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
13430 dof_sec_t
*sec
= (dof_sec_t
*)(daddr
+
13431 (uintptr_t)dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
13433 if (!(sec
->dofs_flags
& DOF_SECF_LOAD
))
13434 continue; /* skip sections that are not loadable */
13436 switch (sec
->dofs_type
) {
13437 case DOF_SECT_URELHDR
:
13438 if (dtrace_dof_relocate(dof
, sec
, ubase
) != 0)
13444 if ((enab
= *enabp
) == NULL
)
13445 enab
= *enabp
= dtrace_enabling_create(vstate
);
13447 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
13448 dof_sec_t
*sec
= (dof_sec_t
*)(daddr
+
13449 (uintptr_t)dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
13451 if (sec
->dofs_type
!= DOF_SECT_ECBDESC
)
13454 if ((ep
= dtrace_dof_ecbdesc(dof
, sec
, vstate
, cr
)) == NULL
) {
13455 dtrace_enabling_destroy(enab
);
13460 dtrace_enabling_add(enab
, ep
);
13467 * Process DOF for any options. This routine assumes that the DOF has been
13468 * at least processed by dtrace_dof_slurp().
13471 dtrace_dof_options(dof_hdr_t
*dof
, dtrace_state_t
*state
)
13476 dof_optdesc_t
*desc
;
13478 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
13479 dof_sec_t
*sec
= (dof_sec_t
*)((uintptr_t)dof
+
13480 (uintptr_t)dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
13482 if (sec
->dofs_type
!= DOF_SECT_OPTDESC
)
13485 if (sec
->dofs_align
!= sizeof (uint64_t)) {
13486 dtrace_dof_error(dof
, "bad alignment in "
13487 "option description");
13491 if ((entsize
= sec
->dofs_entsize
) == 0) {
13492 dtrace_dof_error(dof
, "zeroed option entry size");
13496 if (entsize
< sizeof (dof_optdesc_t
)) {
13497 dtrace_dof_error(dof
, "bad option entry size");
13501 for (offs
= 0; offs
< sec
->dofs_size
; offs
+= entsize
) {
13502 desc
= (dof_optdesc_t
*)((uintptr_t)dof
+
13503 (uintptr_t)sec
->dofs_offset
+ offs
);
13505 if (desc
->dofo_strtab
!= DOF_SECIDX_NONE
) {
13506 dtrace_dof_error(dof
, "non-zero option string");
13510 if (desc
->dofo_value
== DTRACEOPT_UNSET
) {
13511 dtrace_dof_error(dof
, "unset option");
13515 if ((rval
= dtrace_state_option(state
,
13516 desc
->dofo_option
, desc
->dofo_value
)) != 0) {
13517 dtrace_dof_error(dof
, "rejected option");
13527 * DTrace Consumer State Functions
13530 dtrace_dstate_init(dtrace_dstate_t
*dstate
, size_t size
)
13532 size_t hashsize
, maxper
, min
, chunksize
= dstate
->dtds_chunksize
;
13535 dtrace_dynvar_t
*dvar
, *next
, *start
;
13538 ASSERT(MUTEX_HELD(&dtrace_lock
));
13539 ASSERT(dstate
->dtds_base
== NULL
&& dstate
->dtds_percpu
== NULL
);
13541 bzero(dstate
, sizeof (dtrace_dstate_t
));
13543 if ((dstate
->dtds_chunksize
= chunksize
) == 0)
13544 dstate
->dtds_chunksize
= DTRACE_DYNVAR_CHUNKSIZE
;
13546 VERIFY(dstate
->dtds_chunksize
< LONG_MAX
);
13548 if (size
< (min
= dstate
->dtds_chunksize
+ sizeof (dtrace_dynhash_t
)))
13551 if ((base
= kmem_zalloc(size
, KM_NOSLEEP
| KM_NORMALPRI
)) == NULL
)
13554 dstate
->dtds_size
= size
;
13555 dstate
->dtds_base
= base
;
13556 dstate
->dtds_percpu
= kmem_cache_alloc(dtrace_state_cache
, KM_SLEEP
);
13557 bzero(dstate
->dtds_percpu
, NCPU
* sizeof (dtrace_dstate_percpu_t
));
13559 hashsize
= size
/ (dstate
->dtds_chunksize
+ sizeof (dtrace_dynhash_t
));
13561 if (hashsize
!= 1 && (hashsize
& 1))
13564 dstate
->dtds_hashsize
= hashsize
;
13565 dstate
->dtds_hash
= dstate
->dtds_base
;
13568 * Set all of our hash buckets to point to the single sink, and (if
13569 * it hasn't already been set), set the sink's hash value to be the
13570 * sink sentinel value. The sink is needed for dynamic variable
13571 * lookups to know that they have iterated over an entire, valid hash
13574 for (i
= 0; i
< hashsize
; i
++)
13575 dstate
->dtds_hash
[i
].dtdh_chain
= &dtrace_dynhash_sink
;
13577 if (dtrace_dynhash_sink
.dtdv_hashval
!= DTRACE_DYNHASH_SINK
)
13578 dtrace_dynhash_sink
.dtdv_hashval
= DTRACE_DYNHASH_SINK
;
13581 * Determine number of active CPUs. Divide free list evenly among
13584 start
= (dtrace_dynvar_t
*)
13585 ((uintptr_t)base
+ hashsize
* sizeof (dtrace_dynhash_t
));
13586 limit
= (uintptr_t)base
+ size
;
13588 VERIFY((uintptr_t)start
< limit
);
13589 VERIFY((uintptr_t)start
>= (uintptr_t)base
);
13591 maxper
= (limit
- (uintptr_t)start
) / NCPU
;
13592 maxper
= (maxper
/ dstate
->dtds_chunksize
) * dstate
->dtds_chunksize
;
13594 for (i
= 0; i
< NCPU
; i
++) {
13595 dstate
->dtds_percpu
[i
].dtdsc_free
= dvar
= start
;
13598 * If we don't even have enough chunks to make it once through
13599 * NCPUs, we're just going to allocate everything to the first
13600 * CPU. And if we're on the last CPU, we're going to allocate
13601 * whatever is left over. In either case, we set the limit to
13602 * be the limit of the dynamic variable space.
13604 if (maxper
== 0 || i
== NCPU
- 1) {
13605 limit
= (uintptr_t)base
+ size
;
13608 limit
= (uintptr_t)start
+ maxper
;
13609 start
= (dtrace_dynvar_t
*)limit
;
13612 VERIFY(limit
<= (uintptr_t)base
+ size
);
13615 next
= (dtrace_dynvar_t
*)((uintptr_t)dvar
+
13616 dstate
->dtds_chunksize
);
13618 if ((uintptr_t)next
+ dstate
->dtds_chunksize
>= limit
)
13621 VERIFY((uintptr_t)dvar
>= (uintptr_t)base
&&
13622 (uintptr_t)dvar
<= (uintptr_t)base
+ size
);
13623 dvar
->dtdv_next
= next
;
13635 dtrace_dstate_fini(dtrace_dstate_t
*dstate
)
13637 ASSERT(MUTEX_HELD(&cpu_lock
));
13639 if (dstate
->dtds_base
== NULL
)
13642 kmem_free(dstate
->dtds_base
, dstate
->dtds_size
);
13643 kmem_cache_free(dtrace_state_cache
, dstate
->dtds_percpu
);
13647 dtrace_vstate_fini(dtrace_vstate_t
*vstate
)
13650 * Logical XOR, where are you?
13652 ASSERT((vstate
->dtvs_nglobals
== 0) ^ (vstate
->dtvs_globals
!= NULL
));
13654 if (vstate
->dtvs_nglobals
> 0) {
13655 kmem_free(vstate
->dtvs_globals
, vstate
->dtvs_nglobals
*
13656 sizeof (dtrace_statvar_t
*));
13659 if (vstate
->dtvs_ntlocals
> 0) {
13660 kmem_free(vstate
->dtvs_tlocals
, vstate
->dtvs_ntlocals
*
13661 sizeof (dtrace_difv_t
));
13664 ASSERT((vstate
->dtvs_nlocals
== 0) ^ (vstate
->dtvs_locals
!= NULL
));
13666 if (vstate
->dtvs_nlocals
> 0) {
13667 kmem_free(vstate
->dtvs_locals
, vstate
->dtvs_nlocals
*
13668 sizeof (dtrace_statvar_t
*));
13673 dtrace_state_clean(dtrace_state_t
*state
)
13675 if (state
->dts_activity
== DTRACE_ACTIVITY_INACTIVE
)
13678 dtrace_dynvar_clean(&state
->dts_vstate
.dtvs_dynvars
);
13679 dtrace_speculation_clean(state
);
13683 dtrace_state_deadman(dtrace_state_t
*state
)
13689 now
= dtrace_gethrtime();
13691 if (state
!= dtrace_anon
.dta_state
&&
13692 now
- state
->dts_laststatus
>= dtrace_deadman_user
)
13696 * We must be sure that dts_alive never appears to be less than the
13697 * value upon entry to dtrace_state_deadman(), and because we lack a
13698 * dtrace_cas64(), we cannot store to it atomically. We thus instead
13699 * store INT64_MAX to it, followed by a memory barrier, followed by
13700 * the new value. This assures that dts_alive never appears to be
13701 * less than its true value, regardless of the order in which the
13702 * stores to the underlying storage are issued.
13704 state
->dts_alive
= INT64_MAX
;
13705 dtrace_membar_producer();
13706 state
->dts_alive
= now
;
13710 dtrace_state_create(dev_t
*devp
, cred_t
*cr
)
13715 dtrace_state_t
*state
;
13716 dtrace_optval_t
*opt
;
13717 int bufsize
= NCPU
* sizeof (dtrace_buffer_t
), i
;
13719 ASSERT(MUTEX_HELD(&dtrace_lock
));
13720 ASSERT(MUTEX_HELD(&cpu_lock
));
13722 minor
= (minor_t
)(uintptr_t)vmem_alloc(dtrace_minor
, 1,
13723 VM_BESTFIT
| VM_SLEEP
);
13725 if (ddi_soft_state_zalloc(dtrace_softstate
, minor
) != DDI_SUCCESS
) {
13726 vmem_free(dtrace_minor
, (void *)(uintptr_t)minor
, 1);
13730 state
= ddi_get_soft_state(dtrace_softstate
, minor
);
13731 state
->dts_epid
= DTRACE_EPIDNONE
+ 1;
13733 (void) snprintf(c
, sizeof (c
), "dtrace_aggid_%d", minor
);
13734 state
->dts_aggid_arena
= vmem_create(c
, (void *)1, UINT32_MAX
, 1,
13735 NULL
, NULL
, NULL
, 0, VM_SLEEP
| VMC_IDENTIFIER
);
13737 if (devp
!= NULL
) {
13738 major
= getemajor(*devp
);
13740 major
= ddi_driver_major(dtrace_devi
);
13743 state
->dts_dev
= makedevice(major
, minor
);
13746 *devp
= state
->dts_dev
;
13749 * We allocate NCPU buffers. On the one hand, this can be quite
13750 * a bit of memory per instance (nearly 36K on a Starcat). On the
13751 * other hand, it saves an additional memory reference in the probe
13754 state
->dts_buffer
= kmem_zalloc(bufsize
, KM_SLEEP
);
13755 state
->dts_aggbuffer
= kmem_zalloc(bufsize
, KM_SLEEP
);
13756 state
->dts_cleaner
= CYCLIC_NONE
;
13757 state
->dts_deadman
= CYCLIC_NONE
;
13758 state
->dts_vstate
.dtvs_state
= state
;
13760 for (i
= 0; i
< DTRACEOPT_MAX
; i
++)
13761 state
->dts_options
[i
] = DTRACEOPT_UNSET
;
13764 * Set the default options.
13766 opt
= state
->dts_options
;
13767 opt
[DTRACEOPT_BUFPOLICY
] = DTRACEOPT_BUFPOLICY_SWITCH
;
13768 opt
[DTRACEOPT_BUFRESIZE
] = DTRACEOPT_BUFRESIZE_AUTO
;
13769 opt
[DTRACEOPT_NSPEC
] = dtrace_nspec_default
;
13770 opt
[DTRACEOPT_SPECSIZE
] = dtrace_specsize_default
;
13771 opt
[DTRACEOPT_CPU
] = (dtrace_optval_t
)DTRACE_CPUALL
;
13772 opt
[DTRACEOPT_STRSIZE
] = dtrace_strsize_default
;
13773 opt
[DTRACEOPT_STACKFRAMES
] = dtrace_stackframes_default
;
13774 opt
[DTRACEOPT_USTACKFRAMES
] = dtrace_ustackframes_default
;
13775 opt
[DTRACEOPT_CLEANRATE
] = dtrace_cleanrate_default
;
13776 opt
[DTRACEOPT_AGGRATE
] = dtrace_aggrate_default
;
13777 opt
[DTRACEOPT_SWITCHRATE
] = dtrace_switchrate_default
;
13778 opt
[DTRACEOPT_STATUSRATE
] = dtrace_statusrate_default
;
13779 opt
[DTRACEOPT_JSTACKFRAMES
] = dtrace_jstackframes_default
;
13780 opt
[DTRACEOPT_JSTACKSTRSIZE
] = dtrace_jstackstrsize_default
;
13782 state
->dts_activity
= DTRACE_ACTIVITY_INACTIVE
;
13785 * Depending on the user credentials, we set flag bits which alter probe
13786 * visibility or the amount of destructiveness allowed. In the case of
13787 * actual anonymous tracing, or the possession of all privileges, all of
13788 * the normal checks are bypassed.
13790 if (cr
== NULL
|| PRIV_POLICY_ONLY(cr
, PRIV_ALL
, B_FALSE
)) {
13791 state
->dts_cred
.dcr_visible
= DTRACE_CRV_ALL
;
13792 state
->dts_cred
.dcr_action
= DTRACE_CRA_ALL
;
13795 * Set up the credentials for this instantiation. We take a
13796 * hold on the credential to prevent it from disappearing on
13797 * us; this in turn prevents the zone_t referenced by this
13798 * credential from disappearing. This means that we can
13799 * examine the credential and the zone from probe context.
13802 state
->dts_cred
.dcr_cred
= cr
;
13805 * CRA_PROC means "we have *some* privilege for dtrace" and
13806 * unlocks the use of variables like pid, zonename, etc.
13808 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_USER
, B_FALSE
) ||
13809 PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_PROC
, B_FALSE
)) {
13810 state
->dts_cred
.dcr_action
|= DTRACE_CRA_PROC
;
13814 * dtrace_user allows use of syscall and profile providers.
13815 * If the user also has proc_owner and/or proc_zone, we
13816 * extend the scope to include additional visibility and
13817 * destructive power.
13819 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_USER
, B_FALSE
)) {
13820 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_OWNER
, B_FALSE
)) {
13821 state
->dts_cred
.dcr_visible
|=
13822 DTRACE_CRV_ALLPROC
;
13824 state
->dts_cred
.dcr_action
|=
13825 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER
;
13828 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_ZONE
, B_FALSE
)) {
13829 state
->dts_cred
.dcr_visible
|=
13830 DTRACE_CRV_ALLZONE
;
13832 state
->dts_cred
.dcr_action
|=
13833 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE
;
13837 * If we have all privs in whatever zone this is,
13838 * we can do destructive things to processes which
13839 * have altered credentials.
13841 if (priv_isequalset(priv_getset(cr
, PRIV_EFFECTIVE
),
13842 cr
->cr_zone
->zone_privset
)) {
13843 state
->dts_cred
.dcr_action
|=
13844 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG
;
13849 * Holding the dtrace_kernel privilege also implies that
13850 * the user has the dtrace_user privilege from a visibility
13851 * perspective. But without further privileges, some
13852 * destructive actions are not available.
13854 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_KERNEL
, B_FALSE
)) {
13856 * Make all probes in all zones visible. However,
13857 * this doesn't mean that all actions become available
13860 state
->dts_cred
.dcr_visible
|= DTRACE_CRV_KERNEL
|
13861 DTRACE_CRV_ALLPROC
| DTRACE_CRV_ALLZONE
;
13863 state
->dts_cred
.dcr_action
|= DTRACE_CRA_KERNEL
|
13866 * Holding proc_owner means that destructive actions
13867 * for *this* zone are allowed.
13869 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_OWNER
, B_FALSE
))
13870 state
->dts_cred
.dcr_action
|=
13871 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER
;
13874 * Holding proc_zone means that destructive actions
13875 * for this user/group ID in all zones is allowed.
13877 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_ZONE
, B_FALSE
))
13878 state
->dts_cred
.dcr_action
|=
13879 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE
;
13882 * If we have all privs in whatever zone this is,
13883 * we can do destructive things to processes which
13884 * have altered credentials.
13886 if (priv_isequalset(priv_getset(cr
, PRIV_EFFECTIVE
),
13887 cr
->cr_zone
->zone_privset
)) {
13888 state
->dts_cred
.dcr_action
|=
13889 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG
;
13894 * Holding the dtrace_proc privilege gives control over fasttrap
13895 * and pid providers. We need to grant wider destructive
13896 * privileges in the event that the user has proc_owner and/or
13899 if (PRIV_POLICY_ONLY(cr
, PRIV_DTRACE_PROC
, B_FALSE
)) {
13900 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_OWNER
, B_FALSE
))
13901 state
->dts_cred
.dcr_action
|=
13902 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER
;
13904 if (PRIV_POLICY_ONLY(cr
, PRIV_PROC_ZONE
, B_FALSE
))
13905 state
->dts_cred
.dcr_action
|=
13906 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE
;
13914 dtrace_state_buffer(dtrace_state_t
*state
, dtrace_buffer_t
*buf
, int which
)
13916 dtrace_optval_t
*opt
= state
->dts_options
, size
;
13918 int flags
= 0, rval
, factor
, divisor
= 1;
13920 ASSERT(MUTEX_HELD(&dtrace_lock
));
13921 ASSERT(MUTEX_HELD(&cpu_lock
));
13922 ASSERT(which
< DTRACEOPT_MAX
);
13923 ASSERT(state
->dts_activity
== DTRACE_ACTIVITY_INACTIVE
||
13924 (state
== dtrace_anon
.dta_state
&&
13925 state
->dts_activity
== DTRACE_ACTIVITY_ACTIVE
));
13927 if (opt
[which
] == DTRACEOPT_UNSET
|| opt
[which
] == 0)
13930 if (opt
[DTRACEOPT_CPU
] != DTRACEOPT_UNSET
)
13931 cpu
= opt
[DTRACEOPT_CPU
];
13933 if (which
== DTRACEOPT_SPECSIZE
)
13934 flags
|= DTRACEBUF_NOSWITCH
;
13936 if (which
== DTRACEOPT_BUFSIZE
) {
13937 if (opt
[DTRACEOPT_BUFPOLICY
] == DTRACEOPT_BUFPOLICY_RING
)
13938 flags
|= DTRACEBUF_RING
;
13940 if (opt
[DTRACEOPT_BUFPOLICY
] == DTRACEOPT_BUFPOLICY_FILL
)
13941 flags
|= DTRACEBUF_FILL
;
13943 if (state
!= dtrace_anon
.dta_state
||
13944 state
->dts_activity
!= DTRACE_ACTIVITY_ACTIVE
)
13945 flags
|= DTRACEBUF_INACTIVE
;
13948 for (size
= opt
[which
]; size
>= sizeof (uint64_t); size
/= divisor
) {
13950 * The size must be 8-byte aligned. If the size is not 8-byte
13951 * aligned, drop it down by the difference.
13953 if (size
& (sizeof (uint64_t) - 1))
13954 size
-= size
& (sizeof (uint64_t) - 1);
13956 if (size
< state
->dts_reserve
) {
13958 * Buffers always must be large enough to accommodate
13959 * their prereserved space. We return E2BIG instead
13960 * of ENOMEM in this case to allow for user-level
13961 * software to differentiate the cases.
13966 rval
= dtrace_buffer_alloc(buf
, size
, flags
, cpu
, &factor
);
13968 if (rval
!= ENOMEM
) {
13973 if (opt
[DTRACEOPT_BUFRESIZE
] == DTRACEOPT_BUFRESIZE_MANUAL
)
13976 for (divisor
= 2; divisor
< factor
; divisor
<<= 1)
13984 dtrace_state_buffers(dtrace_state_t
*state
)
13986 dtrace_speculation_t
*spec
= state
->dts_speculations
;
13989 if ((rval
= dtrace_state_buffer(state
, state
->dts_buffer
,
13990 DTRACEOPT_BUFSIZE
)) != 0)
13993 if ((rval
= dtrace_state_buffer(state
, state
->dts_aggbuffer
,
13994 DTRACEOPT_AGGSIZE
)) != 0)
13997 for (i
= 0; i
< state
->dts_nspeculations
; i
++) {
13998 if ((rval
= dtrace_state_buffer(state
,
13999 spec
[i
].dtsp_buffer
, DTRACEOPT_SPECSIZE
)) != 0)
14007 dtrace_state_prereserve(dtrace_state_t
*state
)
14010 dtrace_probe_t
*probe
;
14012 state
->dts_reserve
= 0;
14014 if (state
->dts_options
[DTRACEOPT_BUFPOLICY
] != DTRACEOPT_BUFPOLICY_FILL
)
14018 * If our buffer policy is a "fill" buffer policy, we need to set the
14019 * prereserved space to be the space required by the END probes.
14021 probe
= dtrace_probes
[dtrace_probeid_end
- 1];
14022 ASSERT(probe
!= NULL
);
14024 for (ecb
= probe
->dtpr_ecb
; ecb
!= NULL
; ecb
= ecb
->dte_next
) {
14025 if (ecb
->dte_state
!= state
)
14028 state
->dts_reserve
+= ecb
->dte_needed
+ ecb
->dte_alignment
;
14033 dtrace_state_go(dtrace_state_t
*state
, processorid_t
*cpu
)
14035 dtrace_optval_t
*opt
= state
->dts_options
, sz
, nspec
;
14036 dtrace_speculation_t
*spec
;
14037 dtrace_buffer_t
*buf
;
14038 cyc_handler_t hdlr
;
14040 int rval
= 0, i
, bufsize
= NCPU
* sizeof (dtrace_buffer_t
);
14041 dtrace_icookie_t cookie
;
14043 mutex_enter(&cpu_lock
);
14044 mutex_enter(&dtrace_lock
);
14046 if (state
->dts_activity
!= DTRACE_ACTIVITY_INACTIVE
) {
14052 * Before we can perform any checks, we must prime all of the
14053 * retained enablings that correspond to this state.
14055 dtrace_enabling_prime(state
);
14057 if (state
->dts_destructive
&& !state
->dts_cred
.dcr_destructive
) {
14062 dtrace_state_prereserve(state
);
14065 * Now we want to do is try to allocate our speculations.
14066 * We do not automatically resize the number of speculations; if
14067 * this fails, we will fail the operation.
14069 nspec
= opt
[DTRACEOPT_NSPEC
];
14070 ASSERT(nspec
!= DTRACEOPT_UNSET
);
14072 if (nspec
> INT_MAX
) {
14077 spec
= kmem_zalloc(nspec
* sizeof (dtrace_speculation_t
),
14078 KM_NOSLEEP
| KM_NORMALPRI
);
14080 if (spec
== NULL
) {
14085 state
->dts_speculations
= spec
;
14086 state
->dts_nspeculations
= (int)nspec
;
14088 for (i
= 0; i
< nspec
; i
++) {
14089 if ((buf
= kmem_zalloc(bufsize
,
14090 KM_NOSLEEP
| KM_NORMALPRI
)) == NULL
) {
14095 spec
[i
].dtsp_buffer
= buf
;
14098 if (opt
[DTRACEOPT_GRABANON
] != DTRACEOPT_UNSET
) {
14099 if (dtrace_anon
.dta_state
== NULL
) {
14104 if (state
->dts_necbs
!= 0) {
14109 state
->dts_anon
= dtrace_anon_grab();
14110 ASSERT(state
->dts_anon
!= NULL
);
14111 state
= state
->dts_anon
;
14114 * We want "grabanon" to be set in the grabbed state, so we'll
14115 * copy that option value from the grabbing state into the
14118 state
->dts_options
[DTRACEOPT_GRABANON
] =
14119 opt
[DTRACEOPT_GRABANON
];
14121 *cpu
= dtrace_anon
.dta_beganon
;
14124 * If the anonymous state is active (as it almost certainly
14125 * is if the anonymous enabling ultimately matched anything),
14126 * we don't allow any further option processing -- but we
14127 * don't return failure.
14129 if (state
->dts_activity
!= DTRACE_ACTIVITY_INACTIVE
)
14133 if (opt
[DTRACEOPT_AGGSIZE
] != DTRACEOPT_UNSET
&&
14134 opt
[DTRACEOPT_AGGSIZE
] != 0) {
14135 if (state
->dts_aggregations
== NULL
) {
14137 * We're not going to create an aggregation buffer
14138 * because we don't have any ECBs that contain
14139 * aggregations -- set this option to 0.
14141 opt
[DTRACEOPT_AGGSIZE
] = 0;
14144 * If we have an aggregation buffer, we must also have
14145 * a buffer to use as scratch.
14147 if (opt
[DTRACEOPT_BUFSIZE
] == DTRACEOPT_UNSET
||
14148 opt
[DTRACEOPT_BUFSIZE
] < state
->dts_needed
) {
14149 opt
[DTRACEOPT_BUFSIZE
] = state
->dts_needed
;
14154 if (opt
[DTRACEOPT_SPECSIZE
] != DTRACEOPT_UNSET
&&
14155 opt
[DTRACEOPT_SPECSIZE
] != 0) {
14156 if (!state
->dts_speculates
) {
14158 * We're not going to create speculation buffers
14159 * because we don't have any ECBs that actually
14160 * speculate -- set the speculation size to 0.
14162 opt
[DTRACEOPT_SPECSIZE
] = 0;
14167 * The bare minimum size for any buffer that we're actually going to
14168 * do anything to is sizeof (uint64_t).
14170 sz
= sizeof (uint64_t);
14172 if ((state
->dts_needed
!= 0 && opt
[DTRACEOPT_BUFSIZE
] < sz
) ||
14173 (state
->dts_speculates
&& opt
[DTRACEOPT_SPECSIZE
] < sz
) ||
14174 (state
->dts_aggregations
!= NULL
&& opt
[DTRACEOPT_AGGSIZE
] < sz
)) {
14176 * A buffer size has been explicitly set to 0 (or to a size
14177 * that will be adjusted to 0) and we need the space -- we
14178 * need to return failure. We return ENOSPC to differentiate
14179 * it from failing to allocate a buffer due to failure to meet
14180 * the reserve (for which we return E2BIG).
14186 if ((rval
= dtrace_state_buffers(state
)) != 0)
14189 if ((sz
= opt
[DTRACEOPT_DYNVARSIZE
]) == DTRACEOPT_UNSET
)
14190 sz
= dtrace_dstate_defsize
;
14193 rval
= dtrace_dstate_init(&state
->dts_vstate
.dtvs_dynvars
, sz
);
14198 if (opt
[DTRACEOPT_BUFRESIZE
] == DTRACEOPT_BUFRESIZE_MANUAL
)
14200 } while (sz
>>= 1);
14202 opt
[DTRACEOPT_DYNVARSIZE
] = sz
;
14207 if (opt
[DTRACEOPT_STATUSRATE
] > dtrace_statusrate_max
)
14208 opt
[DTRACEOPT_STATUSRATE
] = dtrace_statusrate_max
;
14210 if (opt
[DTRACEOPT_CLEANRATE
] == 0)
14211 opt
[DTRACEOPT_CLEANRATE
] = dtrace_cleanrate_max
;
14213 if (opt
[DTRACEOPT_CLEANRATE
] < dtrace_cleanrate_min
)
14214 opt
[DTRACEOPT_CLEANRATE
] = dtrace_cleanrate_min
;
14216 if (opt
[DTRACEOPT_CLEANRATE
] > dtrace_cleanrate_max
)
14217 opt
[DTRACEOPT_CLEANRATE
] = dtrace_cleanrate_max
;
14219 hdlr
.cyh_func
= (cyc_func_t
)dtrace_state_clean
;
14220 hdlr
.cyh_arg
= state
;
14221 hdlr
.cyh_level
= CY_LOW_LEVEL
;
14224 when
.cyt_interval
= opt
[DTRACEOPT_CLEANRATE
];
14226 state
->dts_cleaner
= cyclic_add(&hdlr
, &when
);
14228 hdlr
.cyh_func
= (cyc_func_t
)dtrace_state_deadman
;
14229 hdlr
.cyh_arg
= state
;
14230 hdlr
.cyh_level
= CY_LOW_LEVEL
;
14233 when
.cyt_interval
= dtrace_deadman_interval
;
14235 state
->dts_alive
= state
->dts_laststatus
= dtrace_gethrtime();
14236 state
->dts_deadman
= cyclic_add(&hdlr
, &when
);
14238 state
->dts_activity
= DTRACE_ACTIVITY_WARMUP
;
14240 if (state
->dts_getf
!= 0 &&
14241 !(state
->dts_cred
.dcr_visible
& DTRACE_CRV_KERNEL
)) {
14243 * We don't have kernel privs but we have at least one call
14244 * to getf(); we need to bump our zone's count, and (if
14245 * this is the first enabling to have an unprivileged call
14246 * to getf()) we need to hook into closef().
14248 state
->dts_cred
.dcr_cred
->cr_zone
->zone_dtrace_getf
++;
14250 if (dtrace_getf
++ == 0) {
14251 ASSERT(dtrace_closef
== NULL
);
14252 dtrace_closef
= dtrace_getf_barrier
;
14257 * Now it's time to actually fire the BEGIN probe. We need to disable
14258 * interrupts here both to record the CPU on which we fired the BEGIN
14259 * probe (the data from this CPU will be processed first at user
14260 * level) and to manually activate the buffer for this CPU.
14262 cookie
= dtrace_interrupt_disable();
14263 *cpu
= CPU
->cpu_id
;
14264 ASSERT(state
->dts_buffer
[*cpu
].dtb_flags
& DTRACEBUF_INACTIVE
);
14265 state
->dts_buffer
[*cpu
].dtb_flags
&= ~DTRACEBUF_INACTIVE
;
14267 dtrace_probe(dtrace_probeid_begin
,
14268 (uint64_t)(uintptr_t)state
, 0, 0, 0, 0);
14269 dtrace_interrupt_enable(cookie
);
14271 * We may have had an exit action from a BEGIN probe; only change our
14272 * state to ACTIVE if we're still in WARMUP.
14274 ASSERT(state
->dts_activity
== DTRACE_ACTIVITY_WARMUP
||
14275 state
->dts_activity
== DTRACE_ACTIVITY_DRAINING
);
14277 if (state
->dts_activity
== DTRACE_ACTIVITY_WARMUP
)
14278 state
->dts_activity
= DTRACE_ACTIVITY_ACTIVE
;
14281 * Regardless of whether or not now we're in ACTIVE or DRAINING, we
14282 * want each CPU to transition its principal buffer out of the
14283 * INACTIVE state. Doing this assures that no CPU will suddenly begin
14284 * processing an ECB halfway down a probe's ECB chain; all CPUs will
14285 * atomically transition from processing none of a state's ECBs to
14286 * processing all of them.
14288 dtrace_xcall(DTRACE_CPUALL
,
14289 (dtrace_xcall_t
)dtrace_buffer_activate
, state
);
14293 dtrace_buffer_free(state
->dts_buffer
);
14294 dtrace_buffer_free(state
->dts_aggbuffer
);
14296 if ((nspec
= state
->dts_nspeculations
) == 0) {
14297 ASSERT(state
->dts_speculations
== NULL
);
14301 spec
= state
->dts_speculations
;
14302 ASSERT(spec
!= NULL
);
14304 for (i
= 0; i
< state
->dts_nspeculations
; i
++) {
14305 if ((buf
= spec
[i
].dtsp_buffer
) == NULL
)
14308 dtrace_buffer_free(buf
);
14309 kmem_free(buf
, bufsize
);
14312 kmem_free(spec
, nspec
* sizeof (dtrace_speculation_t
));
14313 state
->dts_nspeculations
= 0;
14314 state
->dts_speculations
= NULL
;
14317 mutex_exit(&dtrace_lock
);
14318 mutex_exit(&cpu_lock
);
14324 dtrace_state_stop(dtrace_state_t
*state
, processorid_t
*cpu
)
14326 dtrace_icookie_t cookie
;
14328 ASSERT(MUTEX_HELD(&dtrace_lock
));
14330 if (state
->dts_activity
!= DTRACE_ACTIVITY_ACTIVE
&&
14331 state
->dts_activity
!= DTRACE_ACTIVITY_DRAINING
)
14335 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
14336 * to be sure that every CPU has seen it. See below for the details
14337 * on why this is done.
14339 state
->dts_activity
= DTRACE_ACTIVITY_DRAINING
;
14343 * By this point, it is impossible for any CPU to be still processing
14344 * with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to
14345 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
14346 * other CPU in dtrace_buffer_reserve(). This allows dtrace_probe()
14347 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
14348 * iff we're in the END probe.
14350 state
->dts_activity
= DTRACE_ACTIVITY_COOLDOWN
;
14352 ASSERT(state
->dts_activity
== DTRACE_ACTIVITY_COOLDOWN
);
14355 * Finally, we can release the reserve and call the END probe. We
14356 * disable interrupts across calling the END probe to allow us to
14357 * return the CPU on which we actually called the END probe. This
14358 * allows user-land to be sure that this CPU's principal buffer is
14361 state
->dts_reserve
= 0;
14363 cookie
= dtrace_interrupt_disable();
14364 *cpu
= CPU
->cpu_id
;
14365 dtrace_probe(dtrace_probeid_end
,
14366 (uint64_t)(uintptr_t)state
, 0, 0, 0, 0);
14367 dtrace_interrupt_enable(cookie
);
14369 state
->dts_activity
= DTRACE_ACTIVITY_STOPPED
;
14372 if (state
->dts_getf
!= 0 &&
14373 !(state
->dts_cred
.dcr_visible
& DTRACE_CRV_KERNEL
)) {
14375 * We don't have kernel privs but we have at least one call
14376 * to getf(); we need to lower our zone's count, and (if
14377 * this is the last enabling to have an unprivileged call
14378 * to getf()) we need to clear the closef() hook.
14380 ASSERT(state
->dts_cred
.dcr_cred
->cr_zone
->zone_dtrace_getf
> 0);
14381 ASSERT(dtrace_closef
== dtrace_getf_barrier
);
14382 ASSERT(dtrace_getf
> 0);
14384 state
->dts_cred
.dcr_cred
->cr_zone
->zone_dtrace_getf
--;
14386 if (--dtrace_getf
== 0)
14387 dtrace_closef
= NULL
;
14394 dtrace_state_option(dtrace_state_t
*state
, dtrace_optid_t option
,
14395 dtrace_optval_t val
)
14397 ASSERT(MUTEX_HELD(&dtrace_lock
));
14399 if (state
->dts_activity
!= DTRACE_ACTIVITY_INACTIVE
)
14402 if (option
>= DTRACEOPT_MAX
)
14405 if (option
!= DTRACEOPT_CPU
&& val
< 0)
14409 case DTRACEOPT_DESTRUCTIVE
:
14410 if (dtrace_destructive_disallow
)
14413 state
->dts_cred
.dcr_destructive
= 1;
14416 case DTRACEOPT_BUFSIZE
:
14417 case DTRACEOPT_DYNVARSIZE
:
14418 case DTRACEOPT_AGGSIZE
:
14419 case DTRACEOPT_SPECSIZE
:
14420 case DTRACEOPT_STRSIZE
:
14424 if (val
>= LONG_MAX
) {
14426 * If this is an otherwise negative value, set it to
14427 * the highest multiple of 128m less than LONG_MAX.
14428 * Technically, we're adjusting the size without
14429 * regard to the buffer resizing policy, but in fact,
14430 * this has no effect -- if we set the buffer size to
14431 * ~LONG_MAX and the buffer policy is ultimately set to
14432 * be "manual", the buffer allocation is guaranteed to
14433 * fail, if only because the allocation requires two
14434 * buffers. (We set the the size to the highest
14435 * multiple of 128m because it ensures that the size
14436 * will remain a multiple of a megabyte when
14437 * repeatedly halved -- all the way down to 15m.)
14439 val
= LONG_MAX
- (1 << 27) + 1;
14443 state
->dts_options
[option
] = val
;
14449 dtrace_state_destroy(dtrace_state_t
*state
)
14452 dtrace_vstate_t
*vstate
= &state
->dts_vstate
;
14453 minor_t minor
= getminor(state
->dts_dev
);
14454 int i
, bufsize
= NCPU
* sizeof (dtrace_buffer_t
);
14455 dtrace_speculation_t
*spec
= state
->dts_speculations
;
14456 int nspec
= state
->dts_nspeculations
;
14459 ASSERT(MUTEX_HELD(&dtrace_lock
));
14460 ASSERT(MUTEX_HELD(&cpu_lock
));
14463 * First, retract any retained enablings for this state.
14465 dtrace_enabling_retract(state
);
14466 ASSERT(state
->dts_nretained
== 0);
14468 if (state
->dts_activity
== DTRACE_ACTIVITY_ACTIVE
||
14469 state
->dts_activity
== DTRACE_ACTIVITY_DRAINING
) {
14471 * We have managed to come into dtrace_state_destroy() on a
14472 * hot enabling -- almost certainly because of a disorderly
14473 * shutdown of a consumer. (That is, a consumer that is
14474 * exiting without having called dtrace_stop().) In this case,
14475 * we're going to set our activity to be KILLED, and then
14476 * issue a sync to be sure that everyone is out of probe
14477 * context before we start blowing away ECBs.
14479 state
->dts_activity
= DTRACE_ACTIVITY_KILLED
;
14484 * Release the credential hold we took in dtrace_state_create().
14486 if (state
->dts_cred
.dcr_cred
!= NULL
)
14487 crfree(state
->dts_cred
.dcr_cred
);
14490 * Now we can safely disable and destroy any enabled probes. Because
14491 * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
14492 * (especially if they're all enabled), we take two passes through the
14493 * ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and
14494 * in the second we disable whatever is left over.
14496 for (match
= DTRACE_PRIV_KERNEL
; ; match
= 0) {
14497 for (i
= 0; i
< state
->dts_necbs
; i
++) {
14498 if ((ecb
= state
->dts_ecbs
[i
]) == NULL
)
14501 if (match
&& ecb
->dte_probe
!= NULL
) {
14502 dtrace_probe_t
*probe
= ecb
->dte_probe
;
14503 dtrace_provider_t
*prov
= probe
->dtpr_provider
;
14505 if (!(prov
->dtpv_priv
.dtpp_flags
& match
))
14509 dtrace_ecb_disable(ecb
);
14510 dtrace_ecb_destroy(ecb
);
14518 * Before we free the buffers, perform one more sync to assure that
14519 * every CPU is out of probe context.
14523 dtrace_buffer_free(state
->dts_buffer
);
14524 dtrace_buffer_free(state
->dts_aggbuffer
);
14526 for (i
= 0; i
< nspec
; i
++)
14527 dtrace_buffer_free(spec
[i
].dtsp_buffer
);
14529 if (state
->dts_cleaner
!= CYCLIC_NONE
)
14530 cyclic_remove(state
->dts_cleaner
);
14532 if (state
->dts_deadman
!= CYCLIC_NONE
)
14533 cyclic_remove(state
->dts_deadman
);
14535 dtrace_dstate_fini(&vstate
->dtvs_dynvars
);
14536 dtrace_vstate_fini(vstate
);
14537 kmem_free(state
->dts_ecbs
, state
->dts_necbs
* sizeof (dtrace_ecb_t
*));
14539 if (state
->dts_aggregations
!= NULL
) {
14541 for (i
= 0; i
< state
->dts_naggregations
; i
++)
14542 ASSERT(state
->dts_aggregations
[i
] == NULL
);
14544 ASSERT(state
->dts_naggregations
> 0);
14545 kmem_free(state
->dts_aggregations
,
14546 state
->dts_naggregations
* sizeof (dtrace_aggregation_t
*));
14549 kmem_free(state
->dts_buffer
, bufsize
);
14550 kmem_free(state
->dts_aggbuffer
, bufsize
);
14552 for (i
= 0; i
< nspec
; i
++)
14553 kmem_free(spec
[i
].dtsp_buffer
, bufsize
);
14555 kmem_free(spec
, nspec
* sizeof (dtrace_speculation_t
));
14557 dtrace_format_destroy(state
);
14559 vmem_destroy(state
->dts_aggid_arena
);
14560 ddi_soft_state_free(dtrace_softstate
, minor
);
14561 vmem_free(dtrace_minor
, (void *)(uintptr_t)minor
, 1);
14565 * DTrace Anonymous Enabling Functions
14567 static dtrace_state_t
*
14568 dtrace_anon_grab(void)
14570 dtrace_state_t
*state
;
14572 ASSERT(MUTEX_HELD(&dtrace_lock
));
14574 if ((state
= dtrace_anon
.dta_state
) == NULL
) {
14575 ASSERT(dtrace_anon
.dta_enabling
== NULL
);
14579 ASSERT(dtrace_anon
.dta_enabling
!= NULL
);
14580 ASSERT(dtrace_retained
!= NULL
);
14582 dtrace_enabling_destroy(dtrace_anon
.dta_enabling
);
14583 dtrace_anon
.dta_enabling
= NULL
;
14584 dtrace_anon
.dta_state
= NULL
;
14590 dtrace_anon_property(void)
14593 dtrace_state_t
*state
;
14595 char c
[32]; /* enough for "dof-data-" + digits */
14597 ASSERT(MUTEX_HELD(&dtrace_lock
));
14598 ASSERT(MUTEX_HELD(&cpu_lock
));
14600 for (i
= 0; ; i
++) {
14601 (void) snprintf(c
, sizeof (c
), "dof-data-%d", i
);
14603 dtrace_err_verbose
= 1;
14605 if ((dof
= dtrace_dof_property(c
)) == NULL
) {
14606 dtrace_err_verbose
= 0;
14611 * We want to create anonymous state, so we need to transition
14612 * the kernel debugger to indicate that DTrace is active. If
14613 * this fails (e.g. because the debugger has modified text in
14614 * some way), we won't continue with the processing.
14616 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE
) != 0) {
14617 cmn_err(CE_NOTE
, "kernel debugger active; anonymous "
14618 "enabling ignored.");
14619 dtrace_dof_destroy(dof
);
14624 * If we haven't allocated an anonymous state, we'll do so now.
14626 if ((state
= dtrace_anon
.dta_state
) == NULL
) {
14627 state
= dtrace_state_create(NULL
, NULL
);
14628 dtrace_anon
.dta_state
= state
;
14630 if (state
== NULL
) {
14632 * This basically shouldn't happen: the only
14633 * failure mode from dtrace_state_create() is a
14634 * failure of ddi_soft_state_zalloc() that
14635 * itself should never happen. Still, the
14636 * interface allows for a failure mode, and
14637 * we want to fail as gracefully as possible:
14638 * we'll emit an error message and cease
14639 * processing anonymous state in this case.
14641 cmn_err(CE_WARN
, "failed to create "
14642 "anonymous state");
14643 dtrace_dof_destroy(dof
);
14648 rv
= dtrace_dof_slurp(dof
, &state
->dts_vstate
, CRED(),
14649 &dtrace_anon
.dta_enabling
, 0, B_TRUE
);
14652 rv
= dtrace_dof_options(dof
, state
);
14654 dtrace_err_verbose
= 0;
14655 dtrace_dof_destroy(dof
);
14659 * This is malformed DOF; chuck any anonymous state
14662 ASSERT(dtrace_anon
.dta_enabling
== NULL
);
14663 dtrace_state_destroy(state
);
14664 dtrace_anon
.dta_state
= NULL
;
14668 ASSERT(dtrace_anon
.dta_enabling
!= NULL
);
14671 if (dtrace_anon
.dta_enabling
!= NULL
) {
14675 * dtrace_enabling_retain() can only fail because we are
14676 * trying to retain more enablings than are allowed -- but
14677 * we only have one anonymous enabling, and we are guaranteed
14678 * to be allowed at least one retained enabling; we assert
14679 * that dtrace_enabling_retain() returns success.
14681 rval
= dtrace_enabling_retain(dtrace_anon
.dta_enabling
);
14684 dtrace_enabling_dump(dtrace_anon
.dta_enabling
);
14689 * DTrace Helper Functions
14692 dtrace_helper_trace(dtrace_helper_action_t
*helper
,
14693 dtrace_mstate_t
*mstate
, dtrace_vstate_t
*vstate
, int where
)
14695 uint32_t size
, next
, nnext
, i
;
14696 dtrace_helptrace_t
*ent
, *buffer
;
14697 uint16_t flags
= cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
14699 if ((buffer
= dtrace_helptrace_buffer
) == NULL
)
14702 ASSERT(vstate
->dtvs_nlocals
<= dtrace_helptrace_nlocals
);
14705 * What would a tracing framework be without its own tracing
14706 * framework? (Well, a hell of a lot simpler, for starters...)
14708 size
= sizeof (dtrace_helptrace_t
) + dtrace_helptrace_nlocals
*
14709 sizeof (uint64_t) - sizeof (uint64_t);
14712 * Iterate until we can allocate a slot in the trace buffer.
14715 next
= dtrace_helptrace_next
;
14717 if (next
+ size
< dtrace_helptrace_bufsize
) {
14718 nnext
= next
+ size
;
14722 } while (dtrace_cas32(&dtrace_helptrace_next
, next
, nnext
) != next
);
14725 * We have our slot; fill it in.
14727 if (nnext
== size
) {
14728 dtrace_helptrace_wrapped
++;
14732 ent
= (dtrace_helptrace_t
*)((uintptr_t)buffer
+ next
);
14733 ent
->dtht_helper
= helper
;
14734 ent
->dtht_where
= where
;
14735 ent
->dtht_nlocals
= vstate
->dtvs_nlocals
;
14737 ent
->dtht_fltoffs
= (mstate
->dtms_present
& DTRACE_MSTATE_FLTOFFS
) ?
14738 mstate
->dtms_fltoffs
: -1;
14739 ent
->dtht_fault
= DTRACE_FLAGS2FLT(flags
);
14740 ent
->dtht_illval
= cpu_core
[CPU
->cpu_id
].cpuc_dtrace_illval
;
14742 for (i
= 0; i
< vstate
->dtvs_nlocals
; i
++) {
14743 dtrace_statvar_t
*svar
;
14745 if ((svar
= vstate
->dtvs_locals
[i
]) == NULL
)
14748 ASSERT(svar
->dtsv_size
>= NCPU
* sizeof (uint64_t));
14749 ent
->dtht_locals
[i
] =
14750 ((uint64_t *)(uintptr_t)svar
->dtsv_data
)[CPU
->cpu_id
];
14755 dtrace_helper(int which
, dtrace_mstate_t
*mstate
,
14756 dtrace_state_t
*state
, uint64_t arg0
, uint64_t arg1
)
14758 uint16_t *flags
= &cpu_core
[CPU
->cpu_id
].cpuc_dtrace_flags
;
14759 uint64_t sarg0
= mstate
->dtms_arg
[0];
14760 uint64_t sarg1
= mstate
->dtms_arg
[1];
14762 dtrace_helpers_t
*helpers
= curproc
->p_dtrace_helpers
;
14763 dtrace_helper_action_t
*helper
;
14764 dtrace_vstate_t
*vstate
;
14765 dtrace_difo_t
*pred
;
14766 int i
, trace
= dtrace_helptrace_buffer
!= NULL
;
14768 ASSERT(which
>= 0 && which
< DTRACE_NHELPER_ACTIONS
);
14770 if (helpers
== NULL
)
14773 if ((helper
= helpers
->dthps_actions
[which
]) == NULL
)
14776 vstate
= &helpers
->dthps_vstate
;
14777 mstate
->dtms_arg
[0] = arg0
;
14778 mstate
->dtms_arg
[1] = arg1
;
14781 * Now iterate over each helper. If its predicate evaluates to 'true',
14782 * we'll call the corresponding actions. Note that the below calls
14783 * to dtrace_dif_emulate() may set faults in machine state. This is
14784 * okay: our caller (the outer dtrace_dif_emulate()) will simply plow
14785 * the stored DIF offset with its own (which is the desired behavior).
14786 * Also, note the calls to dtrace_dif_emulate() may allocate scratch
14787 * from machine state; this is okay, too.
14789 for (; helper
!= NULL
; helper
= helper
->dtha_next
) {
14790 if ((pred
= helper
->dtha_predicate
) != NULL
) {
14792 dtrace_helper_trace(helper
, mstate
, vstate
, 0);
14794 if (!dtrace_dif_emulate(pred
, mstate
, vstate
, state
))
14797 if (*flags
& CPU_DTRACE_FAULT
)
14801 for (i
= 0; i
< helper
->dtha_nactions
; i
++) {
14803 dtrace_helper_trace(helper
,
14804 mstate
, vstate
, i
+ 1);
14806 rval
= dtrace_dif_emulate(helper
->dtha_actions
[i
],
14807 mstate
, vstate
, state
);
14809 if (*flags
& CPU_DTRACE_FAULT
)
14815 dtrace_helper_trace(helper
, mstate
, vstate
,
14816 DTRACE_HELPTRACE_NEXT
);
14820 dtrace_helper_trace(helper
, mstate
, vstate
,
14821 DTRACE_HELPTRACE_DONE
);
14824 * Restore the arg0 that we saved upon entry.
14826 mstate
->dtms_arg
[0] = sarg0
;
14827 mstate
->dtms_arg
[1] = sarg1
;
14833 dtrace_helper_trace(helper
, mstate
, vstate
,
14834 DTRACE_HELPTRACE_ERR
);
14837 * Restore the arg0 that we saved upon entry.
14839 mstate
->dtms_arg
[0] = sarg0
;
14840 mstate
->dtms_arg
[1] = sarg1
;
14846 dtrace_helper_action_destroy(dtrace_helper_action_t
*helper
,
14847 dtrace_vstate_t
*vstate
)
14851 if (helper
->dtha_predicate
!= NULL
)
14852 dtrace_difo_release(helper
->dtha_predicate
, vstate
);
14854 for (i
= 0; i
< helper
->dtha_nactions
; i
++) {
14855 ASSERT(helper
->dtha_actions
[i
] != NULL
);
14856 dtrace_difo_release(helper
->dtha_actions
[i
], vstate
);
14859 kmem_free(helper
->dtha_actions
,
14860 helper
->dtha_nactions
* sizeof (dtrace_difo_t
*));
14861 kmem_free(helper
, sizeof (dtrace_helper_action_t
));
14865 dtrace_helper_destroygen(int gen
)
14867 proc_t
*p
= curproc
;
14868 dtrace_helpers_t
*help
= p
->p_dtrace_helpers
;
14869 dtrace_vstate_t
*vstate
;
14872 ASSERT(MUTEX_HELD(&dtrace_lock
));
14874 if (help
== NULL
|| gen
> help
->dthps_generation
)
14877 vstate
= &help
->dthps_vstate
;
14879 for (i
= 0; i
< DTRACE_NHELPER_ACTIONS
; i
++) {
14880 dtrace_helper_action_t
*last
= NULL
, *h
, *next
;
14882 for (h
= help
->dthps_actions
[i
]; h
!= NULL
; h
= next
) {
14883 next
= h
->dtha_next
;
14885 if (h
->dtha_generation
== gen
) {
14886 if (last
!= NULL
) {
14887 last
->dtha_next
= next
;
14889 help
->dthps_actions
[i
] = next
;
14892 dtrace_helper_action_destroy(h
, vstate
);
14900 * Interate until we've cleared out all helper providers with the
14901 * given generation number.
14904 dtrace_helper_provider_t
*prov
;
14907 * Look for a helper provider with the right generation. We
14908 * have to start back at the beginning of the list each time
14909 * because we drop dtrace_lock. It's unlikely that we'll make
14910 * more than two passes.
14912 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
14913 prov
= help
->dthps_provs
[i
];
14915 if (prov
->dthp_generation
== gen
)
14920 * If there were no matches, we're done.
14922 if (i
== help
->dthps_nprovs
)
14926 * Move the last helper provider into this slot.
14928 help
->dthps_nprovs
--;
14929 help
->dthps_provs
[i
] = help
->dthps_provs
[help
->dthps_nprovs
];
14930 help
->dthps_provs
[help
->dthps_nprovs
] = NULL
;
14932 mutex_exit(&dtrace_lock
);
14935 * If we have a meta provider, remove this helper provider.
14937 mutex_enter(&dtrace_meta_lock
);
14938 if (dtrace_meta_pid
!= NULL
) {
14939 ASSERT(dtrace_deferred_pid
== NULL
);
14940 dtrace_helper_provider_remove(&prov
->dthp_prov
,
14943 mutex_exit(&dtrace_meta_lock
);
14945 dtrace_helper_provider_destroy(prov
);
14947 mutex_enter(&dtrace_lock
);
14954 dtrace_helper_validate(dtrace_helper_action_t
*helper
)
14959 if ((dp
= helper
->dtha_predicate
) != NULL
)
14960 err
+= dtrace_difo_validate_helper(dp
);
14962 for (i
= 0; i
< helper
->dtha_nactions
; i
++)
14963 err
+= dtrace_difo_validate_helper(helper
->dtha_actions
[i
]);
14969 dtrace_helper_action_add(int which
, dtrace_ecbdesc_t
*ep
)
14971 dtrace_helpers_t
*help
;
14972 dtrace_helper_action_t
*helper
, *last
;
14973 dtrace_actdesc_t
*act
;
14974 dtrace_vstate_t
*vstate
;
14975 dtrace_predicate_t
*pred
;
14976 int count
= 0, nactions
= 0, i
;
14978 if (which
< 0 || which
>= DTRACE_NHELPER_ACTIONS
)
14981 help
= curproc
->p_dtrace_helpers
;
14982 last
= help
->dthps_actions
[which
];
14983 vstate
= &help
->dthps_vstate
;
14985 for (count
= 0; last
!= NULL
; last
= last
->dtha_next
) {
14987 if (last
->dtha_next
== NULL
)
14992 * If we already have dtrace_helper_actions_max helper actions for this
14993 * helper action type, we'll refuse to add a new one.
14995 if (count
>= dtrace_helper_actions_max
)
14998 helper
= kmem_zalloc(sizeof (dtrace_helper_action_t
), KM_SLEEP
);
14999 helper
->dtha_generation
= help
->dthps_generation
;
15001 if ((pred
= ep
->dted_pred
.dtpdd_predicate
) != NULL
) {
15002 ASSERT(pred
->dtp_difo
!= NULL
);
15003 dtrace_difo_hold(pred
->dtp_difo
);
15004 helper
->dtha_predicate
= pred
->dtp_difo
;
15007 for (act
= ep
->dted_action
; act
!= NULL
; act
= act
->dtad_next
) {
15008 if (act
->dtad_kind
!= DTRACEACT_DIFEXPR
)
15011 if (act
->dtad_difo
== NULL
)
15017 helper
->dtha_actions
= kmem_zalloc(sizeof (dtrace_difo_t
*) *
15018 (helper
->dtha_nactions
= nactions
), KM_SLEEP
);
15020 for (act
= ep
->dted_action
, i
= 0; act
!= NULL
; act
= act
->dtad_next
) {
15021 dtrace_difo_hold(act
->dtad_difo
);
15022 helper
->dtha_actions
[i
++] = act
->dtad_difo
;
15025 if (!dtrace_helper_validate(helper
))
15028 if (last
== NULL
) {
15029 help
->dthps_actions
[which
] = helper
;
15031 last
->dtha_next
= helper
;
15034 if (vstate
->dtvs_nlocals
> dtrace_helptrace_nlocals
) {
15035 dtrace_helptrace_nlocals
= vstate
->dtvs_nlocals
;
15036 dtrace_helptrace_next
= 0;
15041 dtrace_helper_action_destroy(helper
, vstate
);
15046 dtrace_helper_provider_register(proc_t
*p
, dtrace_helpers_t
*help
,
15047 dof_helper_t
*dofhp
)
15049 ASSERT(MUTEX_NOT_HELD(&dtrace_lock
));
15051 mutex_enter(&dtrace_meta_lock
);
15052 mutex_enter(&dtrace_lock
);
15054 if (!dtrace_attached() || dtrace_meta_pid
== NULL
) {
15056 * If the dtrace module is loaded but not attached, or if
15057 * there aren't isn't a meta provider registered to deal with
15058 * these provider descriptions, we need to postpone creating
15059 * the actual providers until later.
15062 if (help
->dthps_next
== NULL
&& help
->dthps_prev
== NULL
&&
15063 dtrace_deferred_pid
!= help
) {
15064 help
->dthps_deferred
= 1;
15065 help
->dthps_pid
= p
->p_pid
;
15066 help
->dthps_next
= dtrace_deferred_pid
;
15067 help
->dthps_prev
= NULL
;
15068 if (dtrace_deferred_pid
!= NULL
)
15069 dtrace_deferred_pid
->dthps_prev
= help
;
15070 dtrace_deferred_pid
= help
;
15073 mutex_exit(&dtrace_lock
);
15075 } else if (dofhp
!= NULL
) {
15077 * If the dtrace module is loaded and we have a particular
15078 * helper provider description, pass that off to the
15082 mutex_exit(&dtrace_lock
);
15084 dtrace_helper_provide(dofhp
, p
->p_pid
);
15088 * Otherwise, just pass all the helper provider descriptions
15089 * off to the meta provider.
15093 mutex_exit(&dtrace_lock
);
15095 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
15096 dtrace_helper_provide(&help
->dthps_provs
[i
]->dthp_prov
,
15101 mutex_exit(&dtrace_meta_lock
);
15105 dtrace_helper_provider_add(dof_helper_t
*dofhp
, int gen
)
15107 dtrace_helpers_t
*help
;
15108 dtrace_helper_provider_t
*hprov
, **tmp_provs
;
15109 uint_t tmp_maxprovs
, i
;
15111 ASSERT(MUTEX_HELD(&dtrace_lock
));
15113 help
= curproc
->p_dtrace_helpers
;
15114 ASSERT(help
!= NULL
);
15117 * If we already have dtrace_helper_providers_max helper providers,
15118 * we're refuse to add a new one.
15120 if (help
->dthps_nprovs
>= dtrace_helper_providers_max
)
15124 * Check to make sure this isn't a duplicate.
15126 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
15127 if (dofhp
->dofhp_addr
==
15128 help
->dthps_provs
[i
]->dthp_prov
.dofhp_addr
)
15132 hprov
= kmem_zalloc(sizeof (dtrace_helper_provider_t
), KM_SLEEP
);
15133 hprov
->dthp_prov
= *dofhp
;
15134 hprov
->dthp_ref
= 1;
15135 hprov
->dthp_generation
= gen
;
15138 * Allocate a bigger table for helper providers if it's already full.
15140 if (help
->dthps_maxprovs
== help
->dthps_nprovs
) {
15141 tmp_maxprovs
= help
->dthps_maxprovs
;
15142 tmp_provs
= help
->dthps_provs
;
15144 if (help
->dthps_maxprovs
== 0)
15145 help
->dthps_maxprovs
= 2;
15147 help
->dthps_maxprovs
*= 2;
15148 if (help
->dthps_maxprovs
> dtrace_helper_providers_max
)
15149 help
->dthps_maxprovs
= dtrace_helper_providers_max
;
15151 ASSERT(tmp_maxprovs
< help
->dthps_maxprovs
);
15153 help
->dthps_provs
= kmem_zalloc(help
->dthps_maxprovs
*
15154 sizeof (dtrace_helper_provider_t
*), KM_SLEEP
);
15156 if (tmp_provs
!= NULL
) {
15157 bcopy(tmp_provs
, help
->dthps_provs
, tmp_maxprovs
*
15158 sizeof (dtrace_helper_provider_t
*));
15159 kmem_free(tmp_provs
, tmp_maxprovs
*
15160 sizeof (dtrace_helper_provider_t
*));
15164 help
->dthps_provs
[help
->dthps_nprovs
] = hprov
;
15165 help
->dthps_nprovs
++;
15171 dtrace_helper_provider_destroy(dtrace_helper_provider_t
*hprov
)
15173 mutex_enter(&dtrace_lock
);
15175 if (--hprov
->dthp_ref
== 0) {
15177 mutex_exit(&dtrace_lock
);
15178 dof
= (dof_hdr_t
*)(uintptr_t)hprov
->dthp_prov
.dofhp_dof
;
15179 dtrace_dof_destroy(dof
);
15180 kmem_free(hprov
, sizeof (dtrace_helper_provider_t
));
15182 mutex_exit(&dtrace_lock
);
15187 dtrace_helper_provider_validate(dof_hdr_t
*dof
, dof_sec_t
*sec
)
15189 uintptr_t daddr
= (uintptr_t)dof
;
15190 dof_sec_t
*str_sec
, *prb_sec
, *arg_sec
, *off_sec
, *enoff_sec
;
15191 dof_provider_t
*provider
;
15192 dof_probe_t
*probe
;
15194 char *strtab
, *typestr
;
15195 dof_stridx_t typeidx
;
15197 uint_t nprobes
, j
, k
;
15199 ASSERT(sec
->dofs_type
== DOF_SECT_PROVIDER
);
15201 if (sec
->dofs_offset
& (sizeof (uint_t
) - 1)) {
15202 dtrace_dof_error(dof
, "misaligned section offset");
15207 * The section needs to be large enough to contain the DOF provider
15208 * structure appropriate for the given version.
15210 if (sec
->dofs_size
<
15211 ((dof
->dofh_ident
[DOF_ID_VERSION
] == DOF_VERSION_1
) ?
15212 offsetof(dof_provider_t
, dofpv_prenoffs
) :
15213 sizeof (dof_provider_t
))) {
15214 dtrace_dof_error(dof
, "provider section too small");
15218 provider
= (dof_provider_t
*)(uintptr_t)(daddr
+ sec
->dofs_offset
);
15219 str_sec
= dtrace_dof_sect(dof
, DOF_SECT_STRTAB
, provider
->dofpv_strtab
);
15220 prb_sec
= dtrace_dof_sect(dof
, DOF_SECT_PROBES
, provider
->dofpv_probes
);
15221 arg_sec
= dtrace_dof_sect(dof
, DOF_SECT_PRARGS
, provider
->dofpv_prargs
);
15222 off_sec
= dtrace_dof_sect(dof
, DOF_SECT_PROFFS
, provider
->dofpv_proffs
);
15224 if (str_sec
== NULL
|| prb_sec
== NULL
||
15225 arg_sec
== NULL
|| off_sec
== NULL
)
15230 if (dof
->dofh_ident
[DOF_ID_VERSION
] != DOF_VERSION_1
&&
15231 provider
->dofpv_prenoffs
!= DOF_SECT_NONE
&&
15232 (enoff_sec
= dtrace_dof_sect(dof
, DOF_SECT_PRENOFFS
,
15233 provider
->dofpv_prenoffs
)) == NULL
)
15236 strtab
= (char *)(uintptr_t)(daddr
+ str_sec
->dofs_offset
);
15238 if (provider
->dofpv_name
>= str_sec
->dofs_size
||
15239 strlen(strtab
+ provider
->dofpv_name
) >= DTRACE_PROVNAMELEN
) {
15240 dtrace_dof_error(dof
, "invalid provider name");
15244 if (prb_sec
->dofs_entsize
== 0 ||
15245 prb_sec
->dofs_entsize
> prb_sec
->dofs_size
) {
15246 dtrace_dof_error(dof
, "invalid entry size");
15250 if (prb_sec
->dofs_entsize
& (sizeof (uintptr_t) - 1)) {
15251 dtrace_dof_error(dof
, "misaligned entry size");
15255 if (off_sec
->dofs_entsize
!= sizeof (uint32_t)) {
15256 dtrace_dof_error(dof
, "invalid entry size");
15260 if (off_sec
->dofs_offset
& (sizeof (uint32_t) - 1)) {
15261 dtrace_dof_error(dof
, "misaligned section offset");
15265 if (arg_sec
->dofs_entsize
!= sizeof (uint8_t)) {
15266 dtrace_dof_error(dof
, "invalid entry size");
15270 arg
= (uint8_t *)(uintptr_t)(daddr
+ arg_sec
->dofs_offset
);
15272 nprobes
= prb_sec
->dofs_size
/ prb_sec
->dofs_entsize
;
15275 * Take a pass through the probes to check for errors.
15277 for (j
= 0; j
< nprobes
; j
++) {
15278 probe
= (dof_probe_t
*)(uintptr_t)(daddr
+
15279 prb_sec
->dofs_offset
+ j
* prb_sec
->dofs_entsize
);
15281 if (probe
->dofpr_func
>= str_sec
->dofs_size
) {
15282 dtrace_dof_error(dof
, "invalid function name");
15286 if (strlen(strtab
+ probe
->dofpr_func
) >= DTRACE_FUNCNAMELEN
) {
15287 dtrace_dof_error(dof
, "function name too long");
15291 if (probe
->dofpr_name
>= str_sec
->dofs_size
||
15292 strlen(strtab
+ probe
->dofpr_name
) >= DTRACE_NAMELEN
) {
15293 dtrace_dof_error(dof
, "invalid probe name");
15298 * The offset count must not wrap the index, and the offsets
15299 * must also not overflow the section's data.
15301 if (probe
->dofpr_offidx
+ probe
->dofpr_noffs
<
15302 probe
->dofpr_offidx
||
15303 (probe
->dofpr_offidx
+ probe
->dofpr_noffs
) *
15304 off_sec
->dofs_entsize
> off_sec
->dofs_size
) {
15305 dtrace_dof_error(dof
, "invalid probe offset");
15309 if (dof
->dofh_ident
[DOF_ID_VERSION
] != DOF_VERSION_1
) {
15311 * If there's no is-enabled offset section, make sure
15312 * there aren't any is-enabled offsets. Otherwise
15313 * perform the same checks as for probe offsets
15314 * (immediately above).
15316 if (enoff_sec
== NULL
) {
15317 if (probe
->dofpr_enoffidx
!= 0 ||
15318 probe
->dofpr_nenoffs
!= 0) {
15319 dtrace_dof_error(dof
, "is-enabled "
15320 "offsets with null section");
15323 } else if (probe
->dofpr_enoffidx
+
15324 probe
->dofpr_nenoffs
< probe
->dofpr_enoffidx
||
15325 (probe
->dofpr_enoffidx
+ probe
->dofpr_nenoffs
) *
15326 enoff_sec
->dofs_entsize
> enoff_sec
->dofs_size
) {
15327 dtrace_dof_error(dof
, "invalid is-enabled "
15332 if (probe
->dofpr_noffs
+ probe
->dofpr_nenoffs
== 0) {
15333 dtrace_dof_error(dof
, "zero probe and "
15334 "is-enabled offsets");
15337 } else if (probe
->dofpr_noffs
== 0) {
15338 dtrace_dof_error(dof
, "zero probe offsets");
15342 if (probe
->dofpr_argidx
+ probe
->dofpr_xargc
<
15343 probe
->dofpr_argidx
||
15344 (probe
->dofpr_argidx
+ probe
->dofpr_xargc
) *
15345 arg_sec
->dofs_entsize
> arg_sec
->dofs_size
) {
15346 dtrace_dof_error(dof
, "invalid args");
15350 typeidx
= probe
->dofpr_nargv
;
15351 typestr
= strtab
+ probe
->dofpr_nargv
;
15352 for (k
= 0; k
< probe
->dofpr_nargc
; k
++) {
15353 if (typeidx
>= str_sec
->dofs_size
) {
15354 dtrace_dof_error(dof
, "bad "
15355 "native argument type");
15359 typesz
= strlen(typestr
) + 1;
15360 if (typesz
> DTRACE_ARGTYPELEN
) {
15361 dtrace_dof_error(dof
, "native "
15362 "argument type too long");
15369 typeidx
= probe
->dofpr_xargv
;
15370 typestr
= strtab
+ probe
->dofpr_xargv
;
15371 for (k
= 0; k
< probe
->dofpr_xargc
; k
++) {
15372 if (arg
[probe
->dofpr_argidx
+ k
] > probe
->dofpr_nargc
) {
15373 dtrace_dof_error(dof
, "bad "
15374 "native argument index");
15378 if (typeidx
>= str_sec
->dofs_size
) {
15379 dtrace_dof_error(dof
, "bad "
15380 "translated argument type");
15384 typesz
= strlen(typestr
) + 1;
15385 if (typesz
> DTRACE_ARGTYPELEN
) {
15386 dtrace_dof_error(dof
, "translated argument "
15400 dtrace_helper_slurp(dof_hdr_t
*dof
, dof_helper_t
*dhp
)
15402 dtrace_helpers_t
*help
;
15403 dtrace_vstate_t
*vstate
;
15404 dtrace_enabling_t
*enab
= NULL
;
15405 int i
, gen
, rv
, nhelpers
= 0, nprovs
= 0, destroy
= 1;
15406 uintptr_t daddr
= (uintptr_t)dof
;
15408 ASSERT(MUTEX_HELD(&dtrace_lock
));
15410 if ((help
= curproc
->p_dtrace_helpers
) == NULL
)
15411 help
= dtrace_helpers_create(curproc
);
15413 vstate
= &help
->dthps_vstate
;
15415 if ((rv
= dtrace_dof_slurp(dof
, vstate
, NULL
, &enab
,
15416 dhp
!= NULL
? dhp
->dofhp_addr
: 0, B_FALSE
)) != 0) {
15417 dtrace_dof_destroy(dof
);
15422 * Look for helper providers and validate their descriptions.
15425 for (i
= 0; i
< dof
->dofh_secnum
; i
++) {
15426 dof_sec_t
*sec
= (dof_sec_t
*)(uintptr_t)(daddr
+
15427 dof
->dofh_secoff
+ i
* dof
->dofh_secsize
);
15429 if (sec
->dofs_type
!= DOF_SECT_PROVIDER
)
15432 if (dtrace_helper_provider_validate(dof
, sec
) != 0) {
15433 dtrace_enabling_destroy(enab
);
15434 dtrace_dof_destroy(dof
);
15443 * Now we need to walk through the ECB descriptions in the enabling.
15445 for (i
= 0; i
< enab
->dten_ndesc
; i
++) {
15446 dtrace_ecbdesc_t
*ep
= enab
->dten_desc
[i
];
15447 dtrace_probedesc_t
*desc
= &ep
->dted_probe
;
15449 if (strcmp(desc
->dtpd_provider
, "dtrace") != 0)
15452 if (strcmp(desc
->dtpd_mod
, "helper") != 0)
15455 if (strcmp(desc
->dtpd_func
, "ustack") != 0)
15458 if ((rv
= dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK
,
15461 * Adding this helper action failed -- we are now going
15462 * to rip out the entire generation and return failure.
15464 (void) dtrace_helper_destroygen(help
->dthps_generation
);
15465 dtrace_enabling_destroy(enab
);
15466 dtrace_dof_destroy(dof
);
15473 if (nhelpers
< enab
->dten_ndesc
)
15474 dtrace_dof_error(dof
, "unmatched helpers");
15476 gen
= help
->dthps_generation
++;
15477 dtrace_enabling_destroy(enab
);
15479 if (dhp
!= NULL
&& nprovs
> 0) {
15481 * Now that this is in-kernel, we change the sense of the
15482 * members: dofhp_dof denotes the in-kernel copy of the DOF
15483 * and dofhp_addr denotes the address at user-level.
15485 dhp
->dofhp_addr
= dhp
->dofhp_dof
;
15486 dhp
->dofhp_dof
= (uint64_t)(uintptr_t)dof
;
15488 if (dtrace_helper_provider_add(dhp
, gen
) == 0) {
15489 mutex_exit(&dtrace_lock
);
15490 dtrace_helper_provider_register(curproc
, help
, dhp
);
15491 mutex_enter(&dtrace_lock
);
15498 dtrace_dof_destroy(dof
);
15503 static dtrace_helpers_t
*
15504 dtrace_helpers_create(proc_t
*p
)
15506 dtrace_helpers_t
*help
;
15508 ASSERT(MUTEX_HELD(&dtrace_lock
));
15509 ASSERT(p
->p_dtrace_helpers
== NULL
);
15511 help
= kmem_zalloc(sizeof (dtrace_helpers_t
), KM_SLEEP
);
15512 help
->dthps_actions
= kmem_zalloc(sizeof (dtrace_helper_action_t
*) *
15513 DTRACE_NHELPER_ACTIONS
, KM_SLEEP
);
15515 p
->p_dtrace_helpers
= help
;
15522 dtrace_helpers_destroy(proc_t
*p
)
15524 dtrace_helpers_t
*help
;
15525 dtrace_vstate_t
*vstate
;
15528 mutex_enter(&dtrace_lock
);
15530 ASSERT(p
->p_dtrace_helpers
!= NULL
);
15531 ASSERT(dtrace_helpers
> 0);
15533 help
= p
->p_dtrace_helpers
;
15534 vstate
= &help
->dthps_vstate
;
15537 * We're now going to lose the help from this process.
15539 p
->p_dtrace_helpers
= NULL
;
15540 if (p
== curproc
) {
15544 * It is sometimes necessary to clean up dtrace helpers from a
15545 * an incomplete child process as part of a failed fork
15546 * operation. In such situations, a dtrace_sync() call should
15547 * be unnecessary as the process should be devoid of threads,
15548 * much less any in probe context.
15550 VERIFY(p
->p_stat
== SIDL
);
15554 * Destroy the helper actions.
15556 for (i
= 0; i
< DTRACE_NHELPER_ACTIONS
; i
++) {
15557 dtrace_helper_action_t
*h
, *next
;
15559 for (h
= help
->dthps_actions
[i
]; h
!= NULL
; h
= next
) {
15560 next
= h
->dtha_next
;
15561 dtrace_helper_action_destroy(h
, vstate
);
15566 mutex_exit(&dtrace_lock
);
15569 * Destroy the helper providers.
15571 if (help
->dthps_maxprovs
> 0) {
15572 mutex_enter(&dtrace_meta_lock
);
15573 if (dtrace_meta_pid
!= NULL
) {
15574 ASSERT(dtrace_deferred_pid
== NULL
);
15576 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
15577 dtrace_helper_provider_remove(
15578 &help
->dthps_provs
[i
]->dthp_prov
, p
->p_pid
);
15581 mutex_enter(&dtrace_lock
);
15582 ASSERT(help
->dthps_deferred
== 0 ||
15583 help
->dthps_next
!= NULL
||
15584 help
->dthps_prev
!= NULL
||
15585 help
== dtrace_deferred_pid
);
15588 * Remove the helper from the deferred list.
15590 if (help
->dthps_next
!= NULL
)
15591 help
->dthps_next
->dthps_prev
= help
->dthps_prev
;
15592 if (help
->dthps_prev
!= NULL
)
15593 help
->dthps_prev
->dthps_next
= help
->dthps_next
;
15594 if (dtrace_deferred_pid
== help
) {
15595 dtrace_deferred_pid
= help
->dthps_next
;
15596 ASSERT(help
->dthps_prev
== NULL
);
15599 mutex_exit(&dtrace_lock
);
15602 mutex_exit(&dtrace_meta_lock
);
15604 for (i
= 0; i
< help
->dthps_nprovs
; i
++) {
15605 dtrace_helper_provider_destroy(help
->dthps_provs
[i
]);
15608 kmem_free(help
->dthps_provs
, help
->dthps_maxprovs
*
15609 sizeof (dtrace_helper_provider_t
*));
15612 mutex_enter(&dtrace_lock
);
15614 dtrace_vstate_fini(&help
->dthps_vstate
);
15615 kmem_free(help
->dthps_actions
,
15616 sizeof (dtrace_helper_action_t
*) * DTRACE_NHELPER_ACTIONS
);
15617 kmem_free(help
, sizeof (dtrace_helpers_t
));
15620 mutex_exit(&dtrace_lock
);
15624 dtrace_helpers_duplicate(proc_t
*from
, proc_t
*to
)
15626 dtrace_helpers_t
*help
, *newhelp
;
15627 dtrace_helper_action_t
*helper
, *new, *last
;
15629 dtrace_vstate_t
*vstate
;
15630 int i
, j
, sz
, hasprovs
= 0;
15632 mutex_enter(&dtrace_lock
);
15633 ASSERT(from
->p_dtrace_helpers
!= NULL
);
15634 ASSERT(dtrace_helpers
> 0);
15636 help
= from
->p_dtrace_helpers
;
15637 newhelp
= dtrace_helpers_create(to
);
15638 ASSERT(to
->p_dtrace_helpers
!= NULL
);
15640 newhelp
->dthps_generation
= help
->dthps_generation
;
15641 vstate
= &newhelp
->dthps_vstate
;
15644 * Duplicate the helper actions.
15646 for (i
= 0; i
< DTRACE_NHELPER_ACTIONS
; i
++) {
15647 if ((helper
= help
->dthps_actions
[i
]) == NULL
)
15650 for (last
= NULL
; helper
!= NULL
; helper
= helper
->dtha_next
) {
15651 new = kmem_zalloc(sizeof (dtrace_helper_action_t
),
15653 new->dtha_generation
= helper
->dtha_generation
;
15655 if ((dp
= helper
->dtha_predicate
) != NULL
) {
15656 dp
= dtrace_difo_duplicate(dp
, vstate
);
15657 new->dtha_predicate
= dp
;
15660 new->dtha_nactions
= helper
->dtha_nactions
;
15661 sz
= sizeof (dtrace_difo_t
*) * new->dtha_nactions
;
15662 new->dtha_actions
= kmem_alloc(sz
, KM_SLEEP
);
15664 for (j
= 0; j
< new->dtha_nactions
; j
++) {
15665 dtrace_difo_t
*dp
= helper
->dtha_actions
[j
];
15667 ASSERT(dp
!= NULL
);
15668 dp
= dtrace_difo_duplicate(dp
, vstate
);
15669 new->dtha_actions
[j
] = dp
;
15672 if (last
!= NULL
) {
15673 last
->dtha_next
= new;
15675 newhelp
->dthps_actions
[i
] = new;
15683 * Duplicate the helper providers and register them with the
15684 * DTrace framework.
15686 if (help
->dthps_nprovs
> 0) {
15687 newhelp
->dthps_nprovs
= help
->dthps_nprovs
;
15688 newhelp
->dthps_maxprovs
= help
->dthps_nprovs
;
15689 newhelp
->dthps_provs
= kmem_alloc(newhelp
->dthps_nprovs
*
15690 sizeof (dtrace_helper_provider_t
*), KM_SLEEP
);
15691 for (i
= 0; i
< newhelp
->dthps_nprovs
; i
++) {
15692 newhelp
->dthps_provs
[i
] = help
->dthps_provs
[i
];
15693 newhelp
->dthps_provs
[i
]->dthp_ref
++;
15699 mutex_exit(&dtrace_lock
);
15702 dtrace_helper_provider_register(to
, newhelp
, NULL
);
15706 * DTrace Hook Functions
15709 dtrace_module_loaded(struct modctl
*ctl
)
15711 dtrace_provider_t
*prv
;
15713 mutex_enter(&dtrace_provider_lock
);
15714 mutex_enter(&mod_lock
);
15716 ASSERT(ctl
->mod_busy
);
15719 * We're going to call each providers per-module provide operation
15720 * specifying only this module.
15722 for (prv
= dtrace_provider
; prv
!= NULL
; prv
= prv
->dtpv_next
)
15723 prv
->dtpv_pops
.dtps_provide_module(prv
->dtpv_arg
, ctl
);
15725 mutex_exit(&mod_lock
);
15726 mutex_exit(&dtrace_provider_lock
);
15729 * If we have any retained enablings, we need to match against them.
15730 * Enabling probes requires that cpu_lock be held, and we cannot hold
15731 * cpu_lock here -- it is legal for cpu_lock to be held when loading a
15732 * module. (In particular, this happens when loading scheduling
15733 * classes.) So if we have any retained enablings, we need to dispatch
15734 * our task queue to do the match for us.
15736 mutex_enter(&dtrace_lock
);
15738 if (dtrace_retained
== NULL
) {
15739 mutex_exit(&dtrace_lock
);
15743 (void) taskq_dispatch(dtrace_taskq
,
15744 (task_func_t
*)dtrace_enabling_matchall
, NULL
, TQ_SLEEP
);
15746 mutex_exit(&dtrace_lock
);
15749 * And now, for a little heuristic sleaze: in general, we want to
15750 * match modules as soon as they load. However, we cannot guarantee
15751 * this, because it would lead us to the lock ordering violation
15752 * outlined above. The common case, of course, is that cpu_lock is
15753 * _not_ held -- so we delay here for a clock tick, hoping that that's
15754 * long enough for the task queue to do its work. If it's not, it's
15755 * not a serious problem -- it just means that the module that we
15756 * just loaded may not be immediately instrumentable.
15762 dtrace_module_unloaded(struct modctl
*ctl
)
15764 dtrace_probe_t
template, *probe
, *first
, *next
;
15765 dtrace_provider_t
*prov
;
15767 template.dtpr_mod
= ctl
->mod_modname
;
15769 mutex_enter(&dtrace_provider_lock
);
15770 mutex_enter(&mod_lock
);
15771 mutex_enter(&dtrace_lock
);
15773 if (dtrace_bymod
== NULL
) {
15775 * The DTrace module is loaded (obviously) but not attached;
15776 * we don't have any work to do.
15778 mutex_exit(&dtrace_provider_lock
);
15779 mutex_exit(&mod_lock
);
15780 mutex_exit(&dtrace_lock
);
15784 for (probe
= first
= dtrace_hash_lookup(dtrace_bymod
, &template);
15785 probe
!= NULL
; probe
= probe
->dtpr_nextmod
) {
15786 if (probe
->dtpr_ecb
!= NULL
) {
15787 mutex_exit(&dtrace_provider_lock
);
15788 mutex_exit(&mod_lock
);
15789 mutex_exit(&dtrace_lock
);
15792 * This shouldn't _actually_ be possible -- we're
15793 * unloading a module that has an enabled probe in it.
15794 * (It's normally up to the provider to make sure that
15795 * this can't happen.) However, because dtps_enable()
15796 * doesn't have a failure mode, there can be an
15797 * enable/unload race. Upshot: we don't want to
15798 * assert, but we're not going to disable the
15801 if (dtrace_err_verbose
) {
15802 cmn_err(CE_WARN
, "unloaded module '%s' had "
15803 "enabled probes", ctl
->mod_modname
);
15812 for (first
= NULL
; probe
!= NULL
; probe
= next
) {
15813 ASSERT(dtrace_probes
[probe
->dtpr_id
- 1] == probe
);
15815 dtrace_probes
[probe
->dtpr_id
- 1] = NULL
;
15817 next
= probe
->dtpr_nextmod
;
15818 dtrace_hash_remove(dtrace_bymod
, probe
);
15819 dtrace_hash_remove(dtrace_byfunc
, probe
);
15820 dtrace_hash_remove(dtrace_byname
, probe
);
15822 if (first
== NULL
) {
15824 probe
->dtpr_nextmod
= NULL
;
15826 probe
->dtpr_nextmod
= first
;
15832 * We've removed all of the module's probes from the hash chains and
15833 * from the probe array. Now issue a dtrace_sync() to be sure that
15834 * everyone has cleared out from any probe array processing.
15838 for (probe
= first
; probe
!= NULL
; probe
= first
) {
15839 first
= probe
->dtpr_nextmod
;
15840 prov
= probe
->dtpr_provider
;
15841 prov
->dtpv_pops
.dtps_destroy(prov
->dtpv_arg
, probe
->dtpr_id
,
15843 kmem_free(probe
->dtpr_mod
, strlen(probe
->dtpr_mod
) + 1);
15844 kmem_free(probe
->dtpr_func
, strlen(probe
->dtpr_func
) + 1);
15845 kmem_free(probe
->dtpr_name
, strlen(probe
->dtpr_name
) + 1);
15846 vmem_free(dtrace_arena
, (void *)(uintptr_t)probe
->dtpr_id
, 1);
15847 kmem_free(probe
, sizeof (dtrace_probe_t
));
15850 mutex_exit(&dtrace_lock
);
15851 mutex_exit(&mod_lock
);
15852 mutex_exit(&dtrace_provider_lock
);
15856 dtrace_suspend(void)
15858 dtrace_probe_foreach(offsetof(dtrace_pops_t
, dtps_suspend
));
15862 dtrace_resume(void)
15864 dtrace_probe_foreach(offsetof(dtrace_pops_t
, dtps_resume
));
15868 dtrace_cpu_setup(cpu_setup_t what
, processorid_t cpu
)
15870 ASSERT(MUTEX_HELD(&cpu_lock
));
15871 mutex_enter(&dtrace_lock
);
15875 dtrace_state_t
*state
;
15876 dtrace_optval_t
*opt
, rs
, c
;
15879 * For now, we only allocate a new buffer for anonymous state.
15881 if ((state
= dtrace_anon
.dta_state
) == NULL
)
15884 if (state
->dts_activity
!= DTRACE_ACTIVITY_ACTIVE
)
15887 opt
= state
->dts_options
;
15888 c
= opt
[DTRACEOPT_CPU
];
15890 if (c
!= DTRACE_CPUALL
&& c
!= DTRACEOPT_UNSET
&& c
!= cpu
)
15894 * Regardless of what the actual policy is, we're going to
15895 * temporarily set our resize policy to be manual. We're
15896 * also going to temporarily set our CPU option to denote
15897 * the newly configured CPU.
15899 rs
= opt
[DTRACEOPT_BUFRESIZE
];
15900 opt
[DTRACEOPT_BUFRESIZE
] = DTRACEOPT_BUFRESIZE_MANUAL
;
15901 opt
[DTRACEOPT_CPU
] = (dtrace_optval_t
)cpu
;
15903 (void) dtrace_state_buffers(state
);
15905 opt
[DTRACEOPT_BUFRESIZE
] = rs
;
15906 opt
[DTRACEOPT_CPU
] = c
;
15913 * We don't free the buffer in the CPU_UNCONFIG case. (The
15914 * buffer will be freed when the consumer exits.)
15922 mutex_exit(&dtrace_lock
);
15927 dtrace_cpu_setup_initial(processorid_t cpu
)
15929 (void) dtrace_cpu_setup(CPU_CONFIG
, cpu
);
15933 dtrace_toxrange_add(uintptr_t base
, uintptr_t limit
)
15935 if (dtrace_toxranges
>= dtrace_toxranges_max
) {
15937 dtrace_toxrange_t
*range
;
15939 osize
= dtrace_toxranges_max
* sizeof (dtrace_toxrange_t
);
15942 ASSERT(dtrace_toxrange
== NULL
);
15943 ASSERT(dtrace_toxranges_max
== 0);
15944 dtrace_toxranges_max
= 1;
15946 dtrace_toxranges_max
<<= 1;
15949 nsize
= dtrace_toxranges_max
* sizeof (dtrace_toxrange_t
);
15950 range
= kmem_zalloc(nsize
, KM_SLEEP
);
15952 if (dtrace_toxrange
!= NULL
) {
15953 ASSERT(osize
!= 0);
15954 bcopy(dtrace_toxrange
, range
, osize
);
15955 kmem_free(dtrace_toxrange
, osize
);
15958 dtrace_toxrange
= range
;
15961 ASSERT(dtrace_toxrange
[dtrace_toxranges
].dtt_base
== NULL
);
15962 ASSERT(dtrace_toxrange
[dtrace_toxranges
].dtt_limit
== NULL
);
15964 dtrace_toxrange
[dtrace_toxranges
].dtt_base
= base
;
15965 dtrace_toxrange
[dtrace_toxranges
].dtt_limit
= limit
;
15966 dtrace_toxranges
++;
15970 dtrace_getf_barrier()
15973 * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
15974 * that contain calls to getf(), this routine will be called on every
15975 * closef() before either the underlying vnode is released or the
15976 * file_t itself is freed. By the time we are here, it is essential
15977 * that the file_t can no longer be accessed from a call to getf()
15978 * in probe context -- that assures that a dtrace_sync() can be used
15979 * to clear out any enablings referring to the old structures.
15981 if (curthread
->t_procp
->p_zone
->zone_dtrace_getf
!= 0 ||
15982 kcred
->cr_zone
->zone_dtrace_getf
!= 0)
15987 * DTrace Driver Cookbook Functions
15991 dtrace_attach(dev_info_t
*devi
, ddi_attach_cmd_t cmd
)
15993 dtrace_provider_id_t id
;
15994 dtrace_state_t
*state
= NULL
;
15995 dtrace_enabling_t
*enab
;
15997 mutex_enter(&cpu_lock
);
15998 mutex_enter(&dtrace_provider_lock
);
15999 mutex_enter(&dtrace_lock
);
16001 if (ddi_soft_state_init(&dtrace_softstate
,
16002 sizeof (dtrace_state_t
), 0) != 0) {
16003 cmn_err(CE_NOTE
, "/dev/dtrace failed to initialize soft state");
16004 mutex_exit(&cpu_lock
);
16005 mutex_exit(&dtrace_provider_lock
);
16006 mutex_exit(&dtrace_lock
);
16007 return (DDI_FAILURE
);
16010 if (ddi_create_minor_node(devi
, DTRACEMNR_DTRACE
, S_IFCHR
,
16011 DTRACEMNRN_DTRACE
, DDI_PSEUDO
, NULL
) == DDI_FAILURE
||
16012 ddi_create_minor_node(devi
, DTRACEMNR_HELPER
, S_IFCHR
,
16013 DTRACEMNRN_HELPER
, DDI_PSEUDO
, NULL
) == DDI_FAILURE
) {
16014 cmn_err(CE_NOTE
, "/dev/dtrace couldn't create minor nodes");
16015 ddi_remove_minor_node(devi
, NULL
);
16016 ddi_soft_state_fini(&dtrace_softstate
);
16017 mutex_exit(&cpu_lock
);
16018 mutex_exit(&dtrace_provider_lock
);
16019 mutex_exit(&dtrace_lock
);
16020 return (DDI_FAILURE
);
16023 ddi_report_dev(devi
);
16024 dtrace_devi
= devi
;
16026 dtrace_modload
= dtrace_module_loaded
;
16027 dtrace_modunload
= dtrace_module_unloaded
;
16028 dtrace_cpu_init
= dtrace_cpu_setup_initial
;
16029 dtrace_helpers_cleanup
= dtrace_helpers_destroy
;
16030 dtrace_helpers_fork
= dtrace_helpers_duplicate
;
16031 dtrace_cpustart_init
= dtrace_suspend
;
16032 dtrace_cpustart_fini
= dtrace_resume
;
16033 dtrace_debugger_init
= dtrace_suspend
;
16034 dtrace_debugger_fini
= dtrace_resume
;
16036 register_cpu_setup_func((cpu_setup_func_t
*)dtrace_cpu_setup
, NULL
);
16038 ASSERT(MUTEX_HELD(&cpu_lock
));
16040 dtrace_arena
= vmem_create("dtrace", (void *)1, UINT32_MAX
, 1,
16041 NULL
, NULL
, NULL
, 0, VM_SLEEP
| VMC_IDENTIFIER
);
16042 dtrace_minor
= vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE
,
16043 UINT32_MAX
- DTRACEMNRN_CLONE
, 1, NULL
, NULL
, NULL
, 0,
16044 VM_SLEEP
| VMC_IDENTIFIER
);
16045 dtrace_taskq
= taskq_create("dtrace_taskq", 1, maxclsyspri
,
16048 dtrace_state_cache
= kmem_cache_create("dtrace_state_cache",
16049 sizeof (dtrace_dstate_percpu_t
) * NCPU
, DTRACE_STATE_ALIGN
,
16050 NULL
, NULL
, NULL
, NULL
, NULL
, 0);
16052 ASSERT(MUTEX_HELD(&cpu_lock
));
16053 dtrace_bymod
= dtrace_hash_create(offsetof(dtrace_probe_t
, dtpr_mod
),
16054 offsetof(dtrace_probe_t
, dtpr_nextmod
),
16055 offsetof(dtrace_probe_t
, dtpr_prevmod
));
16057 dtrace_byfunc
= dtrace_hash_create(offsetof(dtrace_probe_t
, dtpr_func
),
16058 offsetof(dtrace_probe_t
, dtpr_nextfunc
),
16059 offsetof(dtrace_probe_t
, dtpr_prevfunc
));
16061 dtrace_byname
= dtrace_hash_create(offsetof(dtrace_probe_t
, dtpr_name
),
16062 offsetof(dtrace_probe_t
, dtpr_nextname
),
16063 offsetof(dtrace_probe_t
, dtpr_prevname
));
16065 if (dtrace_retain_max
< 1) {
16066 cmn_err(CE_WARN
, "illegal value (%lu) for dtrace_retain_max; "
16067 "setting to 1", dtrace_retain_max
);
16068 dtrace_retain_max
= 1;
16072 * Now discover our toxic ranges.
16074 dtrace_toxic_ranges(dtrace_toxrange_add
);
16077 * Before we register ourselves as a provider to our own framework,
16078 * we would like to assert that dtrace_provider is NULL -- but that's
16079 * not true if we were loaded as a dependency of a DTrace provider.
16080 * Once we've registered, we can assert that dtrace_provider is our
16083 (void) dtrace_register("dtrace", &dtrace_provider_attr
,
16084 DTRACE_PRIV_NONE
, 0, &dtrace_provider_ops
, NULL
, &id
);
16086 ASSERT(dtrace_provider
!= NULL
);
16087 ASSERT((dtrace_provider_id_t
)dtrace_provider
== id
);
16089 dtrace_probeid_begin
= dtrace_probe_create((dtrace_provider_id_t
)
16090 dtrace_provider
, NULL
, NULL
, "BEGIN", 0, NULL
);
16091 dtrace_probeid_end
= dtrace_probe_create((dtrace_provider_id_t
)
16092 dtrace_provider
, NULL
, NULL
, "END", 0, NULL
);
16093 dtrace_probeid_error
= dtrace_probe_create((dtrace_provider_id_t
)
16094 dtrace_provider
, NULL
, NULL
, "ERROR", 1, NULL
);
16096 dtrace_anon_property();
16097 mutex_exit(&cpu_lock
);
16100 * If there are already providers, we must ask them to provide their
16101 * probes, and then match any anonymous enabling against them. Note
16102 * that there should be no other retained enablings at this time:
16103 * the only retained enablings at this time should be the anonymous
16106 if (dtrace_anon
.dta_enabling
!= NULL
) {
16107 ASSERT(dtrace_retained
== dtrace_anon
.dta_enabling
);
16109 dtrace_enabling_provide(NULL
);
16110 state
= dtrace_anon
.dta_state
;
16113 * We couldn't hold cpu_lock across the above call to
16114 * dtrace_enabling_provide(), but we must hold it to actually
16115 * enable the probes. We have to drop all of our locks, pick
16116 * up cpu_lock, and regain our locks before matching the
16117 * retained anonymous enabling.
16119 mutex_exit(&dtrace_lock
);
16120 mutex_exit(&dtrace_provider_lock
);
16122 mutex_enter(&cpu_lock
);
16123 mutex_enter(&dtrace_provider_lock
);
16124 mutex_enter(&dtrace_lock
);
16126 if ((enab
= dtrace_anon
.dta_enabling
) != NULL
)
16127 (void) dtrace_enabling_match(enab
, NULL
);
16129 mutex_exit(&cpu_lock
);
16132 mutex_exit(&dtrace_lock
);
16133 mutex_exit(&dtrace_provider_lock
);
16135 if (state
!= NULL
) {
16137 * If we created any anonymous state, set it going now.
16139 (void) dtrace_state_go(state
, &dtrace_anon
.dta_beganon
);
16142 return (DDI_SUCCESS
);
16147 dtrace_open(dev_t
*devp
, int flag
, int otyp
, cred_t
*cred_p
)
16149 dtrace_state_t
*state
;
16154 if (getminor(*devp
) == DTRACEMNRN_HELPER
)
16158 * If this wasn't an open with the "helper" minor, then it must be
16159 * the "dtrace" minor.
16161 if (getminor(*devp
) != DTRACEMNRN_DTRACE
)
16165 * If no DTRACE_PRIV_* bits are set in the credential, then the
16166 * caller lacks sufficient permission to do anything with DTrace.
16168 dtrace_cred2priv(cred_p
, &priv
, &uid
, &zoneid
);
16169 if (priv
== DTRACE_PRIV_NONE
)
16173 * Ask all providers to provide all their probes.
16175 mutex_enter(&dtrace_provider_lock
);
16176 dtrace_probe_provide(NULL
, NULL
);
16177 mutex_exit(&dtrace_provider_lock
);
16179 mutex_enter(&cpu_lock
);
16180 mutex_enter(&dtrace_lock
);
16182 dtrace_membar_producer();
16185 * If the kernel debugger is active (that is, if the kernel debugger
16186 * modified text in some way), we won't allow the open.
16188 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE
) != 0) {
16190 mutex_exit(&cpu_lock
);
16191 mutex_exit(&dtrace_lock
);
16195 if (dtrace_helptrace_enable
&& dtrace_helptrace_buffer
== NULL
) {
16197 * If DTrace helper tracing is enabled, we need to allocate the
16198 * trace buffer and initialize the values.
16200 dtrace_helptrace_buffer
=
16201 kmem_zalloc(dtrace_helptrace_bufsize
, KM_SLEEP
);
16202 dtrace_helptrace_next
= 0;
16203 dtrace_helptrace_wrapped
= 0;
16204 dtrace_helptrace_enable
= 0;
16207 state
= dtrace_state_create(devp
, cred_p
);
16208 mutex_exit(&cpu_lock
);
16210 if (state
== NULL
) {
16211 if (--dtrace_opens
== 0 && dtrace_anon
.dta_enabling
== NULL
)
16212 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE
);
16213 mutex_exit(&dtrace_lock
);
16217 mutex_exit(&dtrace_lock
);
16224 dtrace_close(dev_t dev
, int flag
, int otyp
, cred_t
*cred_p
)
16226 minor_t minor
= getminor(dev
);
16227 dtrace_state_t
*state
;
16228 dtrace_helptrace_t
*buf
= NULL
;
16230 if (minor
== DTRACEMNRN_HELPER
)
16233 state
= ddi_get_soft_state(dtrace_softstate
, minor
);
16235 mutex_enter(&cpu_lock
);
16236 mutex_enter(&dtrace_lock
);
16238 if (state
->dts_anon
) {
16240 * There is anonymous state. Destroy that first.
16242 ASSERT(dtrace_anon
.dta_state
== NULL
);
16243 dtrace_state_destroy(state
->dts_anon
);
16246 if (dtrace_helptrace_disable
) {
16248 * If we have been told to disable helper tracing, set the
16249 * buffer to NULL before calling into dtrace_state_destroy();
16250 * we take advantage of its dtrace_sync() to know that no
16251 * CPU is in probe context with enabled helper tracing
16252 * after it returns.
16254 buf
= dtrace_helptrace_buffer
;
16255 dtrace_helptrace_buffer
= NULL
;
16258 dtrace_state_destroy(state
);
16259 ASSERT(dtrace_opens
> 0);
16262 * Only relinquish control of the kernel debugger interface when there
16263 * are no consumers and no anonymous enablings.
16265 if (--dtrace_opens
== 0 && dtrace_anon
.dta_enabling
== NULL
)
16266 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE
);
16269 kmem_free(buf
, dtrace_helptrace_bufsize
);
16270 dtrace_helptrace_disable
= 0;
16273 mutex_exit(&dtrace_lock
);
16274 mutex_exit(&cpu_lock
);
16281 dtrace_ioctl_helper(int cmd
, intptr_t arg
, int *rv
)
16284 dof_helper_t help
, *dhp
= NULL
;
16287 case DTRACEHIOC_ADDDOF
:
16288 if (copyin((void *)arg
, &help
, sizeof (help
)) != 0) {
16289 dtrace_dof_error(NULL
, "failed to copyin DOF helper");
16294 arg
= (intptr_t)help
.dofhp_dof
;
16297 case DTRACEHIOC_ADD
: {
16298 dof_hdr_t
*dof
= dtrace_dof_copyin(arg
, &rval
);
16303 mutex_enter(&dtrace_lock
);
16306 * dtrace_helper_slurp() takes responsibility for the dof --
16307 * it may free it now or it may save it and free it later.
16309 if ((rval
= dtrace_helper_slurp(dof
, dhp
)) != -1) {
16316 mutex_exit(&dtrace_lock
);
16320 case DTRACEHIOC_REMOVE
: {
16321 mutex_enter(&dtrace_lock
);
16322 rval
= dtrace_helper_destroygen(arg
);
16323 mutex_exit(&dtrace_lock
);
16337 dtrace_ioctl(dev_t dev
, int cmd
, intptr_t arg
, int md
, cred_t
*cr
, int *rv
)
16339 minor_t minor
= getminor(dev
);
16340 dtrace_state_t
*state
;
16343 if (minor
== DTRACEMNRN_HELPER
)
16344 return (dtrace_ioctl_helper(cmd
, arg
, rv
));
16346 state
= ddi_get_soft_state(dtrace_softstate
, minor
);
16348 if (state
->dts_anon
) {
16349 ASSERT(dtrace_anon
.dta_state
== NULL
);
16350 state
= state
->dts_anon
;
16354 case DTRACEIOC_PROVIDER
: {
16355 dtrace_providerdesc_t pvd
;
16356 dtrace_provider_t
*pvp
;
16358 if (copyin((void *)arg
, &pvd
, sizeof (pvd
)) != 0)
16361 pvd
.dtvd_name
[DTRACE_PROVNAMELEN
- 1] = '\0';
16362 mutex_enter(&dtrace_provider_lock
);
16364 for (pvp
= dtrace_provider
; pvp
!= NULL
; pvp
= pvp
->dtpv_next
) {
16365 if (strcmp(pvp
->dtpv_name
, pvd
.dtvd_name
) == 0)
16369 mutex_exit(&dtrace_provider_lock
);
16374 bcopy(&pvp
->dtpv_priv
, &pvd
.dtvd_priv
, sizeof (dtrace_ppriv_t
));
16375 bcopy(&pvp
->dtpv_attr
, &pvd
.dtvd_attr
, sizeof (dtrace_pattr_t
));
16376 if (copyout(&pvd
, (void *)arg
, sizeof (pvd
)) != 0)
16382 case DTRACEIOC_EPROBE
: {
16383 dtrace_eprobedesc_t epdesc
;
16385 dtrace_action_t
*act
;
16391 if (copyin((void *)arg
, &epdesc
, sizeof (epdesc
)) != 0)
16394 mutex_enter(&dtrace_lock
);
16396 if ((ecb
= dtrace_epid2ecb(state
, epdesc
.dtepd_epid
)) == NULL
) {
16397 mutex_exit(&dtrace_lock
);
16401 if (ecb
->dte_probe
== NULL
) {
16402 mutex_exit(&dtrace_lock
);
16406 epdesc
.dtepd_probeid
= ecb
->dte_probe
->dtpr_id
;
16407 epdesc
.dtepd_uarg
= ecb
->dte_uarg
;
16408 epdesc
.dtepd_size
= ecb
->dte_size
;
16410 nrecs
= epdesc
.dtepd_nrecs
;
16411 epdesc
.dtepd_nrecs
= 0;
16412 for (act
= ecb
->dte_action
; act
!= NULL
; act
= act
->dta_next
) {
16413 if (DTRACEACT_ISAGG(act
->dta_kind
) || act
->dta_intuple
)
16416 epdesc
.dtepd_nrecs
++;
16420 * Now that we have the size, we need to allocate a temporary
16421 * buffer in which to store the complete description. We need
16422 * the temporary buffer to be able to drop dtrace_lock()
16423 * across the copyout(), below.
16425 size
= sizeof (dtrace_eprobedesc_t
) +
16426 (epdesc
.dtepd_nrecs
* sizeof (dtrace_recdesc_t
));
16428 buf
= kmem_alloc(size
, KM_SLEEP
);
16429 dest
= (uintptr_t)buf
;
16431 bcopy(&epdesc
, (void *)dest
, sizeof (epdesc
));
16432 dest
+= offsetof(dtrace_eprobedesc_t
, dtepd_rec
[0]);
16434 for (act
= ecb
->dte_action
; act
!= NULL
; act
= act
->dta_next
) {
16435 if (DTRACEACT_ISAGG(act
->dta_kind
) || act
->dta_intuple
)
16441 bcopy(&act
->dta_rec
, (void *)dest
,
16442 sizeof (dtrace_recdesc_t
));
16443 dest
+= sizeof (dtrace_recdesc_t
);
16446 mutex_exit(&dtrace_lock
);
16448 if (copyout(buf
, (void *)arg
, dest
- (uintptr_t)buf
) != 0) {
16449 kmem_free(buf
, size
);
16453 kmem_free(buf
, size
);
16457 case DTRACEIOC_AGGDESC
: {
16458 dtrace_aggdesc_t aggdesc
;
16459 dtrace_action_t
*act
;
16460 dtrace_aggregation_t
*agg
;
16463 dtrace_recdesc_t
*lrec
;
16468 if (copyin((void *)arg
, &aggdesc
, sizeof (aggdesc
)) != 0)
16471 mutex_enter(&dtrace_lock
);
16473 if ((agg
= dtrace_aggid2agg(state
, aggdesc
.dtagd_id
)) == NULL
) {
16474 mutex_exit(&dtrace_lock
);
16478 aggdesc
.dtagd_epid
= agg
->dtag_ecb
->dte_epid
;
16480 nrecs
= aggdesc
.dtagd_nrecs
;
16481 aggdesc
.dtagd_nrecs
= 0;
16483 offs
= agg
->dtag_base
;
16484 lrec
= &agg
->dtag_action
.dta_rec
;
16485 aggdesc
.dtagd_size
= lrec
->dtrd_offset
+ lrec
->dtrd_size
- offs
;
16487 for (act
= agg
->dtag_first
; ; act
= act
->dta_next
) {
16488 ASSERT(act
->dta_intuple
||
16489 DTRACEACT_ISAGG(act
->dta_kind
));
16492 * If this action has a record size of zero, it
16493 * denotes an argument to the aggregating action.
16494 * Because the presence of this record doesn't (or
16495 * shouldn't) affect the way the data is interpreted,
16496 * we don't copy it out to save user-level the
16497 * confusion of dealing with a zero-length record.
16499 if (act
->dta_rec
.dtrd_size
== 0) {
16500 ASSERT(agg
->dtag_hasarg
);
16504 aggdesc
.dtagd_nrecs
++;
16506 if (act
== &agg
->dtag_action
)
16511 * Now that we have the size, we need to allocate a temporary
16512 * buffer in which to store the complete description. We need
16513 * the temporary buffer to be able to drop dtrace_lock()
16514 * across the copyout(), below.
16516 size
= sizeof (dtrace_aggdesc_t
) +
16517 (aggdesc
.dtagd_nrecs
* sizeof (dtrace_recdesc_t
));
16519 buf
= kmem_alloc(size
, KM_SLEEP
);
16520 dest
= (uintptr_t)buf
;
16522 bcopy(&aggdesc
, (void *)dest
, sizeof (aggdesc
));
16523 dest
+= offsetof(dtrace_aggdesc_t
, dtagd_rec
[0]);
16525 for (act
= agg
->dtag_first
; ; act
= act
->dta_next
) {
16526 dtrace_recdesc_t rec
= act
->dta_rec
;
16529 * See the comment in the above loop for why we pass
16530 * over zero-length records.
16532 if (rec
.dtrd_size
== 0) {
16533 ASSERT(agg
->dtag_hasarg
);
16540 rec
.dtrd_offset
-= offs
;
16541 bcopy(&rec
, (void *)dest
, sizeof (rec
));
16542 dest
+= sizeof (dtrace_recdesc_t
);
16544 if (act
== &agg
->dtag_action
)
16548 mutex_exit(&dtrace_lock
);
16550 if (copyout(buf
, (void *)arg
, dest
- (uintptr_t)buf
) != 0) {
16551 kmem_free(buf
, size
);
16555 kmem_free(buf
, size
);
16559 case DTRACEIOC_ENABLE
: {
16561 dtrace_enabling_t
*enab
= NULL
;
16562 dtrace_vstate_t
*vstate
;
16568 * If a NULL argument has been passed, we take this as our
16569 * cue to reevaluate our enablings.
16572 dtrace_enabling_matchall();
16577 if ((dof
= dtrace_dof_copyin(arg
, &rval
)) == NULL
)
16580 mutex_enter(&cpu_lock
);
16581 mutex_enter(&dtrace_lock
);
16582 vstate
= &state
->dts_vstate
;
16584 if (state
->dts_activity
!= DTRACE_ACTIVITY_INACTIVE
) {
16585 mutex_exit(&dtrace_lock
);
16586 mutex_exit(&cpu_lock
);
16587 dtrace_dof_destroy(dof
);
16591 if (dtrace_dof_slurp(dof
, vstate
, cr
, &enab
, 0, B_TRUE
) != 0) {
16592 mutex_exit(&dtrace_lock
);
16593 mutex_exit(&cpu_lock
);
16594 dtrace_dof_destroy(dof
);
16598 if ((rval
= dtrace_dof_options(dof
, state
)) != 0) {
16599 dtrace_enabling_destroy(enab
);
16600 mutex_exit(&dtrace_lock
);
16601 mutex_exit(&cpu_lock
);
16602 dtrace_dof_destroy(dof
);
16606 if ((err
= dtrace_enabling_match(enab
, rv
)) == 0) {
16607 err
= dtrace_enabling_retain(enab
);
16609 dtrace_enabling_destroy(enab
);
16612 mutex_exit(&cpu_lock
);
16613 mutex_exit(&dtrace_lock
);
16614 dtrace_dof_destroy(dof
);
16619 case DTRACEIOC_REPLICATE
: {
16620 dtrace_repldesc_t desc
;
16621 dtrace_probedesc_t
*match
= &desc
.dtrpd_match
;
16622 dtrace_probedesc_t
*create
= &desc
.dtrpd_create
;
16625 if (copyin((void *)arg
, &desc
, sizeof (desc
)) != 0)
16628 match
->dtpd_provider
[DTRACE_PROVNAMELEN
- 1] = '\0';
16629 match
->dtpd_mod
[DTRACE_MODNAMELEN
- 1] = '\0';
16630 match
->dtpd_func
[DTRACE_FUNCNAMELEN
- 1] = '\0';
16631 match
->dtpd_name
[DTRACE_NAMELEN
- 1] = '\0';
16633 create
->dtpd_provider
[DTRACE_PROVNAMELEN
- 1] = '\0';
16634 create
->dtpd_mod
[DTRACE_MODNAMELEN
- 1] = '\0';
16635 create
->dtpd_func
[DTRACE_FUNCNAMELEN
- 1] = '\0';
16636 create
->dtpd_name
[DTRACE_NAMELEN
- 1] = '\0';
16638 mutex_enter(&dtrace_lock
);
16639 err
= dtrace_enabling_replicate(state
, match
, create
);
16640 mutex_exit(&dtrace_lock
);
16645 case DTRACEIOC_PROBEMATCH
:
16646 case DTRACEIOC_PROBES
: {
16647 dtrace_probe_t
*probe
= NULL
;
16648 dtrace_probedesc_t desc
;
16649 dtrace_probekey_t pkey
;
16656 if (copyin((void *)arg
, &desc
, sizeof (desc
)) != 0)
16659 desc
.dtpd_provider
[DTRACE_PROVNAMELEN
- 1] = '\0';
16660 desc
.dtpd_mod
[DTRACE_MODNAMELEN
- 1] = '\0';
16661 desc
.dtpd_func
[DTRACE_FUNCNAMELEN
- 1] = '\0';
16662 desc
.dtpd_name
[DTRACE_NAMELEN
- 1] = '\0';
16665 * Before we attempt to match this probe, we want to give
16666 * all providers the opportunity to provide it.
16668 if (desc
.dtpd_id
== DTRACE_IDNONE
) {
16669 mutex_enter(&dtrace_provider_lock
);
16670 dtrace_probe_provide(&desc
, NULL
);
16671 mutex_exit(&dtrace_provider_lock
);
16675 if (cmd
== DTRACEIOC_PROBEMATCH
) {
16676 dtrace_probekey(&desc
, &pkey
);
16677 pkey
.dtpk_id
= DTRACE_IDNONE
;
16680 dtrace_cred2priv(cr
, &priv
, &uid
, &zoneid
);
16682 mutex_enter(&dtrace_lock
);
16684 if (cmd
== DTRACEIOC_PROBEMATCH
) {
16685 for (i
= desc
.dtpd_id
; i
<= dtrace_nprobes
; i
++) {
16686 if ((probe
= dtrace_probes
[i
- 1]) != NULL
&&
16687 (m
= dtrace_match_probe(probe
, &pkey
,
16688 priv
, uid
, zoneid
)) != 0)
16693 mutex_exit(&dtrace_lock
);
16698 for (i
= desc
.dtpd_id
; i
<= dtrace_nprobes
; i
++) {
16699 if ((probe
= dtrace_probes
[i
- 1]) != NULL
&&
16700 dtrace_match_priv(probe
, priv
, uid
, zoneid
))
16705 if (probe
== NULL
) {
16706 mutex_exit(&dtrace_lock
);
16710 dtrace_probe_description(probe
, &desc
);
16711 mutex_exit(&dtrace_lock
);
16713 if (copyout(&desc
, (void *)arg
, sizeof (desc
)) != 0)
16719 case DTRACEIOC_PROBEARG
: {
16720 dtrace_argdesc_t desc
;
16721 dtrace_probe_t
*probe
;
16722 dtrace_provider_t
*prov
;
16724 if (copyin((void *)arg
, &desc
, sizeof (desc
)) != 0)
16727 if (desc
.dtargd_id
== DTRACE_IDNONE
)
16730 if (desc
.dtargd_ndx
== DTRACE_ARGNONE
)
16733 mutex_enter(&dtrace_provider_lock
);
16734 mutex_enter(&mod_lock
);
16735 mutex_enter(&dtrace_lock
);
16737 if (desc
.dtargd_id
> dtrace_nprobes
) {
16738 mutex_exit(&dtrace_lock
);
16739 mutex_exit(&mod_lock
);
16740 mutex_exit(&dtrace_provider_lock
);
16744 if ((probe
= dtrace_probes
[desc
.dtargd_id
- 1]) == NULL
) {
16745 mutex_exit(&dtrace_lock
);
16746 mutex_exit(&mod_lock
);
16747 mutex_exit(&dtrace_provider_lock
);
16751 mutex_exit(&dtrace_lock
);
16753 prov
= probe
->dtpr_provider
;
16755 if (prov
->dtpv_pops
.dtps_getargdesc
== NULL
) {
16757 * There isn't any typed information for this probe.
16758 * Set the argument number to DTRACE_ARGNONE.
16760 desc
.dtargd_ndx
= DTRACE_ARGNONE
;
16762 desc
.dtargd_native
[0] = '\0';
16763 desc
.dtargd_xlate
[0] = '\0';
16764 desc
.dtargd_mapping
= desc
.dtargd_ndx
;
16766 prov
->dtpv_pops
.dtps_getargdesc(prov
->dtpv_arg
,
16767 probe
->dtpr_id
, probe
->dtpr_arg
, &desc
);
16770 mutex_exit(&mod_lock
);
16771 mutex_exit(&dtrace_provider_lock
);
16773 if (copyout(&desc
, (void *)arg
, sizeof (desc
)) != 0)
16779 case DTRACEIOC_GO
: {
16780 processorid_t cpuid
;
16781 rval
= dtrace_state_go(state
, &cpuid
);
16786 if (copyout(&cpuid
, (void *)arg
, sizeof (cpuid
)) != 0)
16792 case DTRACEIOC_STOP
: {
16793 processorid_t cpuid
;
16795 mutex_enter(&dtrace_lock
);
16796 rval
= dtrace_state_stop(state
, &cpuid
);
16797 mutex_exit(&dtrace_lock
);
16802 if (copyout(&cpuid
, (void *)arg
, sizeof (cpuid
)) != 0)
16808 case DTRACEIOC_DOFGET
: {
16809 dof_hdr_t hdr
, *dof
;
16812 if (copyin((void *)arg
, &hdr
, sizeof (hdr
)) != 0)
16815 mutex_enter(&dtrace_lock
);
16816 dof
= dtrace_dof_create(state
);
16817 mutex_exit(&dtrace_lock
);
16819 len
= MIN(hdr
.dofh_loadsz
, dof
->dofh_loadsz
);
16820 rval
= copyout(dof
, (void *)arg
, len
);
16821 dtrace_dof_destroy(dof
);
16823 return (rval
== 0 ? 0 : EFAULT
);
16826 case DTRACEIOC_AGGSNAP
:
16827 case DTRACEIOC_BUFSNAP
: {
16828 dtrace_bufdesc_t desc
;
16830 dtrace_buffer_t
*buf
;
16832 if (copyin((void *)arg
, &desc
, sizeof (desc
)) != 0)
16835 if (desc
.dtbd_cpu
< 0 || desc
.dtbd_cpu
>= NCPU
)
16838 mutex_enter(&dtrace_lock
);
16840 if (cmd
== DTRACEIOC_BUFSNAP
) {
16841 buf
= &state
->dts_buffer
[desc
.dtbd_cpu
];
16843 buf
= &state
->dts_aggbuffer
[desc
.dtbd_cpu
];
16846 if (buf
->dtb_flags
& (DTRACEBUF_RING
| DTRACEBUF_FILL
)) {
16847 size_t sz
= buf
->dtb_offset
;
16849 if (state
->dts_activity
!= DTRACE_ACTIVITY_STOPPED
) {
16850 mutex_exit(&dtrace_lock
);
16855 * If this buffer has already been consumed, we're
16856 * going to indicate that there's nothing left here
16859 if (buf
->dtb_flags
& DTRACEBUF_CONSUMED
) {
16860 mutex_exit(&dtrace_lock
);
16862 desc
.dtbd_size
= 0;
16863 desc
.dtbd_drops
= 0;
16864 desc
.dtbd_errors
= 0;
16865 desc
.dtbd_oldest
= 0;
16866 sz
= sizeof (desc
);
16868 if (copyout(&desc
, (void *)arg
, sz
) != 0)
16875 * If this is a ring buffer that has wrapped, we want
16876 * to copy the whole thing out.
16878 if (buf
->dtb_flags
& DTRACEBUF_WRAPPED
) {
16879 dtrace_buffer_polish(buf
);
16880 sz
= buf
->dtb_size
;
16883 if (copyout(buf
->dtb_tomax
, desc
.dtbd_data
, sz
) != 0) {
16884 mutex_exit(&dtrace_lock
);
16888 desc
.dtbd_size
= sz
;
16889 desc
.dtbd_drops
= buf
->dtb_drops
;
16890 desc
.dtbd_errors
= buf
->dtb_errors
;
16891 desc
.dtbd_oldest
= buf
->dtb_xamot_offset
;
16892 desc
.dtbd_timestamp
= dtrace_gethrtime();
16894 mutex_exit(&dtrace_lock
);
16896 if (copyout(&desc
, (void *)arg
, sizeof (desc
)) != 0)
16899 buf
->dtb_flags
|= DTRACEBUF_CONSUMED
;
16904 if (buf
->dtb_tomax
== NULL
) {
16905 ASSERT(buf
->dtb_xamot
== NULL
);
16906 mutex_exit(&dtrace_lock
);
16910 cached
= buf
->dtb_tomax
;
16911 ASSERT(!(buf
->dtb_flags
& DTRACEBUF_NOSWITCH
));
16913 dtrace_xcall(desc
.dtbd_cpu
,
16914 (dtrace_xcall_t
)dtrace_buffer_switch
, buf
);
16916 state
->dts_errors
+= buf
->dtb_xamot_errors
;
16919 * If the buffers did not actually switch, then the cross call
16920 * did not take place -- presumably because the given CPU is
16921 * not in the ready set. If this is the case, we'll return
16924 if (buf
->dtb_tomax
== cached
) {
16925 ASSERT(buf
->dtb_xamot
!= cached
);
16926 mutex_exit(&dtrace_lock
);
16930 ASSERT(cached
== buf
->dtb_xamot
);
16933 * We have our snapshot; now copy it out.
16935 if (copyout(buf
->dtb_xamot
, desc
.dtbd_data
,
16936 buf
->dtb_xamot_offset
) != 0) {
16937 mutex_exit(&dtrace_lock
);
16941 desc
.dtbd_size
= buf
->dtb_xamot_offset
;
16942 desc
.dtbd_drops
= buf
->dtb_xamot_drops
;
16943 desc
.dtbd_errors
= buf
->dtb_xamot_errors
;
16944 desc
.dtbd_oldest
= 0;
16945 desc
.dtbd_timestamp
= buf
->dtb_switched
;
16947 mutex_exit(&dtrace_lock
);
16950 * Finally, copy out the buffer description.
16952 if (copyout(&desc
, (void *)arg
, sizeof (desc
)) != 0)
16958 case DTRACEIOC_CONF
: {
16959 dtrace_conf_t conf
;
16961 bzero(&conf
, sizeof (conf
));
16962 conf
.dtc_difversion
= DIF_VERSION
;
16963 conf
.dtc_difintregs
= DIF_DIR_NREGS
;
16964 conf
.dtc_diftupregs
= DIF_DTR_NREGS
;
16965 conf
.dtc_ctfmodel
= CTF_MODEL_NATIVE
;
16967 if (copyout(&conf
, (void *)arg
, sizeof (conf
)) != 0)
16973 case DTRACEIOC_STATUS
: {
16974 dtrace_status_t stat
;
16975 dtrace_dstate_t
*dstate
;
16980 * See the comment in dtrace_state_deadman() for the reason
16981 * for setting dts_laststatus to INT64_MAX before setting
16982 * it to the correct value.
16984 state
->dts_laststatus
= INT64_MAX
;
16985 dtrace_membar_producer();
16986 state
->dts_laststatus
= dtrace_gethrtime();
16988 bzero(&stat
, sizeof (stat
));
16990 mutex_enter(&dtrace_lock
);
16992 if (state
->dts_activity
== DTRACE_ACTIVITY_INACTIVE
) {
16993 mutex_exit(&dtrace_lock
);
16997 if (state
->dts_activity
== DTRACE_ACTIVITY_DRAINING
)
16998 stat
.dtst_exiting
= 1;
17000 nerrs
= state
->dts_errors
;
17001 dstate
= &state
->dts_vstate
.dtvs_dynvars
;
17003 for (i
= 0; i
< NCPU
; i
++) {
17004 dtrace_dstate_percpu_t
*dcpu
= &dstate
->dtds_percpu
[i
];
17006 stat
.dtst_dyndrops
+= dcpu
->dtdsc_drops
;
17007 stat
.dtst_dyndrops_dirty
+= dcpu
->dtdsc_dirty_drops
;
17008 stat
.dtst_dyndrops_rinsing
+= dcpu
->dtdsc_rinsing_drops
;
17010 if (state
->dts_buffer
[i
].dtb_flags
& DTRACEBUF_FULL
)
17011 stat
.dtst_filled
++;
17013 nerrs
+= state
->dts_buffer
[i
].dtb_errors
;
17015 for (j
= 0; j
< state
->dts_nspeculations
; j
++) {
17016 dtrace_speculation_t
*spec
;
17017 dtrace_buffer_t
*buf
;
17019 spec
= &state
->dts_speculations
[j
];
17020 buf
= &spec
->dtsp_buffer
[i
];
17021 stat
.dtst_specdrops
+= buf
->dtb_xamot_drops
;
17025 stat
.dtst_specdrops_busy
= state
->dts_speculations_busy
;
17026 stat
.dtst_specdrops_unavail
= state
->dts_speculations_unavail
;
17027 stat
.dtst_stkstroverflows
= state
->dts_stkstroverflows
;
17028 stat
.dtst_dblerrors
= state
->dts_dblerrors
;
17030 (state
->dts_activity
== DTRACE_ACTIVITY_KILLED
);
17031 stat
.dtst_errors
= nerrs
;
17033 mutex_exit(&dtrace_lock
);
17035 if (copyout(&stat
, (void *)arg
, sizeof (stat
)) != 0)
17041 case DTRACEIOC_FORMAT
: {
17042 dtrace_fmtdesc_t fmt
;
17046 if (copyin((void *)arg
, &fmt
, sizeof (fmt
)) != 0)
17049 mutex_enter(&dtrace_lock
);
17051 if (fmt
.dtfd_format
== 0 ||
17052 fmt
.dtfd_format
> state
->dts_nformats
) {
17053 mutex_exit(&dtrace_lock
);
17058 * Format strings are allocated contiguously and they are
17059 * never freed; if a format index is less than the number
17060 * of formats, we can assert that the format map is non-NULL
17061 * and that the format for the specified index is non-NULL.
17063 ASSERT(state
->dts_formats
!= NULL
);
17064 str
= state
->dts_formats
[fmt
.dtfd_format
- 1];
17065 ASSERT(str
!= NULL
);
17067 len
= strlen(str
) + 1;
17069 if (len
> fmt
.dtfd_length
) {
17070 fmt
.dtfd_length
= len
;
17072 if (copyout(&fmt
, (void *)arg
, sizeof (fmt
)) != 0) {
17073 mutex_exit(&dtrace_lock
);
17077 if (copyout(str
, fmt
.dtfd_string
, len
) != 0) {
17078 mutex_exit(&dtrace_lock
);
17083 mutex_exit(&dtrace_lock
);
17096 dtrace_detach(dev_info_t
*dip
, ddi_detach_cmd_t cmd
)
17098 dtrace_state_t
*state
;
17105 return (DDI_SUCCESS
);
17108 return (DDI_FAILURE
);
17111 mutex_enter(&cpu_lock
);
17112 mutex_enter(&dtrace_provider_lock
);
17113 mutex_enter(&dtrace_lock
);
17115 ASSERT(dtrace_opens
== 0);
17117 if (dtrace_helpers
> 0) {
17118 mutex_exit(&dtrace_provider_lock
);
17119 mutex_exit(&dtrace_lock
);
17120 mutex_exit(&cpu_lock
);
17121 return (DDI_FAILURE
);
17124 if (dtrace_unregister((dtrace_provider_id_t
)dtrace_provider
) != 0) {
17125 mutex_exit(&dtrace_provider_lock
);
17126 mutex_exit(&dtrace_lock
);
17127 mutex_exit(&cpu_lock
);
17128 return (DDI_FAILURE
);
17131 dtrace_provider
= NULL
;
17133 if ((state
= dtrace_anon_grab()) != NULL
) {
17135 * If there were ECBs on this state, the provider should
17136 * have not been allowed to detach; assert that there is
17139 ASSERT(state
->dts_necbs
== 0);
17140 dtrace_state_destroy(state
);
17143 * If we're being detached with anonymous state, we need to
17144 * indicate to the kernel debugger that DTrace is now inactive.
17146 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE
);
17149 bzero(&dtrace_anon
, sizeof (dtrace_anon_t
));
17150 unregister_cpu_setup_func((cpu_setup_func_t
*)dtrace_cpu_setup
, NULL
);
17151 dtrace_cpu_init
= NULL
;
17152 dtrace_helpers_cleanup
= NULL
;
17153 dtrace_helpers_fork
= NULL
;
17154 dtrace_cpustart_init
= NULL
;
17155 dtrace_cpustart_fini
= NULL
;
17156 dtrace_debugger_init
= NULL
;
17157 dtrace_debugger_fini
= NULL
;
17158 dtrace_modload
= NULL
;
17159 dtrace_modunload
= NULL
;
17161 ASSERT(dtrace_getf
== 0);
17162 ASSERT(dtrace_closef
== NULL
);
17164 mutex_exit(&cpu_lock
);
17166 kmem_free(dtrace_probes
, dtrace_nprobes
* sizeof (dtrace_probe_t
*));
17167 dtrace_probes
= NULL
;
17168 dtrace_nprobes
= 0;
17170 dtrace_hash_destroy(dtrace_bymod
);
17171 dtrace_hash_destroy(dtrace_byfunc
);
17172 dtrace_hash_destroy(dtrace_byname
);
17173 dtrace_bymod
= NULL
;
17174 dtrace_byfunc
= NULL
;
17175 dtrace_byname
= NULL
;
17177 kmem_cache_destroy(dtrace_state_cache
);
17178 vmem_destroy(dtrace_minor
);
17179 vmem_destroy(dtrace_arena
);
17181 if (dtrace_toxrange
!= NULL
) {
17182 kmem_free(dtrace_toxrange
,
17183 dtrace_toxranges_max
* sizeof (dtrace_toxrange_t
));
17184 dtrace_toxrange
= NULL
;
17185 dtrace_toxranges
= 0;
17186 dtrace_toxranges_max
= 0;
17189 ddi_remove_minor_node(dtrace_devi
, NULL
);
17190 dtrace_devi
= NULL
;
17192 ddi_soft_state_fini(&dtrace_softstate
);
17194 ASSERT(dtrace_vtime_references
== 0);
17195 ASSERT(dtrace_opens
== 0);
17196 ASSERT(dtrace_retained
== NULL
);
17198 mutex_exit(&dtrace_lock
);
17199 mutex_exit(&dtrace_provider_lock
);
17202 * We don't destroy the task queue until after we have dropped our
17203 * locks (taskq_destroy() may block on running tasks). To prevent
17204 * attempting to do work after we have effectively detached but before
17205 * the task queue has been destroyed, all tasks dispatched via the
17206 * task queue must check that DTrace is still attached before
17207 * performing any operation.
17209 taskq_destroy(dtrace_taskq
);
17210 dtrace_taskq
= NULL
;
17212 return (DDI_SUCCESS
);
17217 dtrace_info(dev_info_t
*dip
, ddi_info_cmd_t infocmd
, void *arg
, void **result
)
17222 case DDI_INFO_DEVT2DEVINFO
:
17223 *result
= (void *)dtrace_devi
;
17224 error
= DDI_SUCCESS
;
17226 case DDI_INFO_DEVT2INSTANCE
:
17227 *result
= (void *)0;
17228 error
= DDI_SUCCESS
;
17231 error
= DDI_FAILURE
;
17236 static struct cb_ops dtrace_cb_ops
= {
17237 dtrace_open
, /* open */
17238 dtrace_close
, /* close */
17239 nulldev
, /* strategy */
17240 nulldev
, /* print */
17244 dtrace_ioctl
, /* ioctl */
17245 nodev
, /* devmap */
17247 nodev
, /* segmap */
17248 nochpoll
, /* poll */
17249 ddi_prop_op
, /* cb_prop_op */
17251 D_NEW
| D_MP
/* Driver compatibility flag */
17254 static struct dev_ops dtrace_ops
= {
17255 DEVO_REV
, /* devo_rev */
17257 dtrace_info
, /* get_dev_info */
17258 nulldev
, /* identify */
17259 nulldev
, /* probe */
17260 dtrace_attach
, /* attach */
17261 dtrace_detach
, /* detach */
17263 &dtrace_cb_ops
, /* driver operations */
17264 NULL
, /* bus operations */
17265 nodev
, /* dev power */
17266 ddi_quiesce_not_needed
, /* quiesce */
17269 static struct modldrv modldrv
= {
17270 &mod_driverops
, /* module type (this is a pseudo driver) */
17271 "Dynamic Tracing", /* name of module */
17272 &dtrace_ops
, /* driver ops */
17275 static struct modlinkage modlinkage
= {
17284 return (mod_install(&modlinkage
));
17288 _info(struct modinfo
*modinfop
)
17290 return (mod_info(&modlinkage
, modinfop
));
17296 return (mod_remove(&modlinkage
));