4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2016 Joyent, Inc.
28 #include <sys/dtrace.h>
29 #include <sys/cmn_err.h>
31 #include <sys/atomic.h>
32 #include <sys/prsystm.h>
33 #include <sys/modctl.h>
34 #include <sys/aio_impl.h>
37 #include <sys/privregs.h>
40 void (*dtrace_cpu_init
)(processorid_t
);
41 void (*dtrace_modload
)(struct modctl
*);
42 void (*dtrace_modunload
)(struct modctl
*);
43 void (*dtrace_helpers_cleanup
)(proc_t
*);
44 void (*dtrace_helpers_fork
)(proc_t
*, proc_t
*);
45 void (*dtrace_cpustart_init
)(void);
46 void (*dtrace_cpustart_fini
)(void);
47 void (*dtrace_cpc_fire
)(uint64_t);
48 void (*dtrace_closef
)(void);
50 void (*dtrace_debugger_init
)(void);
51 void (*dtrace_debugger_fini
)(void);
53 dtrace_vtime_state_t dtrace_vtime_active
= 0;
54 dtrace_cacheid_t dtrace_predcache_id
= DTRACE_CACHEIDNONE
+ 1;
57 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
58 * hardware overflow interrupt handler and the kernel cpc framework to check
59 * whether or not the DTrace cpc provider is currently in use. The variable is
60 * set before counters are enabled with the first enabling and cleared when
61 * the last enabling is disabled. Its value at any given time indicates the
62 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
63 * is held during initial setting to protect races between kcpc_open() and the
64 * first enabling. The locking provided by the DTrace subsystem, the kernel
65 * cpc framework and the cpu management framework protect consumers from race
66 * conditions on enabling and disabling probes.
68 uint32_t dtrace_cpc_in_use
= 0;
70 typedef struct dtrace_hrestime
{
71 lock_t dthr_lock
; /* lock for this element */
72 timestruc_t dthr_hrestime
; /* hrestime value */
73 int64_t dthr_adj
; /* hrestime_adj value */
74 hrtime_t dthr_hrtime
; /* hrtime value */
77 static dtrace_hrestime_t dtrace_hrestime
[2];
80 * Making available adjustable high-resolution time in DTrace is regrettably
81 * more complicated than one might think it should be. The problem is that
82 * the variables related to adjusted high-resolution time (hrestime,
83 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may
84 * be held when we enter probe context. One might think that we could address
85 * this by having a single snapshot copy that is stored under a different lock
86 * from hres_tick(), using the snapshot iff hres_lock is locked in probe
87 * context. Unfortunately, this too won't work: because hres_lock is grabbed
88 * in more than just hres_tick() context, we could enter probe context
89 * concurrently on two different CPUs with both locks (hres_lock and the
90 * snapshot lock) held. As this implies, the fundamental problem is that we
91 * need to have access to a snapshot of these variables that we _know_ will
92 * not be locked in probe context. To effect this, we have two snapshots
93 * protected by two different locks, and we mandate that these snapshots are
94 * recorded in succession by a single thread calling dtrace_hres_tick(). (We
95 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls
96 * hres_tick().) A single thread can't be in two places at once: one of the
97 * snapshot locks is guaranteed to be unheld at all times. The
98 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then
99 * the other to find the unlocked snapshot.
102 dtrace_hres_tick(void)
107 for (i
= 0; i
< 2; i
++) {
108 dtrace_hrestime_t tmp
;
110 spl
= hr_clock_lock();
111 tmp
.dthr_hrestime
= hrestime
;
112 tmp
.dthr_adj
= hrestime_adj
;
113 tmp
.dthr_hrtime
= dtrace_gethrtime();
114 hr_clock_unlock(spl
);
116 lock_set(&dtrace_hrestime
[i
].dthr_lock
);
117 dtrace_hrestime
[i
].dthr_hrestime
= tmp
.dthr_hrestime
;
118 dtrace_hrestime
[i
].dthr_adj
= tmp
.dthr_adj
;
119 dtrace_hrestime
[i
].dthr_hrtime
= tmp
.dthr_hrtime
;
120 dtrace_membar_producer();
123 * To allow for lock-free examination of this lock, we use
124 * the same trick that is used hres_lock; for more details,
125 * see the description of this technique in sun4u/sys/clock.h.
127 dtrace_hrestime
[i
].dthr_lock
++;
132 dtrace_gethrestime(void)
134 dtrace_hrestime_t snap
;
136 int i
= 0, adj
, nslt
;
139 snap
.dthr_lock
= dtrace_hrestime
[i
].dthr_lock
;
140 dtrace_membar_consumer();
141 snap
.dthr_hrestime
= dtrace_hrestime
[i
].dthr_hrestime
;
142 snap
.dthr_hrtime
= dtrace_hrestime
[i
].dthr_hrtime
;
143 snap
.dthr_adj
= dtrace_hrestime
[i
].dthr_adj
;
144 dtrace_membar_consumer();
146 if ((snap
.dthr_lock
& ~1) == dtrace_hrestime
[i
].dthr_lock
)
150 * If we're here, the lock was either locked, or it
151 * transitioned while we were taking the snapshot. Either
152 * way, we're going to try the other dtrace_hrestime element;
153 * we know that it isn't possible for both to be locked
154 * simultaneously, so we will ultimately get a good snapshot.
160 * We have a good snapshot. Now perform any necessary adjustments.
162 nslt
= dtrace_gethrtime() - snap
.dthr_hrtime
;
165 now
= ((hrtime_t
)snap
.dthr_hrestime
.tv_sec
* (hrtime_t
)NANOSEC
) +
166 snap
.dthr_hrestime
.tv_nsec
;
168 if (snap
.dthr_adj
!= 0) {
169 if (snap
.dthr_adj
> 0) {
170 adj
= (nslt
>> adj_shift
);
171 if (adj
> snap
.dthr_adj
)
172 adj
= (int)snap
.dthr_adj
;
174 adj
= -(nslt
>> adj_shift
);
175 if (adj
< snap
.dthr_adj
)
176 adj
= (int)snap
.dthr_adj
;
185 dtrace_vtime_enable(void)
187 dtrace_vtime_state_t state
, nstate
;
190 state
= dtrace_vtime_active
;
193 case DTRACE_VTIME_INACTIVE
:
194 nstate
= DTRACE_VTIME_ACTIVE
;
197 case DTRACE_VTIME_INACTIVE_TNF
:
198 nstate
= DTRACE_VTIME_ACTIVE_TNF
;
201 case DTRACE_VTIME_ACTIVE
:
202 case DTRACE_VTIME_ACTIVE_TNF
:
203 panic("DTrace virtual time already enabled");
207 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active
,
208 state
, nstate
) != state
);
212 dtrace_vtime_disable(void)
214 dtrace_vtime_state_t state
, nstate
;
217 state
= dtrace_vtime_active
;
220 case DTRACE_VTIME_ACTIVE
:
221 nstate
= DTRACE_VTIME_INACTIVE
;
224 case DTRACE_VTIME_ACTIVE_TNF
:
225 nstate
= DTRACE_VTIME_INACTIVE_TNF
;
228 case DTRACE_VTIME_INACTIVE
:
229 case DTRACE_VTIME_INACTIVE_TNF
:
230 panic("DTrace virtual time already disabled");
234 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active
,
235 state
, nstate
) != state
);
239 dtrace_vtime_enable_tnf(void)
241 dtrace_vtime_state_t state
, nstate
;
244 state
= dtrace_vtime_active
;
247 case DTRACE_VTIME_ACTIVE
:
248 nstate
= DTRACE_VTIME_ACTIVE_TNF
;
251 case DTRACE_VTIME_INACTIVE
:
252 nstate
= DTRACE_VTIME_INACTIVE_TNF
;
255 case DTRACE_VTIME_ACTIVE_TNF
:
256 case DTRACE_VTIME_INACTIVE_TNF
:
257 panic("TNF already active");
261 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active
,
262 state
, nstate
) != state
);
266 dtrace_vtime_disable_tnf(void)
268 dtrace_vtime_state_t state
, nstate
;
271 state
= dtrace_vtime_active
;
274 case DTRACE_VTIME_ACTIVE_TNF
:
275 nstate
= DTRACE_VTIME_ACTIVE
;
278 case DTRACE_VTIME_INACTIVE_TNF
:
279 nstate
= DTRACE_VTIME_INACTIVE
;
282 case DTRACE_VTIME_ACTIVE
:
283 case DTRACE_VTIME_INACTIVE
:
284 panic("TNF already inactive");
288 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active
,
289 state
, nstate
) != state
);
293 dtrace_vtime_switch(kthread_t
*next
)
295 dtrace_icookie_t cookie
;
298 if (tnf_tracing_active
) {
299 tnf_thread_switch(next
);
301 if (dtrace_vtime_active
== DTRACE_VTIME_INACTIVE_TNF
)
305 cookie
= dtrace_interrupt_disable();
306 ts
= dtrace_gethrtime();
308 if (curthread
->t_dtrace_start
!= 0) {
309 curthread
->t_dtrace_vtime
+= ts
- curthread
->t_dtrace_start
;
310 curthread
->t_dtrace_start
= 0;
313 next
->t_dtrace_start
= ts
;
315 dtrace_interrupt_enable(cookie
);
318 void (*dtrace_fasttrap_fork_ptr
)(proc_t
*, proc_t
*);
319 void (*dtrace_fasttrap_exec_ptr
)(proc_t
*);
320 void (*dtrace_fasttrap_exit_ptr
)(proc_t
*);
323 * This function is called by cfork() in the event that it appears that
324 * there may be dtrace tracepoints active in the parent process's address
325 * space. This first confirms the existence of dtrace tracepoints in the
326 * parent process and calls into the fasttrap module to remove the
327 * corresponding tracepoints from the child. By knowing that there are
328 * existing tracepoints, and ensuring they can't be removed, we can rely
329 * on the fasttrap module remaining loaded.
332 dtrace_fasttrap_fork(proc_t
*p
, proc_t
*cp
)
334 ASSERT(p
->p_proc_flag
& P_PR_LOCK
);
335 ASSERT(p
->p_dtrace_count
> 0);
336 ASSERT(dtrace_fasttrap_fork_ptr
!= NULL
);
338 dtrace_fasttrap_fork_ptr(p
, cp
);