1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
6 // Based on algorithms and data structures used in
7 // http://code.google.com/p/google-perftools/.
9 // The main difference between this code and the google-perftools
10 // code is that this code is written to allow copying the profile data
11 // to an arbitrary io.Writer, while the google-perftools code always
12 // writes to an operating system file.
14 // The signal handler for the profiling clock tick adds a new stack trace
15 // to a hash table tracking counts for recent traces. Most clock ticks
16 // hit in the cache. In the event of a cache miss, an entry must be
17 // evicted from the hash table, copied to a log that will eventually be
18 // written as profile data. The google-perftools code flushed the
19 // log itself during the signal handler. This code cannot do that, because
20 // the io.Writer might block or need system calls or locks that are not
21 // safe to use from within the signal handler. Instead, we split the log
22 // into two halves and let the signal handler fill one half while a goroutine
23 // is writing out the other half. When the signal handler fills its half, it
24 // offers to swap with the goroutine. If the writer is not done with its half,
25 // we lose the stack trace for this clock tick (and record that loss).
26 // The goroutine interacts with the signal handler by calling getprofile() to
27 // get the next log piece to write, implicitly handing back the last log
30 // The state of this dance between the signal handler and the goroutine
31 // is encoded in the Profile.handoff field. If handoff == 0, then the goroutine
32 // is not using either log half and is waiting (or will soon be waiting) for
33 // a new piece by calling notesleep(&p->wait). If the signal handler
34 // changes handoff from 0 to non-zero, it must call notewakeup(&p->wait)
35 // to wake the goroutine. The value indicates the number of entries in the
36 // log half being handed off. The goroutine leaves the non-zero value in
37 // place until it has finished processing the log half and then flips the number
38 // back to zero. Setting the high bit in handoff means that the profiling is over,
39 // and the goroutine is now in charge of flushing the data left in the hash table
40 // to the log and returning that data.
42 // The handoff field is manipulated using atomic operations.
43 // For the most part, the manipulation of handoff is orderly: if handoff == 0
44 // then the signal handler owns it and can change it to non-zero.
45 // If handoff != 0 then the goroutine owns it and can change it to zero.
46 // If that were the end of the story then we would not need to manipulate
47 // handoff using atomic operations. The operations are needed, however,
48 // in order to let the log closer set the high bit to indicate "EOF" safely
49 // in the situation when normally the goroutine "owns" handoff.
55 typedef struct __go_open_array Slice
;
56 #define array __values
58 #define cap __capacity
68 typedef struct Profile Profile
;
69 typedef struct Bucket Bucket
;
70 typedef struct Entry Entry
;
75 uintptr stack
[MaxStack
];
83 bool on
; // profiling is on
84 Note wait
; // goroutine waits here
85 uintptr count
; // tick count
86 uintptr evicts
; // eviction count
87 uintptr lost
; // lost ticks that need to be logged
88 uintptr totallost
; // total lost ticks
90 // Active recent stack traces.
91 Bucket hash
[HashSize
];
93 // Log of traces evicted from hash.
94 // Signal handler has filled log[toggle][:nlog].
95 // Goroutine is writing log[1-toggle][:handoff].
96 uintptr log
[2][LogSize
/2];
102 // Writer maintains its own toggle to avoid races
103 // looking at signal handler's toggle.
105 bool wholding
; // holding & need to release a log half
106 bool flushing
; // flushing hash table - profile is over
110 static Profile
*prof
;
112 static void tick(uintptr
*, int32
);
113 static void add(Profile
*, uintptr
*, int32
);
114 static bool evict(Profile
*, Entry
*);
115 static bool flushlog(Profile
*);
118 runtime_cpuprofinit(void)
120 runtime_initlock(&lk
);
123 // LostProfileData is a no-op function used in profiles
124 // to mark the number of profiling stack traces that were
125 // discarded due to slow data writers.
126 static void LostProfileData(void) {
129 extern void runtime_SetCPUProfileRate(int32
)
130 __asm__("libgo_runtime.runtime.SetCPUProfileRate");
132 // SetCPUProfileRate sets the CPU profiling rate.
133 // The user documentation is in debug.go.
135 runtime_SetCPUProfileRate(int32 hz
)
140 // Clamp hz to something reasonable.
149 prof
= runtime_SysAlloc(sizeof *prof
);
151 runtime_printf("runtime: cpu profiling cannot allocate memory\n");
156 if(prof
->on
|| prof
->handoff
!= 0) {
157 runtime_printf("runtime: cannot set cpu profile rate until previous profile has finished.\n");
164 // pprof binary header format.
165 // http://code.google.com/p/google-perftools/source/browse/trunk/src/profiledata.cc#117
166 *p
++ = 0; // count for header
167 *p
++ = 3; // depth for header
168 *p
++ = 0; // version number
169 *p
++ = 1000000 / hz
; // period (microseconds)
171 prof
->nlog
= p
- prof
->log
[0];
173 prof
->wholding
= false;
175 prof
->flushing
= false;
176 runtime_noteclear(&prof
->wait
);
178 runtime_setcpuprofilerate(tick
, hz
);
179 } else if(prof
->on
) {
180 runtime_setcpuprofilerate(nil
, 0);
183 // Now add is not running anymore, and getprofile owns the entire log.
184 // Set the high bit in prof->handoff to tell getprofile.
188 runtime_printf("runtime: setcpuprofile(off) twice");
189 if(runtime_cas(&prof
->handoff
, n
, n
|0x80000000))
193 // we did the transition from 0 -> nonzero so we wake getprofile
194 runtime_notewakeup(&prof
->wait
);
201 tick(uintptr
*pc
, int32 n
)
206 // add adds the stack trace to the profile.
207 // It is called from signal handlers and other limited environments
208 // and cannot allocate memory or acquire locks that might be
209 // held at the time of the signal, nor can it use substantial amounts
210 // of stack. It is allowed to call evict.
212 add(Profile
*p
, uintptr
*pc
, int32 n
)
225 h
= h
<<8 | (h
>>(8*(sizeof(h
)-1)));
227 h
+= x
*31 + x
*7 + x
*3;
231 // Add to entry count if already present in table.
232 b
= &p
->hash
[h
%HashSize
];
233 for(i
=0; i
<Assoc
; i
++) {
235 if(e
->depth
!= (uintptr
)n
)
238 if(e
->stack
[j
] != pc
[j
])
245 // Evict entry with smallest count.
247 for(i
=1; i
<Assoc
; i
++)
248 if(b
->entry
[i
].count
< e
->count
)
252 // Could not evict entry. Record lost stack.
260 // Reuse the newly evicted entry.
267 // evict copies the given entry's data into the log, so that
268 // the entry can be reused. evict is called from add, which
269 // is called from the profiling signal handler, so it must not
270 // allocate memory or block. It is safe to call flushLog.
271 // evict returns true if the entry was copied to the log,
272 // false if there was no room available.
274 evict(Profile
*p
, Entry
*e
)
281 log
= p
->log
[p
->toggle
];
282 if(p
->nlog
+nslot
> nelem(p
->log
[0])) {
285 log
= p
->log
[p
->toggle
];
298 // flushlog tries to flush the current log and switch to the other one.
299 // flushlog is called from evict, called from add, called from the signal handler,
300 // so it cannot allocate memory or block. It can try to swap logs with
301 // the writing goroutine, as explained in the comment at the top of this file.
307 if(!runtime_cas(&p
->handoff
, 0, p
->nlog
))
309 runtime_notewakeup(&p
->wait
);
311 p
->toggle
= 1 - p
->toggle
;
312 log
= p
->log
[p
->toggle
];
317 *q
++ = (uintptr
)LostProfileData
;
323 // getprofile blocks until the next block of profiling data is available
324 // and returns it as a []byte. It is called from the writing goroutine.
326 getprofile(Profile
*p
)
341 // Release previous log to signal handling side.
342 // Loop because we are racing against setprofile(off).
346 runtime_printf("runtime: phase error during cpu profile handoff\n");
350 p
->wtoggle
= 1 - p
->wtoggle
;
355 if(runtime_cas(&p
->handoff
, n
, 0))
358 p
->wtoggle
= 1 - p
->wtoggle
;
365 if(!p
->on
&& p
->handoff
== 0)
369 // runtime·entersyscall();
370 runtime_notesleep(&p
->wait
);
371 // runtime·exitsyscall();
372 runtime_noteclear(&p
->wait
);
376 runtime_printf("runtime: phase error during cpu profile wait\n");
379 if(n
== 0x80000000) {
385 // Return new log to caller.
388 ret
.array
= (byte
*)p
->log
[p
->wtoggle
];
389 ret
.len
= n
*sizeof(uintptr
);
395 // Add is no longer being called. We own the log.
396 // Also, p->handoff is non-zero, so flushlog will return false.
397 // Evict the hash table into the log and return it.
398 for(i
=0; i
<HashSize
; i
++) {
400 for(j
=0; j
<Assoc
; j
++) {
402 if(e
->count
> 0 && !evict(p
, e
)) {
403 // Filled the log. Stop the loop and return what we've got.
410 // Return pending log data.
412 // Note that we're using toggle now, not wtoggle,
413 // because we're working on the log directly.
414 ret
.array
= (byte
*)p
->log
[p
->toggle
];
415 ret
.len
= p
->nlog
*sizeof(uintptr
);
421 // Made it through the table without finding anything to log.
422 // Finally done. Clean up and return nil.
424 if(!runtime_cas(&p
->handoff
, p
->handoff
, 0))
425 runtime_printf("runtime: profile flush racing with something\n");
426 return ret
; // set to nil at top of function
429 extern Slice
runtime_CPUProfile(void)
430 __asm__("libgo_runtime.runtime.CPUProfile");
432 // CPUProfile returns the next cpu profile block as a []byte.
433 // The user documentation is in debug.go.
435 runtime_CPUProfile(void)
437 return getprofile(prof
);