1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
6 // Patterned after tcmalloc's algorithms; shorter code.
11 "runtime/internal/atomic"
15 // Export temporarily for gccgo's C code to call:
16 //go:linkname mProf_Malloc runtime.mProf_Malloc
17 //go:linkname mProf_Free runtime.mProf_Free
18 //go:linkname mProf_GC runtime.mProf_GC
19 //go:linkname tracealloc runtime.tracealloc
20 //go:linkname tracefree runtime.tracefree
21 //go:linkname tracegc runtime.tracegc
22 //go:linkname iterate_memprof runtime.iterate_memprof
24 // NOTE(rsc): Everything here could use cas if contention became an issue.
27 // All memory allocations are local and do not escape outside of the profiler.
28 // The profiler is forbidden from referring to garbage-collected memory.
32 memProfile bucketType
= 1 + iota
36 // size of bucket hash table
39 // max depth of stack to record in bucket
45 // A bucket holds per-call-stack profiling information.
46 // The representation is a bit sleazy, inherited from C.
47 // This struct defines the bucket header. It is followed in
48 // memory by the stack words and then the actual record
49 // data, either a memRecord or a blockRecord.
51 // Per-call-stack profiling information.
52 // Lookup by hashing call stack into a linked-list hash table.
60 typ bucketType
// memBucket or blockBucket (includes mutexProfile)
66 // A memRecord is the bucket data for a bucket of type memProfile,
67 // part of the memory profile.
68 type memRecord
struct {
69 // The following complex 3-stage scheme of stats accumulation
70 // is required to obtain a consistent picture of mallocs and frees
71 // for some point in time.
72 // The problem is that mallocs come in real time, while frees
73 // come only after a GC during concurrent sweeping. So if we would
74 // naively count them, we would get a skew toward mallocs.
76 // Mallocs are accounted in recent stats.
77 // Explicit frees are accounted in recent stats.
78 // GC frees are accounted in prev stats.
79 // After GC prev stats are added to final stats and
80 // recent stats are moved into prev stats.
86 // changes between next-to-last GC and last GC
89 prev_alloc_bytes
uintptr
90 prev_free_bytes
uintptr
92 // changes since last GC
95 recent_alloc_bytes
uintptr
96 recent_free_bytes
uintptr
99 // A blockRecord is the bucket data for a bucket of type blockProfile,
100 // which is used in blocking and mutex profiles.
101 type blockRecord
struct {
107 mbuckets
*bucket
// memory profile buckets
108 bbuckets
*bucket
// blocking profile buckets
109 xbuckets
*bucket
// mutex profile buckets
110 buckhash
*[179999]*bucket
114 // newBucket allocates a bucket with the given type and number of stack entries.
115 func newBucket(typ bucketType
, nstk
int) *bucket
{
116 size
:= unsafe
.Sizeof(bucket
{}) + uintptr(nstk
)*unsafe
.Sizeof(location
{})
119 throw("invalid profile bucket type")
121 size
+= unsafe
.Sizeof(memRecord
{})
122 case blockProfile
, mutexProfile
:
123 size
+= unsafe
.Sizeof(blockRecord
{})
126 b
:= (*bucket
)(persistentalloc(size
, 0, &memstats
.buckhash_sys
))
129 b
.nstk
= uintptr(nstk
)
133 // stk returns the slice in b holding the stack.
134 func (b
*bucket
) stk() []location
{
135 stk
:= (*[maxStack
]location
)(add(unsafe
.Pointer(b
), unsafe
.Sizeof(*b
)))
136 return stk
[:b
.nstk
:b
.nstk
]
139 // mp returns the memRecord associated with the memProfile bucket b.
140 func (b
*bucket
) mp() *memRecord
{
141 if b
.typ
!= memProfile
{
142 throw("bad use of bucket.mp")
144 data
:= add(unsafe
.Pointer(b
), unsafe
.Sizeof(*b
)+b
.nstk
*unsafe
.Sizeof(location
{}))
145 return (*memRecord
)(data
)
148 // bp returns the blockRecord associated with the blockProfile bucket b.
149 func (b
*bucket
) bp() *blockRecord
{
150 if b
.typ
!= blockProfile
&& b
.typ
!= mutexProfile
{
151 throw("bad use of bucket.bp")
153 data
:= add(unsafe
.Pointer(b
), unsafe
.Sizeof(*b
)+b
.nstk
*unsafe
.Sizeof(location
{}))
154 return (*blockRecord
)(data
)
157 // Return the bucket for stk[0:nstk], allocating new bucket if needed.
158 func stkbucket(typ bucketType
, size
uintptr, stk
[]location
, alloc
bool) *bucket
{
160 buckhash
= (*[buckHashSize
]*bucket
)(sysAlloc(unsafe
.Sizeof(*buckhash
), &memstats
.buckhash_sys
))
162 throw("runtime: cannot allocate memory")
168 for _
, loc
:= range stk
{
181 i
:= int(h
% buckHashSize
)
182 for b
:= buckhash
[i
]; b
!= nil; b
= b
.next
{
183 if b
.typ
== typ
&& b
.hash
== h
&& b
.size
== size
&& eqslice(b
.stk(), stk
) {
192 // Create new bucket.
193 b
:= newBucket(typ
, len(stk
))
199 if typ
== memProfile
{
202 } else if typ
== mutexProfile
{
212 func eqslice(x
, y
[]location
) bool {
213 if len(x
) != len(y
) {
216 for i
, xi
:= range x
{
225 for b
:= mbuckets
; b
!= nil; b
= b
.allnext
{
227 mp
.allocs
+= mp
.prev_allocs
228 mp
.frees
+= mp
.prev_frees
229 mp
.alloc_bytes
+= mp
.prev_alloc_bytes
230 mp
.free_bytes
+= mp
.prev_free_bytes
232 mp
.prev_allocs
= mp
.recent_allocs
233 mp
.prev_frees
= mp
.recent_frees
234 mp
.prev_alloc_bytes
= mp
.recent_alloc_bytes
235 mp
.prev_free_bytes
= mp
.recent_free_bytes
239 mp
.recent_alloc_bytes
= 0
240 mp
.recent_free_bytes
= 0
244 // Record that a gc just happened: all the 'recent' statistics are now real.
251 // Called by malloc to record a profiled block.
252 func mProf_Malloc(p unsafe
.Pointer
, size
uintptr) {
253 var stk
[maxStack
]location
254 nstk
:= callers(4, stk
[:])
256 b
:= stkbucket(memProfile
, size
, stk
[:nstk
], true)
259 mp
.recent_alloc_bytes
+= size
262 // Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock.
263 // This reduces potential contention and chances of deadlocks.
264 // Since the object must be alive during call to mProf_Malloc,
265 // it's fine to do this non-atomically.
267 setprofilebucket(p
, b
)
271 // Called when freeing a profiled block.
272 func mProf_Free(b
*bucket
, size
uintptr) {
276 mp
.prev_free_bytes
+= size
280 var blockprofilerate
uint64 // in CPU ticks
282 // SetBlockProfileRate controls the fraction of goroutine blocking events
283 // that are reported in the blocking profile. The profiler aims to sample
284 // an average of one blocking event per rate nanoseconds spent blocked.
286 // To include every blocking event in the profile, pass rate = 1.
287 // To turn off profiling entirely, pass rate <= 0.
288 func SetBlockProfileRate(rate
int) {
291 r
= 0 // disable profiling
292 } else if rate
== 1 {
293 r
= 1 // profile everything
295 // convert ns to cycles, use float64 to prevent overflow during multiplication
296 r
= int64(float64(rate
) * float64(tickspersecond()) / (1000 * 1000 * 1000))
302 atomic
.Store64(&blockprofilerate
, uint64(r
))
305 func blockevent(cycles
int64, skip
int) {
309 if blocksampled(cycles
) {
310 saveblockevent(cycles
, skip
+1, blockProfile
, &blockprofilerate
)
314 func blocksampled(cycles
int64) bool {
315 rate
:= int64(atomic
.Load64(&blockprofilerate
))
316 if rate
<= 0 ||
(rate
> cycles
&& int64(fastrand())%rate
> cycles
) {
322 func saveblockevent(cycles
int64, skip
int, which bucketType
, ratep
*uint64) {
325 var stk
[maxStack
]location
326 if gp
.m
.curg
== nil || gp
.m
.curg
== gp
{
327 nstk
= callers(skip
, stk
[:])
329 // FIXME: This should get a traceback of gp.m.curg.
330 // nstk = gcallers(gp.m.curg, skip, stk[:])
331 nstk
= callers(skip
, stk
[:])
334 b
:= stkbucket(which
, 0, stk
[:nstk
], true)
336 b
.bp().cycles
+= cycles
340 var mutexprofilerate
uint64 // fraction sampled
342 // SetMutexProfileFraction controls the fraction of mutex contention events
343 // that are reported in the mutex profile. On average 1/rate events are
344 // reported. The previous rate is returned.
346 // To turn off profiling entirely, pass rate 0.
347 // To just read the current rate, pass rate -1.
348 // (For n>1 the details of sampling may change.)
349 func SetMutexProfileFraction(rate
int) int {
351 return int(mutexprofilerate
)
353 old
:= mutexprofilerate
354 atomic
.Store64(&mutexprofilerate
, uint64(rate
))
358 //go:linkname mutexevent sync.event
359 func mutexevent(cycles
int64, skip
int) {
363 rate
:= int64(atomic
.Load64(&mutexprofilerate
))
364 // TODO(pjw): measure impact of always calling fastrand vs using something
365 // like malloc.go:nextSample()
366 if rate
> 0 && int64(fastrand())%rate
== 0 {
367 saveblockevent(cycles
, skip
+1, mutexProfile
, &mutexprofilerate
)
371 // Go interface to profile data.
373 // A StackRecord describes a single execution stack.
374 type StackRecord
struct {
375 Stack0
[32]uintptr // stack trace for this record; ends at first 0 entry
378 // Stack returns the stack trace associated with the record,
379 // a prefix of r.Stack0.
380 func (r
*StackRecord
) Stack() []uintptr {
381 for i
, v
:= range r
.Stack0
{
389 // MemProfileRate controls the fraction of memory allocations
390 // that are recorded and reported in the memory profile.
391 // The profiler aims to sample an average of
392 // one allocation per MemProfileRate bytes allocated.
394 // To include every allocated block in the profile, set MemProfileRate to 1.
395 // To turn off profiling entirely, set MemProfileRate to 0.
397 // The tools that process the memory profiles assume that the
398 // profile rate is constant across the lifetime of the program
399 // and equal to the current value. Programs that change the
400 // memory profiling rate should do so just once, as early as
401 // possible in the execution of the program (for example,
402 // at the beginning of main).
403 var MemProfileRate
int = 512 * 1024
405 // A MemProfileRecord describes the live objects allocated
406 // by a particular call sequence (stack trace).
407 type MemProfileRecord
struct {
408 AllocBytes
, FreeBytes
int64 // number of bytes allocated, freed
409 AllocObjects
, FreeObjects
int64 // number of objects allocated, freed
410 Stack0
[32]uintptr // stack trace for this record; ends at first 0 entry
413 // InUseBytes returns the number of bytes in use (AllocBytes - FreeBytes).
414 func (r
*MemProfileRecord
) InUseBytes() int64 { return r
.AllocBytes
- r
.FreeBytes
}
416 // InUseObjects returns the number of objects in use (AllocObjects - FreeObjects).
417 func (r
*MemProfileRecord
) InUseObjects() int64 {
418 return r
.AllocObjects
- r
.FreeObjects
421 // Stack returns the stack trace associated with the record,
422 // a prefix of r.Stack0.
423 func (r
*MemProfileRecord
) Stack() []uintptr {
424 for i
, v
:= range r
.Stack0
{
432 // MemProfile returns a profile of memory allocated and freed per allocation
435 // MemProfile returns n, the number of records in the current memory profile.
436 // If len(p) >= n, MemProfile copies the profile into p and returns n, true.
437 // If len(p) < n, MemProfile does not change p and returns n, false.
439 // If inuseZero is true, the profile includes allocation records
440 // where r.AllocBytes > 0 but r.AllocBytes == r.FreeBytes.
441 // These are sites where memory was allocated, but it has all
442 // been released back to the runtime.
444 // The returned profile may be up to two garbage collection cycles old.
445 // This is to avoid skewing the profile toward allocations; because
446 // allocations happen in real time but frees are delayed until the garbage
447 // collector performs sweeping, the profile only accounts for allocations
448 // that have had a chance to be freed by the garbage collector.
450 // Most clients should use the runtime/pprof package or
451 // the testing package's -test.memprofile flag instead
452 // of calling MemProfile directly.
453 func MemProfile(p
[]MemProfileRecord
, inuseZero
bool) (n
int, ok
bool) {
456 for b
:= mbuckets
; b
!= nil; b
= b
.allnext
{
458 if inuseZero || mp
.alloc_bytes
!= mp
.free_bytes
{
461 if mp
.allocs
!= 0 || mp
.frees
!= 0 {
466 // Absolutely no data, suggesting that a garbage collection
467 // has not yet happened. In order to allow profiling when
468 // garbage collection is disabled from the beginning of execution,
469 // accumulate stats as if a GC just happened, and recount buckets.
473 for b
:= mbuckets
; b
!= nil; b
= b
.allnext
{
475 if inuseZero || mp
.alloc_bytes
!= mp
.free_bytes
{
483 for b
:= mbuckets
; b
!= nil; b
= b
.allnext
{
485 if inuseZero || mp
.alloc_bytes
!= mp
.free_bytes
{
495 // Write b's data to r.
496 func record(r
*MemProfileRecord
, b
*bucket
) {
498 r
.AllocBytes
= int64(mp
.alloc_bytes
)
499 r
.FreeBytes
= int64(mp
.free_bytes
)
500 r
.AllocObjects
= int64(mp
.allocs
)
501 r
.FreeObjects
= int64(mp
.frees
)
502 for i
, loc
:= range b
.stk() {
503 if i
>= len(r
.Stack0
) {
508 for i
:= int(b
.nstk
); i
< len(r
.Stack0
); i
++ {
513 func iterate_memprof(fn
func(*bucket
, uintptr, *location
, uintptr, uintptr, uintptr)) {
515 for b
:= mbuckets
; b
!= nil; b
= b
.allnext
{
517 fn(b
, b
.nstk
, &b
.stk()[0], b
.size
, mp
.allocs
, mp
.frees
)
522 // BlockProfileRecord describes blocking events originated
523 // at a particular call sequence (stack trace).
524 type BlockProfileRecord
struct {
530 // BlockProfile returns n, the number of records in the current blocking profile.
531 // If len(p) >= n, BlockProfile copies the profile into p and returns n, true.
532 // If len(p) < n, BlockProfile does not change p and returns n, false.
534 // Most clients should use the runtime/pprof package or
535 // the testing package's -test.blockprofile flag instead
536 // of calling BlockProfile directly.
537 func BlockProfile(p
[]BlockProfileRecord
) (n
int, ok
bool) {
539 for b
:= bbuckets
; b
!= nil; b
= b
.allnext
{
544 for b
:= bbuckets
; b
!= nil; b
= b
.allnext
{
551 for i
, loc
= range b
.stk() {
552 if i
>= len(r
.Stack0
) {
557 for ; i
< len(r
.Stack0
); i
++ {
567 // MutexProfile returns n, the number of records in the current mutex profile.
568 // If len(p) >= n, MutexProfile copies the profile into p and returns n, true.
569 // Otherwise, MutexProfile does not change p, and returns n, false.
571 // Most clients should use the runtime/pprof package
572 // instead of calling MutexProfile directly.
573 func MutexProfile(p
[]BlockProfileRecord
) (n
int, ok
bool) {
575 for b
:= xbuckets
; b
!= nil; b
= b
.allnext
{
580 for b
:= xbuckets
; b
!= nil; b
= b
.allnext
{
583 r
.Count
= int64(bp
.count
)
587 for i
, loc
= range b
.stk() {
588 if i
>= len(r
.Stack0
) {
593 for ; i
< len(r
.Stack0
); i
++ {
603 // ThreadCreateProfile returns n, the number of records in the thread creation profile.
604 // If len(p) >= n, ThreadCreateProfile copies the profile into p and returns n, true.
605 // If len(p) < n, ThreadCreateProfile does not change p and returns n, false.
607 // Most clients should use the runtime/pprof package instead
608 // of calling ThreadCreateProfile directly.
609 func ThreadCreateProfile(p
[]StackRecord
) (n
int, ok
bool) {
610 first
:= (*m
)(atomic
.Loadp(unsafe
.Pointer(&allm
)))
611 for mp
:= first
; mp
!= nil; mp
= mp
.alllink
{
617 for mp
:= first
; mp
!= nil; mp
= mp
.alllink
{
618 for j
:= range mp
.createstack
{
619 p
[i
].Stack0
[j
] = mp
.createstack
[j
].pc
627 // GoroutineProfile returns n, the number of records in the active goroutine stack profile.
628 // If len(p) >= n, GoroutineProfile copies the profile into p and returns n, true.
629 // If len(p) < n, GoroutineProfile does not change p and returns n, false.
631 // Most clients should use the runtime/pprof package instead
632 // of calling GoroutineProfile directly.
633 func GoroutineProfile(p
[]StackRecord
) (n
int, ok
bool) {
636 isOK
:= func(gp1
*g
) bool {
637 // Checking isSystemGoroutine here makes GoroutineProfile
638 // consistent with both NumGoroutine and Stack.
639 return gp1
!= gp
&& readgstatus(gp1
) != _Gdead
&& !isSystemGoroutine(gp1
)
642 stopTheWorld("profile")
645 for _
, gp1
:= range allgs
{
655 // Save current goroutine.
659 // Save other goroutines.
660 for _
, gp1
:= range allgs
{
663 // Should be impossible, but better to return a
664 // truncated profile than to crash the entire process.
678 func saveg(gp
*g
, r
*StackRecord
) {
680 var locbuf
[32]location
681 n
:= callers(1, locbuf
[:])
682 for i
:= 0; i
< n
; i
++ {
683 r
.Stack0
[i
] = locbuf
[i
].pc
685 if n
< len(r
.Stack0
) {
689 // FIXME: Not implemented.
694 // Stack formats a stack trace of the calling goroutine into buf
695 // and returns the number of bytes written to buf.
696 // If all is true, Stack formats stack traces of all other goroutines
697 // into buf after the trace for the current goroutine.
698 func Stack(buf
[]byte, all
bool) int {
700 stopTheWorld("stack trace")
706 // Force traceback=1 to override GOTRACEBACK setting,
707 // so that Stack's results are consistent.
708 // GOTRACEBACK is only about crash dumps.
710 gp
.writebuf
= buf
[0:0:len(buf
)]
727 // Tracing of alloc/free/gc.
731 func tracealloc(p unsafe
.Pointer
, size
uintptr, typ
*_type
) {
736 print("tracealloc(", p
, ", ", hex(size
), ")\n")
738 print("tracealloc(", p
, ", ", hex(size
), ", ", *typ
.string, ")\n")
740 if gp
.m
.curg
== nil || gp
== gp
.m
.curg
{
744 goroutineheader(gp
.m
.curg
)
745 // FIXME: Can't do traceback of other g.
752 func tracefree(p unsafe
.Pointer
, size
uintptr) {
756 print("tracefree(", p
, ", ", hex(size
), ")\n")
769 // running on m->g0 stack; show all non-g0 goroutines
771 print("end tracegc\n")