libgo/go/runtime/mcache.go

   1 // Copyright 2009 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package runtime
   6
   7 import "unsafe"
   8
   9 // Per-thread (in Go, per-P) cache for small objects.
  10 // No locking needed because it is per-thread (per-P).
  11 //
  12 // mcaches are allocated from non-GC'd memory, so any heap pointers
  13 // must be specially handled.
  14 //
  15 //go:notinheap
  16 type mcache struct {
  17         // The following members are accessed on every malloc,
  18         // so they are grouped here for better caching.
  19         next_sample int32   // trigger heap sample after allocating this many bytes
  20         local_scan  uintptr // bytes of scannable heap allocated
  21
  22         // Allocator cache for tiny objects w/o pointers.
  23         // See "Tiny allocator" comment in malloc.go.
  24
  25         // tiny points to the beginning of the current tiny block, or
  26         // nil if there is no current tiny block.
  27         //
  28         // tiny is a heap pointer. Since mcache is in non-GC'd memory,
  29         // we handle it by clearing it in releaseAll during mark
  30         // termination.
  31         tiny             uintptr
  32         tinyoffset       uintptr
  33         local_tinyallocs uintptr // number of tiny allocs not counted in other stats
  34
  35         // The rest is not accessed on every malloc.
  36         alloc [_NumSizeClasses]*mspan // spans to allocate from
  37
  38         // Local allocator stats, flushed during GC.
  39         local_nlookup    uintptr                  // number of pointer lookups
  40         local_largefree  uintptr                  // bytes freed for large objects (>maxsmallsize)
  41         local_nlargefree uintptr                  // number of frees for large objects (>maxsmallsize)
  42         local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize)
  43 }
  44
  45 // A gclink is a node in a linked list of blocks, like mlink,
  46 // but it is opaque to the garbage collector.
  47 // The GC does not trace the pointers during collection,
  48 // and the compiler does not emit write barriers for assignments
  49 // of gclinkptr values. Code should store references to gclinks
  50 // as gclinkptr, not as *gclink.
  51 type gclink struct {
  52         next gclinkptr
  53 }
  54
  55 // A gclinkptr is a pointer to a gclink, but it is opaque
  56 // to the garbage collector.
  57 type gclinkptr uintptr
  58
  59 // ptr returns the *gclink form of p.
  60 // The result should be used for accessing fields, not stored
  61 // in other data structures.
  62 func (p gclinkptr) ptr() *gclink {
  63         return (*gclink)(unsafe.Pointer(p))
  64 }
  65
  66 // dummy MSpan that contains no free objects.
  67 var emptymspan mspan
  68
  69 func allocmcache() *mcache {
  70         lock(&mheap_.lock)
  71         c := (*mcache)(mheap_.cachealloc.alloc())
  72         unlock(&mheap_.lock)
  73         for i := 0; i < _NumSizeClasses; i++ {
  74                 c.alloc[i] = &emptymspan
  75         }
  76         c.next_sample = nextSample()
  77         return c
  78 }
  79
  80 func freemcache(c *mcache) {
  81         systemstack(func() {
  82                 c.releaseAll()
  83
  84                 // NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate
  85                 // with the stealing of gcworkbufs during garbage collection to avoid
  86                 // a race where the workbuf is double-freed.
  87                 // gcworkbuffree(c.gcworkbuf)
  88
  89                 lock(&mheap_.lock)
  90                 purgecachedstats(c)
  91                 mheap_.cachealloc.free(unsafe.Pointer(c))
  92                 unlock(&mheap_.lock)
  93         })
  94 }
  95
  96 // Gets a span that has a free object in it and assigns it
  97 // to be the cached span for the given sizeclass. Returns this span.
  98 func (c *mcache) refill(sizeclass int32) *mspan {
  99         _g_ := getg()
 100
 101         _g_.m.locks++
 102         // Return the current cached span to the central lists.
 103         s := c.alloc[sizeclass]
 104
 105         if uintptr(s.allocCount) != s.nelems {
 106                 throw("refill of span with free space remaining")
 107         }
 108
 109         if s != &emptymspan {
 110                 s.incache = false
 111         }
 112
 113         // Get a new cached span from the central lists.
 114         s = mheap_.central[sizeclass].mcentral.cacheSpan()
 115         if s == nil {
 116                 throw("out of memory")
 117         }
 118
 119         if uintptr(s.allocCount) == s.nelems {
 120                 throw("span has no free space")
 121         }
 122
 123         c.alloc[sizeclass] = s
 124         _g_.m.locks--
 125         return s
 126 }
 127
 128 func (c *mcache) releaseAll() {
 129         for i := 0; i < _NumSizeClasses; i++ {
 130                 s := c.alloc[i]
 131                 if s != &emptymspan {
 132                         mheap_.central[i].mcentral.uncacheSpan(s)
 133                         c.alloc[i] = &emptymspan
 134                 }
 135         }
 136         // Clear tinyalloc pool.
 137         c.tiny = 0
 138         c.tinyoffset = 0
 139 }