libgo/go/runtime/mcache.go

   1 // Copyright 2009 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package runtime
   6
   7 import "unsafe"
   8
   9 // Per-thread (in Go, per-P) cache for small objects.
  10 // No locking needed because it is per-thread (per-P).
  11 //
  12 // mcaches are allocated from non-GC'd memory, so any heap pointers
  13 // must be specially handled.
  14 //
  15 //go:notinheap
  16 type mcache struct {
  17         // The following members are accessed on every malloc,
  18         // so they are grouped here for better caching.
  19         next_sample int32   // trigger heap sample after allocating this many bytes
  20         local_scan  uintptr // bytes of scannable heap allocated
  21
  22         // Allocator cache for tiny objects w/o pointers.
  23         // See "Tiny allocator" comment in malloc.go.
  24
  25         // tiny points to the beginning of the current tiny block, or
  26         // nil if there is no current tiny block.
  27         //
  28         // tiny is a heap pointer. Since mcache is in non-GC'd memory,
  29         // we handle it by clearing it in releaseAll during mark
  30         // termination.
  31         tiny             uintptr
  32         tinyoffset       uintptr
  33         local_tinyallocs uintptr // number of tiny allocs not counted in other stats
  34
  35         // The rest is not accessed on every malloc.
  36
  37         alloc [numSpanClasses]*mspan // spans to allocate from, indexed by spanClass
  38
  39         // Local allocator stats, flushed during GC.
  40         local_nlookup    uintptr                  // number of pointer lookups
  41         local_largefree  uintptr                  // bytes freed for large objects (>maxsmallsize)
  42         local_nlargefree uintptr                  // number of frees for large objects (>maxsmallsize)
  43         local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize)
  44 }
  45
  46 // A gclink is a node in a linked list of blocks, like mlink,
  47 // but it is opaque to the garbage collector.
  48 // The GC does not trace the pointers during collection,
  49 // and the compiler does not emit write barriers for assignments
  50 // of gclinkptr values. Code should store references to gclinks
  51 // as gclinkptr, not as *gclink.
  52 type gclink struct {
  53         next gclinkptr
  54 }
  55
  56 // A gclinkptr is a pointer to a gclink, but it is opaque
  57 // to the garbage collector.
  58 type gclinkptr uintptr
  59
  60 // ptr returns the *gclink form of p.
  61 // The result should be used for accessing fields, not stored
  62 // in other data structures.
  63 func (p gclinkptr) ptr() *gclink {
  64         return (*gclink)(unsafe.Pointer(p))
  65 }
  66
  67 // dummy MSpan that contains no free objects.
  68 var emptymspan mspan
  69
  70 func allocmcache() *mcache {
  71         lock(&mheap_.lock)
  72         c := (*mcache)(mheap_.cachealloc.alloc())
  73         unlock(&mheap_.lock)
  74         for i := range c.alloc {
  75                 c.alloc[i] = &emptymspan
  76         }
  77         c.next_sample = nextSample()
  78         return c
  79 }
  80
  81 func freemcache(c *mcache) {
  82         systemstack(func() {
  83                 c.releaseAll()
  84
  85                 // NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate
  86                 // with the stealing of gcworkbufs during garbage collection to avoid
  87                 // a race where the workbuf is double-freed.
  88                 // gcworkbuffree(c.gcworkbuf)
  89
  90                 lock(&mheap_.lock)
  91                 purgecachedstats(c)
  92                 mheap_.cachealloc.free(unsafe.Pointer(c))
  93                 unlock(&mheap_.lock)
  94         })
  95 }
  96
  97 // Gets a span that has a free object in it and assigns it
  98 // to be the cached span for the given sizeclass. Returns this span.
  99 func (c *mcache) refill(spc spanClass) {
 100         _g_ := getg()
 101
 102         _g_.m.locks++
 103         // Return the current cached span to the central lists.
 104         s := c.alloc[spc]
 105
 106         if uintptr(s.allocCount) != s.nelems {
 107                 throw("refill of span with free space remaining")
 108         }
 109
 110         if s != &emptymspan {
 111                 s.incache = false
 112         }
 113
 114         // Get a new cached span from the central lists.
 115         s = mheap_.central[spc].mcentral.cacheSpan()
 116         if s == nil {
 117                 throw("out of memory")
 118         }
 119
 120         if uintptr(s.allocCount) == s.nelems {
 121                 throw("span has no free space")
 122         }
 123
 124         c.alloc[spc] = s
 125         _g_.m.locks--
 126 }
 127
 128 func (c *mcache) releaseAll() {
 129         for i := range c.alloc {
 130                 s := c.alloc[i]
 131                 if s != &emptymspan {
 132                         mheap_.central[i].mcentral.uncacheSpan(s)
 133                         c.alloc[i] = &emptymspan
 134                 }
 135         }
 136         // Clear tinyalloc pool.
 137         c.tiny = 0
 138         c.tinyoffset = 0
 139 }