libgo/go/runtime/mcheckmark.go

   1 // Copyright 2020 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // GC checkmarks
   6 //
   7 // In a concurrent garbage collector, one worries about failing to mark
   8 // a live object due to mutations without write barriers or bugs in the
   9 // collector implementation. As a sanity check, the GC has a 'checkmark'
  10 // mode that retraverses the object graph with the world stopped, to make
  11 // sure that everything that should be marked is marked.
  12
  13 package runtime
  14
  15 import (
  16         "internal/goarch"
  17         "runtime/internal/atomic"
  18         "unsafe"
  19 )
  20
  21 // A checkmarksMap stores the GC marks in "checkmarks" mode. It is a
  22 // per-arena bitmap with a bit for every word in the arena. The mark
  23 // is stored on the bit corresponding to the first word of the marked
  24 // allocation.
  25 //
  26 //go:notinheap
  27 type checkmarksMap [heapArenaBytes / goarch.PtrSize / 8]uint8
  28
  29 // If useCheckmark is true, marking of an object uses the checkmark
  30 // bits instead of the standard mark bits.
  31 var useCheckmark = false
  32
  33 // startCheckmarks prepares for the checkmarks phase.
  34 //
  35 // The world must be stopped.
  36 func startCheckmarks() {
  37         assertWorldStopped()
  38
  39         // Clear all checkmarks.
  40         for _, ai := range mheap_.allArenas {
  41                 arena := mheap_.arenas[ai.l1()][ai.l2()]
  42                 bitmap := arena.checkmarks
  43
  44                 if bitmap == nil {
  45                         // Allocate bitmap on first use.
  46                         bitmap = (*checkmarksMap)(persistentalloc(unsafe.Sizeof(*bitmap), 0, &memstats.gcMiscSys))
  47                         if bitmap == nil {
  48                                 throw("out of memory allocating checkmarks bitmap")
  49                         }
  50                         arena.checkmarks = bitmap
  51                 } else {
  52                         // Otherwise clear the existing bitmap.
  53                         for i := range bitmap {
  54                                 bitmap[i] = 0
  55                         }
  56                 }
  57         }
  58         // Enable checkmarking.
  59         useCheckmark = true
  60 }
  61
  62 // endCheckmarks ends the checkmarks phase.
  63 func endCheckmarks() {
  64         if gcMarkWorkAvailable(nil) {
  65                 throw("GC work not flushed")
  66         }
  67         useCheckmark = false
  68 }
  69
  70 // setCheckmark throws if marking object is a checkmarks violation,
  71 // and otherwise sets obj's checkmark. It returns true if obj was
  72 // already checkmarked.
  73 func setCheckmark(obj, base, off uintptr, mbits markBits, forStack bool) bool {
  74         if !mbits.isMarked() {
  75                 // Stack scanning is conservative, so we can see a
  76                 // reference to an object not previously found.
  77                 // Assume the object was correctly not marked and
  78                 // ignore the pointer.
  79                 if forStack {
  80                         return false
  81                 }
  82                 printlock()
  83                 print("runtime: checkmarks found unexpected unmarked object obj=", hex(obj), "\n")
  84                 print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
  85
  86                 // Dump the source (base) object
  87                 gcDumpObject("base", base, off)
  88
  89                 // Dump the object
  90                 gcDumpObject("obj", obj, ^uintptr(0))
  91
  92                 getg().m.traceback = 2
  93                 throw("checkmark found unmarked object")
  94         }
  95
  96         ai := arenaIndex(obj)
  97         arena := mheap_.arenas[ai.l1()][ai.l2()]
  98         arenaWord := (obj / heapArenaBytes / 8) % uintptr(len(arena.checkmarks))
  99         mask := byte(1 << ((obj / heapArenaBytes) % 8))
 100         bytep := &arena.checkmarks[arenaWord]
 101
 102         if atomic.Load8(bytep)&mask != 0 {
 103                 // Already checkmarked.
 104                 return true
 105         }
 106
 107         atomic.Or8(bytep, mask)
 108         return false
 109 }