1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // This implements the write barrier buffer. The write barrier itself
6 // is gcWriteBarrier and is implemented in assembly.
8 // The write barrier has a fast path and a slow path. The fast path
9 // simply enqueues to a per-P write barrier buffer. It's written in
10 // assembly and doesn't clobber any general purpose registers, so it
11 // doesn't have the usual overheads of a Go call.
13 // When the buffer fills up, the write barrier invokes the slow path
14 // (wbBufFlush) to flush the buffer to the GC work queues. In this
15 // path, since the compiler didn't spill registers, we spill *all*
16 // registers and disallow any GC safe points that could observe the
17 // stack frame (since we don't know the types of the spilled
23 "runtime/internal/sys"
27 // testSmallBuf forces a small write barrier buffer to stress write
29 const testSmallBuf
= false
31 // wbBuf is a per-P buffer of pointers queued by the write barrier.
32 // This buffer is flushed to the GC workbufs when it fills up and on
33 // various GC transitions.
35 // This is closely related to a "sequential store buffer" (SSB),
36 // except that SSBs are usually used for maintaining remembered sets,
37 // while this is used for marking.
39 // next points to the next slot in buf. It must not be a
40 // pointer type because it can point past the end of buf and
41 // must be updated without write barriers.
43 // This is a pointer rather than an index to optimize the
44 // write barrier assembly.
47 // end points to just past the end of buf. It must not be a
48 // pointer type because it points past the end of buf and must
49 // be updated without write barriers.
52 // buf stores a series of pointers to execute write barriers
53 // on. This must be a multiple of wbBufEntryPointers because
54 // the write barrier only checks for overflow once per entry.
55 buf
[wbBufEntryPointers
* wbBufEntries
]uintptr
59 // wbBufEntries is the number of write barriers between
60 // flushes of the write barrier buffer.
62 // This trades latency for throughput amortization. Higher
63 // values amortize flushing overhead more, but increase the
64 // latency of flushing. Higher values also increase the cache
65 // footprint of the buffer.
67 // TODO: What is the latency cost of this? Tune this value.
70 // wbBufEntryPointers is the number of pointers added to the
71 // buffer by each write barrier.
72 wbBufEntryPointers
= 2
75 // reset empties b by resetting its next and end pointers.
76 func (b
*wbBuf
) reset() {
77 start
:= uintptr(unsafe
.Pointer(&b
.buf
[0]))
79 if gcBlackenPromptly || writeBarrier
.cgo
{
80 // Effectively disable the buffer by forcing a flush
82 b
.end
= uintptr(unsafe
.Pointer(&b
.buf
[wbBufEntryPointers
]))
83 } else if testSmallBuf
{
84 // For testing, allow two barriers in the buffer. If
85 // we only did one, then barriers of non-heap pointers
86 // would be no-ops. This lets us combine a buffered
87 // barrier with a flush at a later time.
88 b
.end
= uintptr(unsafe
.Pointer(&b
.buf
[2*wbBufEntryPointers
]))
90 b
.end
= start
+ uintptr(len(b
.buf
))*unsafe
.Sizeof(b
.buf
[0])
93 if (b
.end
-b
.next
)%(wbBufEntryPointers
*unsafe
.Sizeof(b
.buf
[0])) != 0 {
94 throw("bad write barrier buffer bounds")
98 // discard resets b's next pointer, but not its end pointer.
100 // This must be nosplit because it's called by wbBufFlush.
103 func (b
*wbBuf
) discard() {
104 b
.next
= uintptr(unsafe
.Pointer(&b
.buf
[0]))
107 // putFast adds old and new to the write barrier buffer and returns
108 // false if a flush is necessary. Callers should use this as:
110 // buf := &getg().m.p.ptr().wbBuf
111 // if !buf.putFast(old, new) {
115 // The arguments to wbBufFlush depend on whether the caller is doing
116 // its own cgo pointer checks. If it is, then this can be
117 // wbBufFlush(nil, 0). Otherwise, it must pass the slot address and
120 // Since buf is a per-P resource, the caller must ensure there are no
121 // preemption points while buf is in use.
123 // It must be nowritebarrierrec to because write barriers here would
124 // corrupt the write barrier buffer. It (and everything it calls, if
125 // it called anything) has to be nosplit to avoid scheduling on to a
126 // different P and a different buffer.
128 //go:nowritebarrierrec
130 func (b
*wbBuf
) putFast(old
, new uintptr) bool {
131 p
:= (*[2]uintptr)(unsafe
.Pointer(b
.next
))
134 b
.next
+= 2 * sys
.PtrSize
135 return b
.next
!= b
.end
138 // wbBufFlush flushes the current P's write barrier buffer to the GC
139 // workbufs. It is passed the slot and value of the write barrier that
140 // caused the flush so that it can implement cgocheck.
142 // This must not have write barriers because it is part of the write
143 // barrier implementation.
145 // This and everything it calls must be nosplit because 1) the stack
146 // contains untyped slots from gcWriteBarrier and 2) there must not be
147 // a GC safe point between the write barrier test in the caller and
148 // flushing the buffer.
150 // TODO: A "go:nosplitrec" annotation would be perfect for this.
152 //go:nowritebarrierrec
154 func wbBufFlush(dst
*uintptr, src
uintptr) {
155 // Note: Every possible return from this function must reset
156 // the buffer's next pointer to prevent buffer overflow.
158 if getg().m
.dying
> 0 {
159 // We're going down. Not much point in write barriers
160 // and this way we can allow write barriers in the
162 getg().m
.p
.ptr().wbBuf
.discard()
166 if writeBarrier
.cgo
&& dst
!= nil {
167 // This must be called from the stack that did the
168 // write. It's nosplit all the way down.
169 cgoCheckWriteBarrier(dst
, src
)
170 if !writeBarrier
.needed
{
171 // We were only called for cgocheck.
172 getg().m
.p
.ptr().wbBuf
.discard()
177 // Switch to the system stack so we don't have to worry about
178 // the untyped stack slots or safe points.
180 wbBufFlush1(getg().m
.p
.ptr())
184 // wbBufFlush1 flushes p's write barrier buffer to the GC work queue.
186 // This must not have write barriers because it is part of the write
187 // barrier implementation, so this may lead to infinite loops or
188 // buffer corruption.
190 // This must be non-preemptible because it uses the P's workbuf.
192 //go:nowritebarrierrec
194 func wbBufFlush1(_p_
*p
) {
195 // Get the buffered pointers.
196 start
:= uintptr(unsafe
.Pointer(&_p_
.wbBuf
.buf
[0]))
197 n
:= (_p_
.wbBuf
.next
- start
) / unsafe
.Sizeof(_p_
.wbBuf
.buf
[0])
198 ptrs
:= _p_
.wbBuf
.buf
[:n
]
204 // Slow path for checkmark mode.
205 for _
, ptr
:= range ptrs
{
211 // Mark all of the pointers in the buffer and record only the
212 // pointers we greyed. We use the buffer itself to temporarily
213 // record greyed pointers.
215 // TODO: Should scanobject/scanblock just stuff pointers into
216 // the wbBuf? Then this would become the sole greying path.
219 arenaStart
:= mheap_
.arena_start
220 for _
, ptr
:= range ptrs
{
221 if ptr
< arenaStart
{
222 // nil pointers are very common, especially
223 // for the "old" values. Filter out these and
224 // other "obvious" non-heap pointers ASAP.
226 // TODO: Should we filter out nils in the fast
227 // path to reduce the rate of flushes?
230 // TODO: This doesn't use hbits, so calling
231 // heapBitsForObject seems a little silly. We could
232 // easily separate this out since heapBitsForObject
233 // just calls heapBitsForAddr(obj) to get hbits.
234 obj
, _
, span
, objIndex
:= heapBitsForObject(ptr
, 0, 0, false)
238 // TODO: Consider making two passes where the first
239 // just prefetches the mark bits.
240 mbits
:= span
.markBitsForIndex(objIndex
)
241 if mbits
.isMarked() {
245 if span
.spanclass
.noscan() {
246 gcw
.bytesMarked
+= uint64(span
.elemsize
)
253 // Enqueue the greyed objects.
254 gcw
.putBatch(ptrs
[:pos
])
255 if gcphase
== _GCmarktermination || gcBlackenPromptly
{
256 // Ps aren't allowed to cache work during mark