1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // This file contains the implementation of Go channels.
10 // At least one of c.sendq and c.recvq is empty,
11 // except for the case of an unbuffered channel with a single goroutine
12 // blocked on it for both sending and receiving using a select statement,
13 // in which case the length of c.sendq and c.recvq is limited only by the
14 // size of the select statement.
16 // For buffered channels, also:
17 // c.qcount > 0 implies that c.recvq is empty.
18 // c.qcount < c.dataqsiz implies that c.sendq is empty.
21 "runtime/internal/atomic"
25 // For gccgo, use go:linkname to rename compiler-called functions to
26 // themselves, so that the compiler will export them.
28 //go:linkname makechan runtime.makechan
29 //go:linkname makechan64 runtime.makechan64
30 //go:linkname chansend1 runtime.chansend1
31 //go:linkname chanrecv1 runtime.chanrecv1
32 //go:linkname chanrecv2 runtime.chanrecv2
33 //go:linkname closechan runtime.closechan
37 hchanSize
= unsafe
.Sizeof(hchan
{}) + uintptr(-int(unsafe
.Sizeof(hchan
{}))&(maxAlign
-1))
42 qcount
uint // total data in the queue
43 dataqsiz
uint // size of the circular queue
44 buf unsafe
.Pointer
// points to an array of dataqsiz elements
47 elemtype
*_type
// element type
48 sendx
uint // send index
49 recvx
uint // receive index
50 recvq waitq
// list of recv waiters
51 sendq waitq
// list of send waiters
53 // lock protects all fields in hchan, as well as several
54 // fields in sudogs blocked on this channel.
56 // Do not change another G's status while holding this lock
57 // (in particular, do not ready a G), as this can deadlock
58 // with stack shrinking.
67 //go:linkname reflect_makechan reflect.makechan
68 func reflect_makechan(t
*chantype
, size
int) *hchan
{
69 return makechan(t
, size
)
72 func makechan64(t
*chantype
, size
int64) *hchan
{
73 if int64(int(size
)) != size
{
74 panic(plainError("makechan: size out of range"))
77 return makechan(t
, int(size
))
80 func makechan(t
*chantype
, size
int) *hchan
{
83 // compiler checks this but be safe.
84 if elem
.size
>= 1<<16 {
85 throw("makechan: invalid channel element type")
87 if hchanSize%maxAlign
!= 0 || elem
.align
> maxAlign
{
88 throw("makechan: bad alignment")
91 if size
< 0 ||
uintptr(size
) > maxSliceCap(elem
.size
) ||
uintptr(size
)*elem
.size
> _MaxMem
-hchanSize
{
92 panic(plainError("makechan: size out of range"))
95 // Hchan does not contain pointers interesting for GC when elements stored in buf do not contain pointers.
96 // buf points into the same allocation, elemtype is persistent.
97 // SudoG's are referenced from their owning thread so they can't be collected.
98 // TODO(dvyukov,rlh): Rethink when collector can move allocated objects.
101 case size
== 0 || elem
.size
== 0:
102 // Queue or element size is zero.
103 c
= (*hchan
)(mallocgc(hchanSize
, nil, true))
104 // Race detector uses this location for synchronization.
105 c
.buf
= unsafe
.Pointer(c
)
106 case elem
.kind
&kindNoPointers
!= 0:
107 // Elements do not contain pointers.
108 // Allocate hchan and buf in one call.
109 c
= (*hchan
)(mallocgc(hchanSize
+uintptr(size
)*elem
.size
, nil, true))
110 c
.buf
= add(unsafe
.Pointer(c
), hchanSize
)
112 // Elements contain pointers.
114 c
.buf
= mallocgc(uintptr(size
)*elem
.size
, elem
, true)
117 c
.elemsize
= uint16(elem
.size
)
119 c
.dataqsiz
= uint(size
)
122 print("makechan: chan=", c
, "; elemsize=", elem
.size
, "; dataqsiz=", size
, "\n")
127 // chanbuf(c, i) is pointer to the i'th slot in the buffer.
128 func chanbuf(c
*hchan
, i
uint) unsafe
.Pointer
{
129 return add(c
.buf
, uintptr(i
)*uintptr(c
.elemsize
))
132 // entry point for c <- x from compiled code
134 func chansend1(c
*hchan
, elem unsafe
.Pointer
) {
135 chansend(c
, elem
, true, getcallerpc())
139 * generic single channel send/recv
140 * If block is not nil,
141 * then the protocol will not
142 * sleep but return if it could
145 * sleep can wake up with g.param == nil
146 * when a channel involved in the sleep has
147 * been closed. it is easiest to loop and re-run
148 * the operation; we'll see that it's now closed.
150 func chansend(c
*hchan
, ep unsafe
.Pointer
, block
bool, callerpc
uintptr) bool {
155 gopark(nil, nil, "chan send (nil chan)", traceEvGoStop
, 2)
160 print("chansend: chan=", c
, "\n")
164 racereadpc(unsafe
.Pointer(c
), callerpc
, funcPC(chansend
))
167 // Fast path: check for failed non-blocking operation without acquiring the lock.
169 // After observing that the channel is not closed, we observe that the channel is
170 // not ready for sending. Each of these observations is a single word-sized read
171 // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel).
172 // Because a closed channel cannot transition from 'ready for sending' to
173 // 'not ready for sending', even if the channel is closed between the two observations,
174 // they imply a moment between the two when the channel was both not yet closed
175 // and not ready for sending. We behave as if we observed the channel at that moment,
176 // and report that the send cannot proceed.
178 // It is okay if the reads are reordered here: if we observe that the channel is not
179 // ready for sending and then observe that it is not closed, that implies that the
180 // channel wasn't closed during the first observation.
181 if !block
&& c
.closed == 0 && ((c
.dataqsiz
== 0 && c
.recvq
.first
== nil) ||
182 (c
.dataqsiz
> 0 && c
.qcount
== c
.dataqsiz
)) {
187 if blockprofilerate
> 0 {
195 panic(plainError("send on closed channel"))
198 if sg
:= c
.recvq
.dequeue(); sg
!= nil {
199 // Found a waiting receiver. We pass the value we want to send
200 // directly to the receiver, bypassing the channel buffer (if any).
201 send(c
, sg
, ep
, func() { unlock(&c
.lock
) }, 3)
205 if c
.qcount
< c
.dataqsiz
{
206 // Space is available in the channel buffer. Enqueue the element to send.
207 qp
:= chanbuf(c
, c
.sendx
)
212 typedmemmove(c
.elemtype
, qp
, ep
)
214 if c
.sendx
== c
.dataqsiz
{
227 // Block on the channel. Some receiver will complete our operation for us.
229 mysg
:= acquireSudog()
232 mysg
.releasetime
= -1
234 // No stack splits between assigning elem and enqueuing mysg
235 // on gp.waiting where copystack can find it.
239 mysg
.isSelect
= false
243 c
.sendq
.enqueue(mysg
)
244 goparkunlock(&c
.lock
, "chan send", traceEvGoBlockSend
, 3)
246 // someone woke us up.
247 if mysg
!= gp
.waiting
{
248 throw("G waiting list is corrupted")
253 throw("chansend: spurious wakeup")
255 panic(plainError("send on closed channel"))
258 if mysg
.releasetime
> 0 {
259 blockevent(mysg
.releasetime
-t0
, 2)
266 // send processes a send operation on an empty channel c.
267 // The value ep sent by the sender is copied to the receiver sg.
268 // The receiver is then woken up to go on its merry way.
269 // Channel c must be empty and locked. send unlocks c with unlockf.
270 // sg must already be dequeued from c.
271 // ep must be non-nil and point to the heap or the caller's stack.
272 func send(c
*hchan
, sg
*sudog
, ep unsafe
.Pointer
, unlockf
func(), skip
int) {
277 // Pretend we go through the buffer, even though
278 // we copy directly. Note that we need to increment
279 // the head/tail locations only when raceenabled.
280 qp
:= chanbuf(c
, c
.recvx
)
283 raceacquireg(sg
.g
, qp
)
284 racereleaseg(sg
.g
, qp
)
286 if c
.recvx
== c
.dataqsiz
{
289 c
.sendx
= c
.recvx
// c.sendx = (c.sendx+1) % c.dataqsiz
293 sendDirect(c
.elemtype
, sg
, ep
)
298 gp
.param
= unsafe
.Pointer(sg
)
299 if sg
.releasetime
!= 0 {
300 sg
.releasetime
= cputicks()
305 // Sends and receives on unbuffered or empty-buffered channels are the
306 // only operations where one running goroutine writes to the stack of
307 // another running goroutine. The GC assumes that stack writes only
308 // happen when the goroutine is running and are only done by that
309 // goroutine. Using a write barrier is sufficient to make up for
310 // violating that assumption, but the write barrier has to work.
311 // typedmemmove will call bulkBarrierPreWrite, but the target bytes
312 // are not in the heap, so that will not help. We arrange to call
313 // memmove and typeBitsBulkBarrier instead.
315 func sendDirect(t
*_type
, sg
*sudog
, src unsafe
.Pointer
) {
316 // src is on our stack, dst is a slot on another stack.
318 // Once we read sg.elem out of sg, it will no longer
319 // be updated if the destination's stack gets copied (shrunk).
320 // So make sure that no preemption points can happen between read & use.
322 typeBitsBulkBarrier(t
, uintptr(dst
), uintptr(src
), t
.size
)
323 memmove(dst
, src
, t
.size
)
326 func recvDirect(t
*_type
, sg
*sudog
, dst unsafe
.Pointer
) {
327 // dst is on our stack or the heap, src is on another stack.
328 // The channel is locked, so src will not move during this
331 typeBitsBulkBarrier(t
, uintptr(dst
), uintptr(src
), t
.size
)
332 memmove(dst
, src
, t
.size
)
335 func closechan(c
*hchan
) {
337 panic(plainError("close of nil channel"))
343 panic(plainError("close of closed channel"))
347 callerpc
:= getcallerpc()
348 racewritepc(unsafe
.Pointer(c
), callerpc
, funcPC(closechan
))
349 racerelease(unsafe
.Pointer(c
))
356 // release all readers
358 sg
:= c
.recvq
.dequeue()
363 typedmemclr(c
.elemtype
, sg
.elem
)
366 if sg
.releasetime
!= 0 {
367 sg
.releasetime
= cputicks()
372 raceacquireg(gp
, unsafe
.Pointer(c
))
374 gp
.schedlink
.set(glist
)
378 // release all writers (they will panic)
380 sg
:= c
.sendq
.dequeue()
385 if sg
.releasetime
!= 0 {
386 sg
.releasetime
= cputicks()
391 raceacquireg(gp
, unsafe
.Pointer(c
))
393 gp
.schedlink
.set(glist
)
398 // Ready all Gs now that we've dropped the channel lock.
401 glist
= glist
.schedlink
.ptr()
407 // entry points for <- c from compiled code
409 func chanrecv1(c
*hchan
, elem unsafe
.Pointer
) {
410 chanrecv(c
, elem
, true)
414 func chanrecv2(c
*hchan
, elem unsafe
.Pointer
) (received
bool) {
415 _
, received
= chanrecv(c
, elem
, true)
419 // chanrecv receives on channel c and writes the received data to ep.
420 // ep may be nil, in which case received data is ignored.
421 // If block == false and no elements are available, returns (false, false).
422 // Otherwise, if c is closed, zeros *ep and returns (true, false).
423 // Otherwise, fills in *ep with an element and returns (true, true).
424 // A non-nil ep must point to the heap or the caller's stack.
425 func chanrecv(c
*hchan
, ep unsafe
.Pointer
, block
bool) (selected
, received
bool) {
426 // raceenabled: don't need to check ep, as it is always on the stack
427 // or is new memory allocated by reflect.
430 print("chanrecv: chan=", c
, "\n")
437 gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop
, 2)
441 // Fast path: check for failed non-blocking operation without acquiring the lock.
443 // After observing that the channel is not ready for receiving, we observe that the
444 // channel is not closed. Each of these observations is a single word-sized read
445 // (first c.sendq.first or c.qcount, and second c.closed).
446 // Because a channel cannot be reopened, the later observation of the channel
447 // being not closed implies that it was also not closed at the moment of the
448 // first observation. We behave as if we observed the channel at that moment
449 // and report that the receive cannot proceed.
451 // The order of operations is important here: reversing the operations can lead to
452 // incorrect behavior when racing with a close.
453 if !block
&& (c
.dataqsiz
== 0 && c
.sendq
.first
== nil ||
454 c
.dataqsiz
> 0 && atomic
.Loaduint(&c
.qcount
) == 0) &&
455 atomic
.Load(&c
.closed) == 0 {
460 if blockprofilerate
> 0 {
466 if c
.closed != 0 && c
.qcount
== 0 {
468 raceacquire(unsafe
.Pointer(c
))
472 typedmemclr(c
.elemtype
, ep
)
477 if sg
:= c
.sendq
.dequeue(); sg
!= nil {
478 // Found a waiting sender. If buffer is size 0, receive value
479 // directly from sender. Otherwise, receive from head of queue
480 // and add sender's value to the tail of the queue (both map to
481 // the same buffer slot because the queue is full).
482 recv(c
, sg
, ep
, func() { unlock(&c
.lock
) }, 3)
487 // Receive directly from queue
488 qp
:= chanbuf(c
, c
.recvx
)
494 typedmemmove(c
.elemtype
, ep
, qp
)
496 typedmemclr(c
.elemtype
, qp
)
498 if c
.recvx
== c
.dataqsiz
{
511 // no sender available: block on this channel.
513 mysg
:= acquireSudog()
516 mysg
.releasetime
= -1
518 // No stack splits between assigning elem and enqueuing mysg
519 // on gp.waiting where copystack can find it.
524 mysg
.isSelect
= false
527 c
.recvq
.enqueue(mysg
)
528 goparkunlock(&c
.lock
, "chan receive", traceEvGoBlockRecv
, 3)
530 // someone woke us up
531 if mysg
!= gp
.waiting
{
532 throw("G waiting list is corrupted")
535 if mysg
.releasetime
> 0 {
536 blockevent(mysg
.releasetime
-t0
, 2)
538 closed := gp
.param
== nil
545 // recv processes a receive operation on a full channel c.
546 // There are 2 parts:
547 // 1) The value sent by the sender sg is put into the channel
548 // and the sender is woken up to go on its merry way.
549 // 2) The value received by the receiver (the current G) is
551 // For synchronous channels, both values are the same.
552 // For asynchronous channels, the receiver gets its data from
553 // the channel buffer and the sender's data is put in the
555 // Channel c must be full and locked. recv unlocks c with unlockf.
556 // sg must already be dequeued from c.
557 // A non-nil ep must point to the heap or the caller's stack.
558 func recv(c
*hchan
, sg
*sudog
, ep unsafe
.Pointer
, unlockf
func(), skip
int) {
564 // copy data from sender
565 recvDirect(c
.elemtype
, sg
, ep
)
568 // Queue is full. Take the item at the
569 // head of the queue. Make the sender enqueue
570 // its item at the tail of the queue. Since the
571 // queue is full, those are both the same slot.
572 qp
:= chanbuf(c
, c
.recvx
)
576 raceacquireg(sg
.g
, qp
)
577 racereleaseg(sg
.g
, qp
)
579 // copy data from queue to receiver
581 typedmemmove(c
.elemtype
, ep
, qp
)
583 // copy data from sender to queue
584 typedmemmove(c
.elemtype
, qp
, sg
.elem
)
586 if c
.recvx
== c
.dataqsiz
{
589 c
.sendx
= c
.recvx
// c.sendx = (c.sendx+1) % c.dataqsiz
594 gp
.param
= unsafe
.Pointer(sg
)
595 if sg
.releasetime
!= 0 {
596 sg
.releasetime
= cputicks()
601 // compiler implements
612 // if selectnbsend(c, v) {
618 func selectnbsend(c
*hchan
, elem unsafe
.Pointer
) (selected
bool) {
619 return chansend(c
, elem
, false, getcallerpc())
622 // compiler implements
633 // if selectnbrecv(&v, c) {
639 func selectnbrecv(elem unsafe
.Pointer
, c
*hchan
) (selected
bool) {
640 selected
, _
= chanrecv(c
, elem
, false)
644 // compiler implements
655 // if c != nil && selectnbrecv2(&v, &ok, c) {
661 func selectnbrecv2(elem unsafe
.Pointer
, received
*bool, c
*hchan
) (selected
bool) {
662 // TODO(khr): just return 2 values from this function, now that it is in Go.
663 selected
, *received
= chanrecv(c
, elem
, false)
667 //go:linkname reflect_chansend reflect.chansend
668 func reflect_chansend(c
*hchan
, elem unsafe
.Pointer
, nb
bool) (selected
bool) {
669 return chansend(c
, elem
, !nb
, getcallerpc())
672 //go:linkname reflect_chanrecv reflect.chanrecv
673 func reflect_chanrecv(c
*hchan
, nb
bool, elem unsafe
.Pointer
) (selected
bool, received
bool) {
674 return chanrecv(c
, elem
, !nb
)
677 //go:linkname reflect_chanlen reflect.chanlen
678 func reflect_chanlen(c
*hchan
) int {
685 //go:linkname reflect_chancap reflect.chancap
686 func reflect_chancap(c
*hchan
) int {
690 return int(c
.dataqsiz
)
693 //go:linkname reflect_chanclose reflect.chanclose
694 func reflect_chanclose(c
*hchan
) {
698 func (q
*waitq
) enqueue(sgp
*sudog
) {
712 func (q
*waitq
) dequeue() *sudog
{
725 sgp
.next
= nil // mark as removed (see dequeueSudog)
728 // if a goroutine was put on this queue because of a
729 // select, there is a small window between the goroutine
730 // being woken up by a different case and it grabbing the
731 // channel locks. Once it has the lock
732 // it removes itself from the queue, so we won't see it after that.
733 // We use a flag in the G struct to tell us when someone
734 // else has won the race to signal this goroutine but the goroutine
735 // hasn't removed itself from the queue yet.
737 if !atomic
.Cas(&sgp
.g
.selectDone
, 0, 1) {
746 func racesync(c
*hchan
, sg
*sudog
) {
747 racerelease(chanbuf(c
, 0))
748 raceacquireg(sg
.g
, chanbuf(c
, 0))
749 racereleaseg(sg
.g
, chanbuf(c
, 0))
750 raceacquire(chanbuf(c
, 0))