1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // +build aix darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows
10 "runtime/internal/atomic"
14 // Export temporarily for gccgo's C code to call:
15 //go:linkname netpoll runtime.netpoll
17 // Integrated network poller (platform-independent part).
18 // A particular implementation (epoll/kqueue) must define the following functions:
19 // func netpollinit() // to initialize the poller
20 // func netpollopen(fd uintptr, pd *pollDesc) int32 // to arm edge-triggered notifications
21 // and associate fd with pd.
22 // An implementation must call the following function to denote that the pd is ready.
23 // func netpollready(gpp **g, pd *pollDesc, mode int32)
25 // pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer
26 // goroutines respectively. The semaphore can be in the following states:
27 // pdReady - io readiness notification is pending;
28 // a goroutine consumes the notification by changing the state to nil.
29 // pdWait - a goroutine prepares to park on the semaphore, but not yet parked;
30 // the goroutine commits to park by changing the state to G pointer,
31 // or, alternatively, concurrent io notification changes the state to READY,
32 // or, alternatively, concurrent timeout/close changes the state to nil.
33 // G pointer - the goroutine is blocked on the semaphore;
34 // io notification or timeout/close changes the state to READY or nil respectively
35 // and unparks the goroutine.
36 // nil - nothing of the above.
42 const pollBlockSize
= 4 * 1024
44 // Network poller descriptor.
49 type pollDesc
struct {
50 link
*pollDesc
// in pollcache, protected by pollcache.lock
52 // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations.
53 // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime.
54 // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification)
55 // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated
56 // in a lock-free way by all operations.
57 // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg),
58 // that will blow up when GC starts moving objects.
59 lock mutex
// protects the following fields
62 seq
uintptr // protects from stale timers and ready notifications
63 rg
uintptr // pdReady, pdWait, G waiting for read or nil
64 rt timer
// read deadline timer (set if rt.f != nil)
65 rd
int64 // read deadline
66 wg
uintptr // pdReady, pdWait, G waiting for write or nil
67 wt timer
// write deadline timer
68 wd
int64 // write deadline
69 user
uint32 // user settable cookie
72 type pollCache
struct {
75 // PollDesc objects must be type-stable,
76 // because we can get ready notification from epoll/kqueue
77 // after the descriptor is closed/reused.
78 // Stale notifications are detected using seq variable,
79 // seq is incremented when deadlines are changed or descriptor is reused.
87 //go:linkname net_runtime_pollServerInit net.runtime_pollServerInit
88 func net_runtime_pollServerInit() {
90 atomic
.Store(&netpollInited
, 1)
93 func netpollinited() bool {
94 return atomic
.Load(&netpollInited
) != 0
97 //go:linkname net_runtime_pollOpen net.runtime_pollOpen
98 func net_runtime_pollOpen(fd
uintptr) (*pollDesc
, int) {
99 pd
:= pollcache
.alloc()
101 if pd
.wg
!= 0 && pd
.wg
!= pdReady
{
102 throw("netpollOpen: blocked write on free descriptor")
104 if pd
.rg
!= 0 && pd
.rg
!= pdReady
{
105 throw("netpollOpen: blocked read on free descriptor")
117 errno
= netpollopen(fd
, pd
)
118 return pd
, int(errno
)
121 //go:linkname net_runtime_pollClose net.runtime_pollClose
122 func net_runtime_pollClose(pd
*pollDesc
) {
124 throw("netpollClose: close w/o unblock")
126 if pd
.wg
!= 0 && pd
.wg
!= pdReady
{
127 throw("netpollClose: blocked write on closing descriptor")
129 if pd
.rg
!= 0 && pd
.rg
!= pdReady
{
130 throw("netpollClose: blocked read on closing descriptor")
136 func (c
*pollCache
) free(pd
*pollDesc
) {
143 //go:linkname net_runtime_pollReset net.runtime_pollReset
144 func net_runtime_pollReset(pd
*pollDesc
, mode
int) int {
145 err
:= netpollcheckerr(pd
, int32(mode
))
151 } else if mode
== 'w' {
157 //go:linkname net_runtime_pollWait net.runtime_pollWait
158 func net_runtime_pollWait(pd
*pollDesc
, mode
int) int {
159 err
:= netpollcheckerr(pd
, int32(mode
))
163 // As for now only Solaris uses level-triggered IO.
164 if GOOS
== "solaris" {
167 for !netpollblock(pd
, int32(mode
), false) {
168 err
= netpollcheckerr(pd
, int32(mode
))
172 // Can happen if timeout has fired and unblocked us,
173 // but before we had a chance to run, timeout has been reset.
174 // Pretend it has not happened and retry.
179 //go:linkname net_runtime_pollWaitCanceled net.runtime_pollWaitCanceled
180 func net_runtime_pollWaitCanceled(pd
*pollDesc
, mode
int) {
181 // This function is used only on windows after a failed attempt to cancel
182 // a pending async IO operation. Wait for ioready, ignore closing or timeouts.
183 for !netpollblock(pd
, int32(mode
), true) {
187 //go:linkname net_runtime_pollSetDeadline net.runtime_pollSetDeadline
188 func net_runtime_pollSetDeadline(pd
*pollDesc
, d
int64, mode
int) {
194 pd
.seq
++ // invalidate current timers
195 // Reset current timers.
205 if d
!= 0 && d
<= nanotime() {
208 if mode
== 'r' || mode
== 'r'+'w' {
211 if mode
== 'w' || mode
== 'r'+'w' {
214 if pd
.rd
> 0 && pd
.rd
== pd
.wd
{
215 pd
.rt
.f
= netpollDeadline
217 // Copy current seq into the timer arg.
218 // Timer func will check the seq against current descriptor seq,
219 // if they differ the descriptor was reused or timers were reset.
225 pd
.rt
.f
= netpollReadDeadline
232 pd
.wt
.f
= netpollWriteDeadline
239 // If we set the new deadline in the past, unblock currently pending IO if any.
241 atomicstorep(unsafe
.Pointer(&wg
), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock
243 rg
= netpollunblock(pd
, 'r', false)
246 wg
= netpollunblock(pd
, 'w', false)
257 //go:linkname net_runtime_pollUnblock net.runtime_pollUnblock
258 func net_runtime_pollUnblock(pd
*pollDesc
) {
261 throw("netpollUnblock: already closing")
266 atomicstorep(unsafe
.Pointer(&rg
), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock
267 rg
= netpollunblock(pd
, 'r', false)
268 wg
= netpollunblock(pd
, 'w', false)
286 // make pd ready, newly runnable goroutines (if any) are returned in rg/wg
287 // May run during STW, so write barriers are not allowed.
289 func netpollready(gpp
*guintptr
, pd
*pollDesc
, mode
int32) {
291 if mode
== 'r' || mode
== 'r'+'w' {
292 rg
.set(netpollunblock(pd
, 'r', true))
294 if mode
== 'w' || mode
== 'r'+'w' {
295 wg
.set(netpollunblock(pd
, 'w', true))
298 rg
.ptr().schedlink
= *gpp
302 wg
.ptr().schedlink
= *gpp
307 func netpollcheckerr(pd
*pollDesc
, mode
int32) int {
309 return 1 // errClosing
311 if (mode
== 'r' && pd
.rd
< 0) ||
(mode
== 'w' && pd
.wd
< 0) {
312 return 2 // errTimeout
317 func netpollblockcommit(gp
*g
, gpp unsafe
.Pointer
) bool {
318 return atomic
.Casuintptr((*uintptr)(gpp
), pdWait
, uintptr(unsafe
.Pointer(gp
)))
321 // returns true if IO is ready, or false if timedout or closed
322 // waitio - wait only for completed IO, ignore errors
323 func netpollblock(pd
*pollDesc
, mode
int32, waitio
bool) bool {
329 // set the gpp semaphore to WAIT
337 throw("netpollblock: double wait")
339 if atomic
.Casuintptr(gpp
, 0, pdWait
) {
344 // need to recheck error states after setting gpp to WAIT
345 // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
346 // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg
347 if waitio ||
netpollcheckerr(pd
, mode
) == 0 {
348 gopark(netpollblockcommit
, unsafe
.Pointer(gpp
), "IO wait", traceEvGoBlockNet
, 5)
350 // be careful to not lose concurrent READY notification
351 old
:= atomic
.Xchguintptr(gpp
, 0)
353 throw("netpollblock: corrupted state")
355 return old
== pdReady
358 func netpollunblock(pd
*pollDesc
, mode
int32, ioready
bool) *g
{
369 if old
== 0 && !ioready
{
370 // Only set READY for ioready. runtime_pollWait
371 // will check for timeout/cancel before waiting.
378 if atomic
.Casuintptr(gpp
, old
, new) {
379 if old
== pdReady || old
== pdWait
{
382 return (*g
)(unsafe
.Pointer(old
))
387 func netpolldeadlineimpl(pd
*pollDesc
, seq
uintptr, read
, write
bool) {
389 // Seq arg is seq when the timer was set.
390 // If it's stale, ignore the timer event.
392 // The descriptor was reused or timers were reset.
398 if pd
.rd
<= 0 || pd
.rt
.f
== nil {
399 throw("netpolldeadlineimpl: inconsistent read deadline")
402 atomicstorep(unsafe
.Pointer(&pd
.rt
.f
), nil) // full memory barrier between store to rd and load of rg in netpollunblock
403 rg
= netpollunblock(pd
, 'r', false)
407 if pd
.wd
<= 0 || pd
.wt
.f
== nil && !read
{
408 throw("netpolldeadlineimpl: inconsistent write deadline")
411 atomicstorep(unsafe
.Pointer(&pd
.wt
.f
), nil) // full memory barrier between store to wd and load of wg in netpollunblock
412 wg
= netpollunblock(pd
, 'w', false)
423 func netpollDeadline(arg
interface{}, seq
uintptr) {
424 netpolldeadlineimpl(arg
.(*pollDesc
), seq
, true, true)
427 func netpollReadDeadline(arg
interface{}, seq
uintptr) {
428 netpolldeadlineimpl(arg
.(*pollDesc
), seq
, true, false)
431 func netpollWriteDeadline(arg
interface{}, seq
uintptr) {
432 netpolldeadlineimpl(arg
.(*pollDesc
), seq
, false, true)
435 func (c
*pollCache
) alloc() *pollDesc
{
438 const pdSize
= unsafe
.Sizeof(pollDesc
{})
439 n
:= pollBlockSize
/ pdSize
443 // Must be in non-GC memory because can be referenced
444 // only from epoll/kqueue internals.
445 mem
:= persistentalloc(n
*pdSize
, 0, &memstats
.other_sys
)
446 for i
:= uintptr(0); i
< n
; i
++ {
447 pd
:= (*pollDesc
)(add(mem
, i
*pdSize
))