1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
9 "runtime/internal/atomic"
10 "runtime/internal/sys"
14 // Functions called by C code.
15 //go:linkname main runtime.main
16 //go:linkname goparkunlock runtime.goparkunlock
17 //go:linkname newextram runtime.newextram
18 //go:linkname acquirep runtime.acquirep
19 //go:linkname releasep runtime.releasep
20 //go:linkname incidlelocked runtime.incidlelocked
21 //go:linkname schedinit runtime.schedinit
22 //go:linkname ready runtime.ready
23 //go:linkname gcprocs runtime.gcprocs
24 //go:linkname stopm runtime.stopm
25 //go:linkname handoffp runtime.handoffp
26 //go:linkname wakep runtime.wakep
27 //go:linkname stoplockedm runtime.stoplockedm
28 //go:linkname schedule runtime.schedule
29 //go:linkname execute runtime.execute
30 //go:linkname goexit1 runtime.goexit1
31 //go:linkname reentersyscall runtime.reentersyscall
32 //go:linkname reentersyscallblock runtime.reentersyscallblock
33 //go:linkname exitsyscall runtime.exitsyscall
34 //go:linkname gfget runtime.gfget
35 //go:linkname helpgc runtime.helpgc
36 //go:linkname kickoff runtime.kickoff
37 //go:linkname mstart1 runtime.mstart1
38 //go:linkname mexit runtime.mexit
39 //go:linkname globrunqput runtime.globrunqput
40 //go:linkname pidleget runtime.pidleget
42 // Exported for test (see runtime/testdata/testprogcgo/dropm_stub.go).
43 //go:linkname getm runtime.getm
45 // Function called by misc/cgo/test.
46 //go:linkname lockedOSThread runtime.lockedOSThread
48 // C functions for thread and context management.
52 func malg(bool, bool, *unsafe
.Pointer
, *uintptr) *g
55 func resetNewG(*g
, *unsafe
.Pointer
, *uintptr)
58 func makeGContext(*g
, unsafe
.Pointer
, uintptr)
59 func getTraceback(me
, gp
*g
)
61 func _cgo_notify_runtime_init_done()
62 func alreadyInCallers() bool
65 // Functions created by the compiler.
66 //extern __go_init_main
72 var buildVersion
= sys
.TheVersion
74 // Goroutine scheduler
75 // The scheduler's job is to distribute ready-to-run goroutines over worker threads.
77 // The main concepts are:
79 // M - worker thread, or machine.
80 // P - processor, a resource that is required to execute Go code.
81 // M must have an associated P to execute Go code, however it can be
82 // blocked or in a syscall w/o an associated P.
84 // Design doc at https://golang.org/s/go11sched.
86 // Worker thread parking/unparking.
87 // We need to balance between keeping enough running worker threads to utilize
88 // available hardware parallelism and parking excessive running worker threads
89 // to conserve CPU resources and power. This is not simple for two reasons:
90 // (1) scheduler state is intentionally distributed (in particular, per-P work
91 // queues), so it is not possible to compute global predicates on fast paths;
92 // (2) for optimal thread management we would need to know the future (don't park
93 // a worker thread when a new goroutine will be readied in near future).
95 // Three rejected approaches that would work badly:
96 // 1. Centralize all scheduler state (would inhibit scalability).
97 // 2. Direct goroutine handoff. That is, when we ready a new goroutine and there
98 // is a spare P, unpark a thread and handoff it the thread and the goroutine.
99 // This would lead to thread state thrashing, as the thread that readied the
100 // goroutine can be out of work the very next moment, we will need to park it.
101 // Also, it would destroy locality of computation as we want to preserve
102 // dependent goroutines on the same thread; and introduce additional latency.
103 // 3. Unpark an additional thread whenever we ready a goroutine and there is an
104 // idle P, but don't do handoff. This would lead to excessive thread parking/
105 // unparking as the additional threads will instantly park without discovering
108 // The current approach:
109 // We unpark an additional thread when we ready a goroutine if (1) there is an
110 // idle P and there are no "spinning" worker threads. A worker thread is considered
111 // spinning if it is out of local work and did not find work in global run queue/
112 // netpoller; the spinning state is denoted in m.spinning and in sched.nmspinning.
113 // Threads unparked this way are also considered spinning; we don't do goroutine
114 // handoff so such threads are out of work initially. Spinning threads do some
115 // spinning looking for work in per-P run queues before parking. If a spinning
116 // thread finds work it takes itself out of the spinning state and proceeds to
117 // execution. If it does not find work it takes itself out of the spinning state
119 // If there is at least one spinning thread (sched.nmspinning>1), we don't unpark
120 // new threads when readying goroutines. To compensate for that, if the last spinning
121 // thread finds work and stops spinning, it must unpark a new spinning thread.
122 // This approach smooths out unjustified spikes of thread unparking,
123 // but at the same time guarantees eventual maximal CPU parallelism utilization.
125 // The main implementation complication is that we need to be very careful during
126 // spinning->non-spinning thread transition. This transition can race with submission
127 // of a new goroutine, and either one part or another needs to unpark another worker
128 // thread. If they both fail to do that, we can end up with semi-persistent CPU
129 // underutilization. The general pattern for goroutine readying is: submit a goroutine
130 // to local work queue, #StoreLoad-style memory barrier, check sched.nmspinning.
131 // The general pattern for spinning->non-spinning transition is: decrement nmspinning,
132 // #StoreLoad-style memory barrier, check all per-P work queues for new work.
133 // Note that all this complexity does not apply to global run queue as we are not
134 // sloppy about thread unparking when submitting to global queue. Also see comments
135 // for nmspinning manipulation.
142 // main_init_done is a signal used by cgocallbackg that initialization
143 // has been completed. It is made before _cgo_notify_runtime_init_done,
144 // so all cgo calls can rely on it existing. When main_init is complete,
145 // it is closed, meaning cgocallbackg can reliably receive from it.
146 var main_init_done
chan bool
148 // mainStarted indicates that the main M has started.
151 // runtimeInitTime is the nanotime() at which the runtime started.
152 var runtimeInitTime
int64
154 // Value to use for signal mask for newly created M's.
155 var initSigmask sigset
157 // The main goroutine.
161 // Max stack size is 1 GB on 64-bit, 250 MB on 32-bit.
162 // Using decimal instead of binary GB and MB because
163 // they look nicer in the stack overflow failure message.
164 if sys
.PtrSize
== 8 {
165 maxstacksize
= 1000000000
167 maxstacksize
= 250000000
170 // Allow newproc to start new Ms.
173 if GOARCH
!= "wasm" { // no threads on wasm yet, so no sysmon
179 // Lock the main goroutine onto this, the main OS thread,
180 // during initialization. Most programs won't care, but a few
181 // do require certain calls to be made by the main thread.
182 // Those can arrange for main.main to run in the main thread
183 // by calling runtime.LockOSThread during initialization
184 // to preserve the lock.
188 throw("runtime.main not on m0")
191 // Defer unlock so that runtime.Goexit during init does the unlock too.
199 // Record when the world started. Must be after runtime_init
200 // because nanotime on some platforms depends on startNano.
201 runtimeInitTime
= nanotime()
203 main_init_done
= make(chan bool)
205 // Start the template thread in case we enter Go from
206 // a C-created thread and need to create a new thread.
207 startTemplateThread()
208 _cgo_notify_runtime_init_done()
211 fn
:= main_init
// make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime
214 close(main_init_done
)
219 // For gccgo we have to wait until after main is initialized
220 // to enable GC, because initializing main registers the GC roots.
223 if isarchive || islibrary
{
224 // A program compiled with -buildmode=c-archive or c-shared
225 // has a main, but it is not executed.
228 fn
= main_main
// make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime
234 // Make racy client program work: if panicking on
235 // another goroutine at the same time as main returns,
236 // let the other goroutine finish printing the panic trace.
237 // Once it does, it will exit. See issues 3934 and 20018.
238 if atomic
.Load(&runningPanicDefers
) != 0 {
239 // Running deferred functions should not take long.
240 for c
:= 0; c
< 1000; c
++ {
241 if atomic
.Load(&runningPanicDefers
) == 0 {
247 if atomic
.Load(&panicking
) != 0 {
248 gopark(nil, nil, waitReasonPanicWait
, traceEvGoStop
, 1)
258 // os_beforeExit is called from os.Exit(0).
259 //go:linkname os_beforeExit os.runtime_beforeExit
260 func os_beforeExit() {
266 // start forcegc helper goroutine
268 expectSystemGoroutine()
272 func forcegchelper() {
278 if forcegc
.idle
!= 0 {
279 throw("forcegc: phase error")
281 atomic
.Store(&forcegc
.idle
, 1)
282 goparkunlock(&forcegc
.lock
, waitReasonForceGGIdle
, traceEvGoBlock
, 1)
283 // this goroutine is explicitly resumed by sysmon
284 if debug
.gctrace
> 0 {
287 // Time-triggered, fully concurrent.
288 gcStart(gcBackgroundMode
, gcTrigger
{kind
: gcTriggerTime
, now
: nanotime()})
294 // Gosched yields the processor, allowing other goroutines to run. It does not
295 // suspend the current goroutine, so execution resumes automatically.
301 // goschedguarded yields the processor like gosched, but also checks
302 // for forbidden states and opts out of the yield in those cases.
304 func goschedguarded() {
305 mcall(goschedguarded_m
)
308 // Puts the current goroutine into a waiting state and calls unlockf.
309 // If unlockf returns false, the goroutine is resumed.
310 // unlockf must not access this G's stack, as it may be moved between
311 // the call to gopark and the call to unlockf.
312 // Reason explains why the goroutine has been parked.
313 // It is displayed in stack traces and heap dumps.
314 // Reasons should be unique and descriptive.
315 // Do not re-use reasons, add new ones.
316 func gopark(unlockf
func(*g
, unsafe
.Pointer
) bool, lock unsafe
.Pointer
, reason waitReason
, traceEv
byte, traceskip
int) {
317 if reason
!= waitReasonSleep
{
318 checkTimeouts() // timeouts may expire while two goroutines keep the scheduler busy
322 status
:= readgstatus(gp
)
323 if status
!= _Grunning
&& status
!= _Gscanrunning
{
324 throw("gopark: bad g status")
327 mp
.waitunlockf
= *(*unsafe
.Pointer
)(unsafe
.Pointer(&unlockf
))
328 gp
.waitreason
= reason
329 mp
.waittraceev
= traceEv
330 mp
.waittraceskip
= traceskip
332 // can't do anything that might move the G between Ms here.
336 // Puts the current goroutine into a waiting state and unlocks the lock.
337 // The goroutine can be made runnable again by calling goready(gp).
338 func goparkunlock(lock
*mutex
, reason waitReason
, traceEv
byte, traceskip
int) {
339 gopark(parkunlock_c
, unsafe
.Pointer(lock
), reason
, traceEv
, traceskip
)
342 func goready(gp
*g
, traceskip
int) {
344 ready(gp
, traceskip
, true)
349 func acquireSudog() *sudog
{
350 // Delicate dance: the semaphore implementation calls
351 // acquireSudog, acquireSudog calls new(sudog),
352 // new calls malloc, malloc can call the garbage collector,
353 // and the garbage collector calls the semaphore implementation
355 // Break the cycle by doing acquirem/releasem around new(sudog).
356 // The acquirem/releasem increments m.locks during new(sudog),
357 // which keeps the garbage collector from being invoked.
360 if len(pp
.sudogcache
) == 0 {
361 lock(&sched
.sudoglock
)
362 // First, try to grab a batch from central cache.
363 for len(pp
.sudogcache
) < cap(pp
.sudogcache
)/2 && sched
.sudogcache
!= nil {
364 s
:= sched
.sudogcache
365 sched
.sudogcache
= s
.next
367 pp
.sudogcache
= append(pp
.sudogcache
, s
)
369 unlock(&sched
.sudoglock
)
370 // If the central cache is empty, allocate a new one.
371 if len(pp
.sudogcache
) == 0 {
372 pp
.sudogcache
= append(pp
.sudogcache
, new(sudog
))
375 n
:= len(pp
.sudogcache
)
376 s
:= pp
.sudogcache
[n
-1]
377 pp
.sudogcache
[n
-1] = nil
378 pp
.sudogcache
= pp
.sudogcache
[:n
-1]
380 throw("acquireSudog: found s.elem != nil in cache")
387 func releaseSudog(s
*sudog
) {
389 throw("runtime: sudog with non-nil elem")
392 throw("runtime: sudog with non-false isSelect")
395 throw("runtime: sudog with non-nil next")
398 throw("runtime: sudog with non-nil prev")
400 if s
.waitlink
!= nil {
401 throw("runtime: sudog with non-nil waitlink")
404 throw("runtime: sudog with non-nil c")
408 throw("runtime: releaseSudog with non-nil gp.param")
410 mp
:= acquirem() // avoid rescheduling to another P
412 if len(pp
.sudogcache
) == cap(pp
.sudogcache
) {
413 // Transfer half of local cache to the central cache.
414 var first
, last
*sudog
415 for len(pp
.sudogcache
) > cap(pp
.sudogcache
)/2 {
416 n
:= len(pp
.sudogcache
)
417 p
:= pp
.sudogcache
[n
-1]
418 pp
.sudogcache
[n
-1] = nil
419 pp
.sudogcache
= pp
.sudogcache
[:n
-1]
427 lock(&sched
.sudoglock
)
428 last
.next
= sched
.sudogcache
429 sched
.sudogcache
= first
430 unlock(&sched
.sudoglock
)
432 pp
.sudogcache
= append(pp
.sudogcache
, s
)
436 // funcPC returns the entry PC of the function f.
437 // It assumes that f is a func value. Otherwise the behavior is undefined.
438 // CAREFUL: In programs with plugins, funcPC can return different values
439 // for the same function (because there are actually multiple copies of
440 // the same function in the address space). To be safe, don't use the
441 // results of this function in any == expression. It is only safe to
442 // use the result as an address at which to start executing code.
444 // For gccgo note that this differs from the gc implementation; the gc
445 // implementation adds sys.PtrSize to the address of the interface
446 // value, but GCC's alias analysis decides that that can not be a
447 // reference to the second field of the interface, and in some cases
448 // it drops the initialization of the second field as a dead store.
450 func funcPC(f
interface{}) uintptr {
451 i
:= (*iface
)(unsafe
.Pointer(&f
))
452 return **(**uintptr)(i
.data
)
455 func lockedOSThread() bool {
457 return gp
.lockedm
!= 0 && gp
.m
.lockedg
!= 0
465 func allgadd(gp
*g
) {
466 if readgstatus(gp
) == _Gidle
{
467 throw("allgadd: bad status Gidle")
471 allgs
= append(allgs
, gp
)
472 allglen
= uintptr(len(allgs
))
477 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.
478 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
482 // cpuinit extracts the environment variable GODEBUGCPU from the environment on
483 // Linux and Darwin if the GOEXPERIMENT debugcpu was set and calls internal/cpu.Initialize.
485 const prefix
= "GODEBUGCPU="
488 if haveexperiment("debugcpu") && (GOOS
== "linux" || GOOS
== "darwin") {
489 cpu
.DebugOptions
= true
491 // Similar to goenv_unix but extracts the environment value for
492 // GODEBUGCPU directly.
493 // TODO(moehrmann): remove when general goenvs() can be called before cpuinit()
495 for argv_index(argv
, argc
+1+n
) != nil {
499 for i
:= int32(0); i
< n
; i
++ {
500 p
:= argv_index(argv
, argc
+1+i
)
501 s
:= *(*string)(unsafe
.Pointer(&stringStruct
{unsafe
.Pointer(p
), findnull(p
)}))
503 if hasprefix(s
, prefix
) {
504 env
= gostring(p
)[len(prefix
):]
513 // The bootstrap sequence is:
517 // make & queue new G
518 // call runtime·mstart
520 // The new G calls runtime·main.
529 sched
.maxmcount
= 10000
533 cpuinit() // must run before alginit
534 alginit() // maps must not be used before this call
537 initSigmask
= _g_
.m
.sigmask
544 sched
.lastpoll
= uint64(nanotime())
546 if n
, ok
:= atoi32(gogetenv("GOMAXPROCS")); ok
&& n
> 0 {
549 if procresize(procs
) != nil {
550 throw("unknown runnable goroutine during bootstrap")
553 // For cgocheck > 1, we turn on the write barrier at all times
554 // and check all pointer writes. We can't do this until after
555 // procresize because the write barrier needs a P.
556 if debug
.cgocheck
> 1 {
557 writeBarrier
.cgo
= true
558 writeBarrier
.enabled
= true
559 for _
, p
:= range allp
{
564 if buildVersion
== "" {
565 // Condition should never trigger. This code just serves
566 // to ensure runtime·buildVersion is kept in the resulting binary.
567 buildVersion
= "unknown"
571 func dumpgstatus(gp
*g
) {
573 print("runtime: gp: gp=", gp
, ", goid=", gp
.goid
, ", gp->atomicstatus=", readgstatus(gp
), "\n")
574 print("runtime: g: g=", _g_
, ", goid=", _g_
.goid
, ", g->atomicstatus=", readgstatus(_g_
), "\n")
578 // sched lock is held
579 if mcount() > sched
.maxmcount
{
580 print("runtime: program exceeds ", sched
.maxmcount
, "-thread limit\n")
581 throw("thread exhaustion")
585 func mcommoninit(mp
*m
) {
588 // g0 stack won't make sense for user (and is not necessary unwindable).
590 callers(1, mp
.createstack
[:])
594 if sched
.mnext
+1 < sched
.mnext
{
595 throw("runtime: thread ID overflow")
601 mp
.fastrand
[0] = 1597334677 * uint32(mp
.id
)
602 mp
.fastrand
[1] = uint32(cputicks())
603 if mp
.fastrand
[0]|mp
.fastrand
[1] == 0 {
609 // Add to allm so garbage collector doesn't free g->m
610 // when it is just in a register or thread-local storage.
613 // NumCgoCall() iterates over allm w/o schedlock,
614 // so we need to publish it safely.
615 atomicstorep(unsafe
.Pointer(&allm
), unsafe
.Pointer(mp
))
619 // Mark gp ready to run.
620 func ready(gp
*g
, traceskip
int, next
bool) {
622 traceGoUnpark(gp
, traceskip
)
625 status
:= readgstatus(gp
)
629 _g_
.m
.locks
++ // disable preemption because it can be holding p in a local var
630 if status
&^_Gscan
!= _Gwaiting
{
632 throw("bad g->status in ready")
635 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
636 casgstatus(gp
, _Gwaiting
, _Grunnable
)
637 runqput(_g_
.m
.p
.ptr(), gp
, next
)
638 if atomic
.Load(&sched
.npidle
) != 0 && atomic
.Load(&sched
.nmspinning
) == 0 {
644 func gcprocs() int32 {
645 // Figure out how many CPUs to use during GC.
646 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
655 if n
> sched
.nmidle
+1 { // one M is currently running
662 func needaddgcproc() bool {
671 n
-= sched
.nmidle
+ 1 // one M is currently running
676 func helpgc(nproc
int32) {
680 for n
:= int32(1); n
< nproc
; n
++ { // one M is currently running
681 if allp
[pos
].mcache
== _g_
.m
.mcache
{
686 throw("gcprocs inconsistency")
690 mp
.mcache
= allp
[pos
].mcache
697 // freezeStopWait is a large value that freezetheworld sets
698 // sched.stopwait to in order to request that all Gs permanently stop.
699 const freezeStopWait
= 0x7fffffff
701 // freezing is set to non-zero if the runtime is trying to freeze the
705 // Similar to stopTheWorld but best-effort and can be called several times.
706 // There is no reverse operation, used during crashing.
707 // This function must not lock any mutexes.
708 func freezetheworld() {
709 atomic
.Store(&freezing
, 1)
710 // stopwait and preemption requests can be lost
711 // due to races with concurrently executing threads,
712 // so try several times
713 for i
:= 0; i
< 5; i
++ {
714 // this should tell the scheduler to not start any new goroutines
715 sched
.stopwait
= freezeStopWait
716 atomic
.Store(&sched
.gcwaiting
, 1)
717 // this should stop running goroutines
719 break // no running goroutines
729 func isscanstatus(status
uint32) bool {
730 if status
== _Gscan
{
731 throw("isscanstatus: Bad status Gscan")
733 return status
&_Gscan
== _Gscan
736 // All reads and writes of g's status go through readgstatus, casgstatus
737 // castogscanstatus, casfrom_Gscanstatus.
739 func readgstatus(gp
*g
) uint32 {
740 return atomic
.Load(&gp
.atomicstatus
)
743 // Ownership of gcscanvalid:
745 // If gp is running (meaning status == _Grunning or _Grunning|_Gscan),
746 // then gp owns gp.gcscanvalid, and other goroutines must not modify it.
748 // Otherwise, a second goroutine can lock the scan state by setting _Gscan
749 // in the status bit and then modify gcscanvalid, and then unlock the scan state.
751 // Note that the first condition implies an exception to the second:
752 // if a second goroutine changes gp's status to _Grunning|_Gscan,
753 // that second goroutine still does not have the right to modify gcscanvalid.
755 // The Gscanstatuses are acting like locks and this releases them.
756 // If it proves to be a performance hit we should be able to make these
757 // simple atomic stores but for now we are going to throw if
758 // we see an inconsistent state.
759 func casfrom_Gscanstatus(gp
*g
, oldval
, newval
uint32) {
762 // Check that transition is valid.
765 print("runtime: casfrom_Gscanstatus bad oldval gp=", gp
, ", oldval=", hex(oldval
), ", newval=", hex(newval
), "\n")
767 throw("casfrom_Gscanstatus:top gp->status is not in scan state")
772 if newval
== oldval
&^_Gscan
{
773 success
= atomic
.Cas(&gp
.atomicstatus
, oldval
, newval
)
777 print("runtime: casfrom_Gscanstatus failed gp=", gp
, ", oldval=", hex(oldval
), ", newval=", hex(newval
), "\n")
779 throw("casfrom_Gscanstatus: gp->status is not in scan state")
783 // This will return false if the gp is not in the expected status and the cas fails.
784 // This acts like a lock acquire while the casfromgstatus acts like a lock release.
785 func castogscanstatus(gp
*g
, oldval
, newval
uint32) bool {
791 if newval
== oldval|_Gscan
{
792 return atomic
.Cas(&gp
.atomicstatus
, oldval
, newval
)
795 print("runtime: castogscanstatus oldval=", hex(oldval
), " newval=", hex(newval
), "\n")
796 throw("castogscanstatus")
800 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
801 // and casfrom_Gscanstatus instead.
802 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
803 // put it in the Gscan state is finished.
805 func casgstatus(gp
*g
, oldval
, newval
uint32) {
806 if (oldval
&_Gscan
!= 0) ||
(newval
&_Gscan
!= 0) || oldval
== newval
{
808 print("runtime: casgstatus: oldval=", hex(oldval
), " newval=", hex(newval
), "\n")
809 throw("casgstatus: bad incoming values")
813 if oldval
== _Grunning
&& gp
.gcscanvalid
{
814 // If oldvall == _Grunning, then the actual status must be
815 // _Grunning or _Grunning|_Gscan; either way,
816 // we own gp.gcscanvalid, so it's safe to read.
817 // gp.gcscanvalid must not be true when we are running.
819 print("runtime: casgstatus ", hex(oldval
), "->", hex(newval
), " gp.status=", hex(gp
.atomicstatus
), " gp.gcscanvalid=true\n")
824 // See https://golang.org/cl/21503 for justification of the yield delay.
825 const yieldDelay
= 5 * 1000
828 // loop if gp->atomicstatus is in a scan state giving
829 // GC time to finish and change the state to oldval.
830 for i
:= 0; !atomic
.Cas(&gp
.atomicstatus
, oldval
, newval
); i
++ {
831 if oldval
== _Gwaiting
&& gp
.atomicstatus
== _Grunnable
{
832 throw("casgstatus: waiting for Gwaiting but is Grunnable")
834 // Help GC if needed.
835 // if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
836 // gp.preemptscan = false
837 // systemstack(func() {
841 // But meanwhile just yield.
843 nextYield
= nanotime() + yieldDelay
845 if nanotime() < nextYield
{
846 for x
:= 0; x
< 10 && gp
.atomicstatus
!= oldval
; x
++ {
851 nextYield
= nanotime() + yieldDelay
/2
854 if newval
== _Grunning
{
855 gp
.gcscanvalid
= false
859 // scang blocks until gp's stack has been scanned.
860 // It might be scanned by scang or it might be scanned by the goroutine itself.
861 // Either way, the stack scan has completed when scang returns.
862 func scang(gp
*g
, gcw
*gcWork
) {
863 // Invariant; we (the caller, markroot for a specific goroutine) own gp.gcscandone.
864 // Nothing is racing with us now, but gcscandone might be set to true left over
865 // from an earlier round of stack scanning (we scan twice per GC).
866 // We use gcscandone to record whether the scan has been done during this round.
868 gp
.gcscandone
= false
870 // See https://golang.org/cl/21503 for justification of the yield delay.
871 const yieldDelay
= 10 * 1000
874 // Endeavor to get gcscandone set to true,
875 // either by doing the stack scan ourselves or by coercing gp to scan itself.
876 // gp.gcscandone can transition from false to true when we're not looking
877 // (if we asked for preemption), so any time we lock the status using
878 // castogscanstatus we have to double-check that the scan is still not done.
880 for i
:= 0; !gp
.gcscandone
; i
++ {
881 switch s
:= readgstatus(gp
); s
{
884 throw("stopg: invalid status")
892 // Stack being switched. Go around again.
894 case _Grunnable
, _Gsyscall
, _Gwaiting
:
895 // Claim goroutine by setting scan bit.
896 // Racing with execution or readying of gp.
897 // The scan bit keeps them from running
898 // the goroutine until we're done.
899 if castogscanstatus(gp
, s
, s|_Gscan
) {
901 // Don't try to scan the stack
902 // if the goroutine is going to do
916 // newstack is doing a scan for us right now. Wait.
919 // checkPreempt is scanning. Wait.
922 // Goroutine running. Try to preempt execution so it can scan itself.
923 // The preemption handler (in newstack) does the actual scan.
925 // Optimization: if there is already a pending preemption request
926 // (from the previous loop iteration), don't bother with the atomics.
927 if gp
.preemptscan
&& gp
.preempt
{
931 // Ask for preemption and self scan.
932 if castogscanstatus(gp
, _Grunning
, _Gscanrunning
) {
934 gp
.preemptscan
= true
937 casfrom_Gscanstatus(gp
, _Gscanrunning
, _Grunning
)
942 nextYield
= nanotime() + yieldDelay
944 if nanotime() < nextYield
{
948 nextYield
= nanotime() + yieldDelay
/2
952 gp
.preemptscan
= false // cancel scan request if no longer needed
955 // The GC requests that this routine be moved from a scanmumble state to a mumble state.
956 func restartg(gp
*g
) {
961 throw("restartg: unexpected status")
969 casfrom_Gscanstatus(gp
, s
, s
&^_Gscan
)
973 // stopTheWorld stops all P's from executing goroutines, interrupting
974 // all goroutines at GC safe points and records reason as the reason
975 // for the stop. On return, only the current goroutine's P is running.
976 // stopTheWorld must not be called from a system stack and the caller
977 // must not hold worldsema. The caller must call startTheWorld when
978 // other P's should resume execution.
980 // stopTheWorld is safe for multiple goroutines to call at the
981 // same time. Each will execute its own stop, and the stops will
984 // This is also used by routines that do stack dumps. If the system is
985 // in panic or being exited, this may not reliably stop all
987 func stopTheWorld(reason
string) {
988 semacquire(&worldsema
)
989 getg().m
.preemptoff
= reason
990 systemstack(stopTheWorldWithSema
)
993 // startTheWorld undoes the effects of stopTheWorld.
994 func startTheWorld() {
995 systemstack(func() { startTheWorldWithSema(false) })
996 // worldsema must be held over startTheWorldWithSema to ensure
997 // gomaxprocs cannot change while worldsema is held.
998 semrelease(&worldsema
)
999 getg().m
.preemptoff
= ""
1002 // Holding worldsema grants an M the right to try to stop the world
1003 // and prevents gomaxprocs from changing concurrently.
1004 var worldsema
uint32 = 1
1006 // stopTheWorldWithSema is the core implementation of stopTheWorld.
1007 // The caller is responsible for acquiring worldsema and disabling
1008 // preemption first and then should stopTheWorldWithSema on the system
1011 // semacquire(&worldsema, 0)
1012 // m.preemptoff = "reason"
1013 // systemstack(stopTheWorldWithSema)
1015 // When finished, the caller must either call startTheWorld or undo
1016 // these three operations separately:
1018 // m.preemptoff = ""
1019 // systemstack(startTheWorldWithSema)
1020 // semrelease(&worldsema)
1022 // It is allowed to acquire worldsema once and then execute multiple
1023 // startTheWorldWithSema/stopTheWorldWithSema pairs.
1024 // Other P's are able to execute between successive calls to
1025 // startTheWorldWithSema and stopTheWorldWithSema.
1026 // Holding worldsema causes any other goroutines invoking
1027 // stopTheWorld to block.
1028 func stopTheWorldWithSema() {
1031 // If we hold a lock, then we won't be able to stop another M
1032 // that is blocked trying to acquire the lock.
1033 if _g_
.m
.locks
> 0 {
1034 throw("stopTheWorld: holding locks")
1038 sched
.stopwait
= gomaxprocs
1039 atomic
.Store(&sched
.gcwaiting
, 1)
1042 _g_
.m
.p
.ptr().status
= _Pgcstop
// Pgcstop is only diagnostic.
1044 // try to retake all P's in Psyscall status
1045 for _
, p
:= range allp
{
1047 if s
== _Psyscall
&& atomic
.Cas(&p
.status
, s
, _Pgcstop
) {
1065 wait
:= sched
.stopwait
> 0
1068 // wait for remaining P's to stop voluntarily
1071 // wait for 100us, then try to re-preempt in case of any races
1072 if notetsleep(&sched
.stopnote
, 100*1000) {
1073 noteclear(&sched
.stopnote
)
1082 if sched
.stopwait
!= 0 {
1083 bad
= "stopTheWorld: not stopped (stopwait != 0)"
1085 for _
, p
:= range allp
{
1086 if p
.status
!= _Pgcstop
{
1087 bad
= "stopTheWorld: not stopped (status != _Pgcstop)"
1091 if atomic
.Load(&freezing
) != 0 {
1092 // Some other thread is panicking. This can cause the
1093 // sanity checks above to fail if the panic happens in
1094 // the signal handler on a stopped thread. Either way,
1095 // we should halt this thread.
1109 func startTheWorldWithSema(emitTraceEvent
bool) int64 {
1112 _g_
.m
.locks
++ // disable preemption because it can be holding p in a local var
1113 if netpollinited() {
1114 gp
:= netpoll(false) // non-blocking
1117 add
:= needaddgcproc()
1125 p1
:= procresize(procs
)
1127 if sched
.sysmonwait
!= 0 {
1128 sched
.sysmonwait
= 0
1129 notewakeup(&sched
.sysmonnote
)
1140 throw("startTheWorld: inconsistent mp->nextp")
1143 notewakeup(&mp
.park
)
1145 // Start M to run P. Do not start another M below.
1151 // Capture start-the-world time before doing clean-up tasks.
1152 startTime
:= nanotime()
1157 // Wakeup an additional proc in case we have excessive runnable goroutines
1158 // in local queues or in the global queue. If we don't, the proc will park itself.
1159 // If we have lots of excessive work, resetspinning will unpark additional procs as necessary.
1160 if atomic
.Load(&sched
.npidle
) != 0 && atomic
.Load(&sched
.nmspinning
) == 0 {
1165 // If GC could have used another helper proc, start one now,
1166 // in the hope that it will be available next time.
1167 // It would have been even better to start it before the collection,
1168 // but doing so requires allocating memory, so it's tricky to
1169 // coordinate. This lazy approach works out in practice:
1170 // we don't mind if the first couple gc rounds don't have quite
1171 // the maximum number of procs.
1179 // First function run by a new goroutine.
1180 // This is passed to makecontext.
1184 if gp
.traceback
!= 0 {
1191 // When running on the g0 stack we can wind up here without a p,
1192 // for example from mcall(exitsyscall0) in exitsyscall, in
1193 // which case we can not run a write barrier.
1194 // It is also possible for us to get here from the systemstack
1195 // call in wbBufFlush, at which point the write barrier buffer
1196 // is full and we can not run a write barrier.
1197 // Setting gp.entry = nil or gp.param = nil will try to run a
1198 // write barrier, so if we are on the g0 stack due to mcall
1199 // (systemstack calls mcall) then clear the field using uintptr.
1200 // This is OK when gp.param is gp.m.curg, as curg will be kept
1201 // alive elsewhere, and gp.entry always points into g, or
1202 // to a statically allocated value, or (in the case of mcall)
1204 if gp
== gp
.m
.g0
&& gp
.param
== unsafe
.Pointer(gp
.m
.curg
) {
1205 *(*uintptr)(unsafe
.Pointer(&gp
.entry
)) = 0
1206 *(*uintptr)(unsafe
.Pointer(&gp
.param
)) = 0
1207 } else if gp
.m
.p
== 0 {
1208 throw("no p in kickoff")
1221 if _g_
!= _g_
.m
.g0
{
1222 throw("bad runtime·mstart")
1227 // Install signal handlers; after minit so that minit can
1228 // prepare the thread to be able to handle the signals.
1229 // For gccgo minit was called by C code.
1234 if fn
:= _g_
.m
.mstartfn
; fn
!= nil {
1238 if _g_
.m
.helpgc
!= 0 {
1241 } else if _g_
.m
!= &m0
{
1242 acquirep(_g_
.m
.nextp
.ptr())
1248 // mstartm0 implements part of mstart1 that only runs on the m0.
1250 // Write barriers are allowed here because we know the GC can't be
1251 // running yet, so they'll be no-ops.
1253 //go:yeswritebarrierrec
1255 // Create an extra M for callbacks on threads not created by Go.
1256 // An extra M is also needed on Windows for callbacks created by
1257 // syscall.NewCallback. See issue #6751 for details.
1258 if (iscgo || GOOS
== "windows") && !cgoHasExtraM
{
1265 // mexit tears down and exits the current thread.
1267 // Don't call this directly to exit the thread, since it must run at
1268 // the top of the thread stack. Instead, use gogo(&_g_.m.g0.sched) to
1269 // unwind the stack to the point that exits the thread.
1271 // It is entered with m.p != nil, so write barriers are allowed. It
1272 // will release the P before exiting.
1274 //go:yeswritebarrierrec
1275 func mexit(osStack
bool) {
1280 // This is the main thread. Just wedge it.
1282 // On Linux, exiting the main thread puts the process
1283 // into a non-waitable zombie state. On Plan 9,
1284 // exiting the main thread unblocks wait even though
1285 // other threads are still running. On Solaris we can
1286 // neither exitThread nor return from mstart. Other
1287 // bad things probably happen on other platforms.
1289 // We could try to clean up this M more before wedging
1290 // it, but that complicates signal handling.
1291 handoffp(releasep())
1297 throw("locked m0 woke up")
1303 // Free the gsignal stack.
1304 if m
.gsignal
!= nil {
1305 stackfree(m
.gsignal
)
1308 // Remove m from allm.
1310 for pprev
:= &allm
; *pprev
!= nil; pprev
= &(*pprev
).alllink
{
1316 throw("m not found in allm")
1319 // Delay reaping m until it's done with the stack.
1321 // If this is using an OS stack, the OS will free it
1322 // so there's no need for reaping.
1323 atomic
.Store(&m
.freeWait
, 1)
1324 // Put m on the free list, though it will not be reaped until
1325 // freeWait is 0. Note that the free list must not be linked
1326 // through alllink because some functions walk allm without
1327 // locking, so may be using alllink.
1328 m
.freelink
= sched
.freem
1334 handoffp(releasep())
1335 // After this point we must not have write barriers.
1337 // Invoke the deadlock detector. This must happen after
1338 // handoffp because it may have started a new M to take our
1346 // Return from mstart and let the system thread
1347 // library free the g0 stack and terminate the thread.
1351 // mstart is the thread's entry point, so there's nothing to
1352 // return to. Exit the thread directly. exitThread will clear
1353 // m.freeWait when it's done with the stack and the m can be
1355 exitThread(&m
.freeWait
)
1358 // forEachP calls fn(p) for every P p when p reaches a GC safe point.
1359 // If a P is currently executing code, this will bring the P to a GC
1360 // safe point and execute fn on that P. If the P is not executing code
1361 // (it is idle or in a syscall), this will call fn(p) directly while
1362 // preventing the P from exiting its state. This does not ensure that
1363 // fn will run on every CPU executing Go code, but it acts as a global
1364 // memory barrier. GC uses this as a "ragged barrier."
1366 // The caller must hold worldsema.
1369 func forEachP(fn
func(*p
)) {
1371 _p_
:= getg().m
.p
.ptr()
1374 if sched
.safePointWait
!= 0 {
1375 throw("forEachP: sched.safePointWait != 0")
1377 sched
.safePointWait
= gomaxprocs
- 1
1378 sched
.safePointFn
= fn
1380 // Ask all Ps to run the safe point function.
1381 for _
, p
:= range allp
{
1383 atomic
.Store(&p
.runSafePointFn
, 1)
1388 // Any P entering _Pidle or _Psyscall from now on will observe
1389 // p.runSafePointFn == 1 and will call runSafePointFn when
1390 // changing its status to _Pidle/_Psyscall.
1392 // Run safe point function for all idle Ps. sched.pidle will
1393 // not change because we hold sched.lock.
1394 for p
:= sched
.pidle
.ptr(); p
!= nil; p
= p
.link
.ptr() {
1395 if atomic
.Cas(&p
.runSafePointFn
, 1, 0) {
1397 sched
.safePointWait
--
1401 wait
:= sched
.safePointWait
> 0
1404 // Run fn for the current P.
1407 // Force Ps currently in _Psyscall into _Pidle and hand them
1408 // off to induce safe point function execution.
1409 for _
, p
:= range allp
{
1411 if s
== _Psyscall
&& p
.runSafePointFn
== 1 && atomic
.Cas(&p
.status
, s
, _Pidle
) {
1421 // Wait for remaining Ps to run fn.
1424 // Wait for 100us, then try to re-preempt in
1425 // case of any races.
1427 // Requires system stack.
1428 if notetsleep(&sched
.safePointNote
, 100*1000) {
1429 noteclear(&sched
.safePointNote
)
1435 if sched
.safePointWait
!= 0 {
1436 throw("forEachP: not done")
1438 for _
, p
:= range allp
{
1439 if p
.runSafePointFn
!= 0 {
1440 throw("forEachP: P did not run fn")
1445 sched
.safePointFn
= nil
1450 // runSafePointFn runs the safe point function, if any, for this P.
1451 // This should be called like
1453 // if getg().m.p.runSafePointFn != 0 {
1457 // runSafePointFn must be checked on any transition in to _Pidle or
1458 // _Psyscall to avoid a race where forEachP sees that the P is running
1459 // just before the P goes into _Pidle/_Psyscall and neither forEachP
1460 // nor the P run the safe-point function.
1461 func runSafePointFn() {
1462 p
:= getg().m
.p
.ptr()
1463 // Resolve the race between forEachP running the safe-point
1464 // function on this P's behalf and this P running the
1465 // safe-point function directly.
1466 if !atomic
.Cas(&p
.runSafePointFn
, 1, 0) {
1469 sched
.safePointFn(p
)
1471 sched
.safePointWait
--
1472 if sched
.safePointWait
== 0 {
1473 notewakeup(&sched
.safePointNote
)
1478 // Allocate a new m unassociated with any thread.
1479 // Can use p for allocation context if needed.
1480 // fn is recorded as the new m's m.mstartfn.
1482 // This function is allowed to have write barriers even if the caller
1483 // isn't because it borrows _p_.
1485 //go:yeswritebarrierrec
1486 func allocm(_p_
*p
, fn
func(), allocatestack
bool) (mp
*m
, g0Stack unsafe
.Pointer
, g0StackSize
uintptr) {
1488 _g_
.m
.locks
++ // disable GC because it can be called from sysmon
1490 acquirep(_p_
) // temporarily borrow p for mallocs in this function
1493 // Release the free M list. We need to do this somewhere and
1494 // this may free up a stack we can use.
1495 if sched
.freem
!= nil {
1498 for freem
:= sched
.freem
; freem
!= nil; {
1499 if freem
.freeWait
!= 0 {
1500 next
:= freem
.freelink
1501 freem
.freelink
= newList
1507 freem
= freem
.freelink
1509 sched
.freem
= newList
1517 mp
.g0
= malg(allocatestack
, false, &g0Stack
, &g0StackSize
)
1520 if _p_
== _g_
.m
.p
.ptr() {
1525 return mp
, g0Stack
, g0StackSize
1528 // needm is called when a cgo callback happens on a
1529 // thread without an m (a thread not created by Go).
1530 // In this case, needm is expected to find an m to use
1531 // and return with m, g initialized correctly.
1532 // Since m and g are not set now (likely nil, but see below)
1533 // needm is limited in what routines it can call. In particular
1534 // it can only call nosplit functions (textflag 7) and cannot
1535 // do any scheduling that requires an m.
1537 // In order to avoid needing heavy lifting here, we adopt
1538 // the following strategy: there is a stack of available m's
1539 // that can be stolen. Using compare-and-swap
1540 // to pop from the stack has ABA races, so we simulate
1541 // a lock by doing an exchange (via casp) to steal the stack
1542 // head and replace the top pointer with MLOCKED (1).
1543 // This serves as a simple spin lock that we can use even
1544 // without an m. The thread that locks the stack in this way
1545 // unlocks the stack by storing a valid stack head pointer.
1547 // In order to make sure that there is always an m structure
1548 // available to be stolen, we maintain the invariant that there
1549 // is always one more than needed. At the beginning of the
1550 // program (if cgo is in use) the list is seeded with a single m.
1551 // If needm finds that it has taken the last m off the list, its job
1552 // is - once it has installed its own m so that it can do things like
1553 // allocate memory - to create a spare m and put it on the list.
1555 // Each of these extra m's also has a g0 and a curg that are
1556 // pressed into service as the scheduling stack and current
1557 // goroutine for the duration of the cgo callback.
1559 // When the callback is done with the m, it calls dropm to
1560 // put the m back on the list.
1562 func needm(x
byte) {
1563 if (iscgo || GOOS
== "windows") && !cgoHasExtraM
{
1564 // Can happen if C/C++ code calls Go from a global ctor.
1565 // Can also happen on Windows if a global ctor uses a
1566 // callback created by syscall.NewCallback. See issue #6751
1569 // Can not throw, because scheduler is not initialized yet.
1570 write(2, unsafe
.Pointer(&earlycgocallback
[0]), int32(len(earlycgocallback
)))
1574 // Lock extra list, take head, unlock popped list.
1575 // nilokay=false is safe here because of the invariant above,
1576 // that the extra list always contains or will soon contain
1578 mp
:= lockextra(false)
1580 // Set needextram when we've just emptied the list,
1581 // so that the eventual call into cgocallbackg will
1582 // allocate a new m for the extra list. We delay the
1583 // allocation until then so that it can be done
1584 // after exitsyscall makes sure it is okay to be
1585 // running at all (that is, there's no garbage collection
1586 // running right now).
1587 mp
.needextram
= mp
.schedlink
== 0
1589 unlockextra(mp
.schedlink
.ptr())
1591 // Save and block signals before installing g.
1592 // Once g is installed, any incoming signals will try to execute,
1593 // but we won't have the sigaltstack settings and other data
1594 // set up appropriately until the end of minit, which will
1595 // unblock the signals. This is the same dance as when
1596 // starting a new m to run Go code via newosproc.
1600 // Install g (= m->curg).
1603 // Initialize this thread to use the m.
1609 // mp.curg is now a real goroutine.
1610 casgstatus(mp
.curg
, _Gdead
, _Gsyscall
)
1611 atomic
.Xadd(&sched
.ngsys
, -1)
1614 var earlycgocallback
= []byte("fatal error: cgo callback before cgo call\n")
1616 // newextram allocates m's and puts them on the extra list.
1617 // It is called with a working local m, so that it can do things
1618 // like call schedlock and allocate.
1620 c
:= atomic
.Xchg(&extraMWaiters
, 0)
1622 for i
:= uint32(0); i
< c
; i
++ {
1626 // Make sure there is at least one extra M.
1627 mp
:= lockextra(true)
1635 // oneNewExtraM allocates an m and puts it on the extra list.
1636 func oneNewExtraM() {
1637 // Create extra goroutine locked to extra m.
1638 // The goroutine is the context in which the cgo callback will run.
1639 // The sched.pc will never be returned to, but setting it to
1640 // goexit makes clear to the traceback routines where
1641 // the goroutine stack ends.
1642 mp
, g0SP
, g0SPSize
:= allocm(nil, nil, true)
1643 gp
:= malg(true, false, nil, nil)
1644 gp
.gcscanvalid
= true
1645 gp
.gcscandone
= true
1646 // malg returns status as _Gidle. Change to _Gdead before
1647 // adding to allg where GC can see it. We use _Gdead to hide
1648 // this from tracebacks and stack scans since it isn't a
1649 // "real" goroutine until needm grabs it.
1650 casgstatus(gp
, _Gidle
, _Gdead
)
1656 gp
.goid
= int64(atomic
.Xadd64(&sched
.goidgen
, 1))
1657 // put on allg for garbage collector
1660 // The context for gp will be set up in needm.
1661 // Here we need to set the context for g0.
1662 makeGContext(mp
.g0
, g0SP
, g0SPSize
)
1664 // gp is now on the allg list, but we don't want it to be
1665 // counted by gcount. It would be more "proper" to increment
1666 // sched.ngfree, but that requires locking. Incrementing ngsys
1667 // has the same effect.
1668 atomic
.Xadd(&sched
.ngsys
, +1)
1670 // Add m to the extra list.
1671 mnext
:= lockextra(true)
1672 mp
.schedlink
.set(mnext
)
1677 // dropm is called when a cgo callback has called needm but is now
1678 // done with the callback and returning back into the non-Go thread.
1679 // It puts the current m back onto the extra list.
1681 // The main expense here is the call to signalstack to release the
1682 // m's signal stack, and then the call to needm on the next callback
1683 // from this thread. It is tempting to try to save the m for next time,
1684 // which would eliminate both these costs, but there might not be
1685 // a next time: the current thread (which Go does not control) might exit.
1686 // If we saved the m for that thread, there would be an m leak each time
1687 // such a thread exited. Instead, we acquire and release an m on each
1688 // call. These should typically not be scheduling operations, just a few
1689 // atomics, so the cost should be small.
1691 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
1692 // variable using pthread_key_create. Unlike the pthread keys we already use
1693 // on OS X, this dummy key would never be read by Go code. It would exist
1694 // only so that we could register at thread-exit-time destructor.
1695 // That destructor would put the m back onto the extra list.
1696 // This is purely a performance optimization. The current version,
1697 // in which dropm happens on each cgo call, is still correct too.
1698 // We may have to keep the current version on systems with cgo
1699 // but without pthreads, like Windows.
1701 // CgocallBackDone calls this after releasing p, so no write barriers.
1702 //go:nowritebarrierrec
1704 // Clear m and g, and return m to the extra list.
1705 // After the call to setg we can only call nosplit functions
1706 // with no pointer manipulation.
1709 // Return mp.curg to dead state.
1710 casgstatus(mp
.curg
, _Gsyscall
, _Gdead
)
1711 atomic
.Xadd(&sched
.ngsys
, +1)
1713 // Block signals before unminit.
1714 // Unminit unregisters the signal handling stack (but needs g on some systems).
1715 // Setg(nil) clears g, which is the signal handler's cue not to run Go handlers.
1716 // It's important not to try to handle a signal between those two steps.
1717 sigmask
:= mp
.sigmask
1721 // gccgo sets the stack to Gdead here, because the splitstack
1722 // context is not initialized.
1723 atomic
.Store(&mp
.curg
.atomicstatus
, _Gdead
)
1725 mp
.curg
.gcnextsp
= 0
1727 mnext
:= lockextra(true)
1729 mp
.schedlink
.set(mnext
)
1733 // Commit the release of mp.
1736 msigrestore(sigmask
)
1739 // A helper function for EnsureDropM.
1740 func getm() uintptr {
1741 return uintptr(unsafe
.Pointer(getg().m
))
1745 var extraMCount
uint32 // Protected by lockextra
1746 var extraMWaiters
uint32
1748 // lockextra locks the extra list and returns the list head.
1749 // The caller must unlock the list by storing a new list head
1750 // to extram. If nilokay is true, then lockextra will
1751 // return a nil list head if that's what it finds. If nilokay is false,
1752 // lockextra will keep waiting until the list head is no longer nil.
1754 //go:nowritebarrierrec
1755 func lockextra(nilokay
bool) *m
{
1760 old
:= atomic
.Loaduintptr(&extram
)
1766 if old
== 0 && !nilokay
{
1768 // Add 1 to the number of threads
1769 // waiting for an M.
1770 // This is cleared by newextram.
1771 atomic
.Xadd(&extraMWaiters
, 1)
1777 if atomic
.Casuintptr(&extram
, old
, locked
) {
1778 return (*m
)(unsafe
.Pointer(old
))
1787 //go:nowritebarrierrec
1788 func unlockextra(mp
*m
) {
1789 atomic
.Storeuintptr(&extram
, uintptr(unsafe
.Pointer(mp
)))
1792 // execLock serializes exec and clone to avoid bugs or unspecified behaviour
1793 // around exec'ing while creating/destroying threads. See issue #19546.
1794 var execLock rwmutex
1796 // newmHandoff contains a list of m structures that need new OS threads.
1797 // This is used by newm in situations where newm itself can't safely
1798 // start an OS thread.
1799 var newmHandoff
struct {
1802 // newm points to a list of M structures that need new OS
1803 // threads. The list is linked through m.schedlink.
1806 // waiting indicates that wake needs to be notified when an m
1807 // is put on the list.
1811 // haveTemplateThread indicates that the templateThread has
1812 // been started. This is not protected by lock. Use cas to set
1814 haveTemplateThread
uint32
1817 // Create a new m. It will start off with a call to fn, or else the scheduler.
1818 // fn needs to be static and not a heap allocated closure.
1819 // May run with m.p==nil, so write barriers are not allowed.
1820 //go:nowritebarrierrec
1821 func newm(fn
func(), _p_
*p
) {
1822 mp
, _
, _
:= allocm(_p_
, fn
, false)
1824 mp
.sigmask
= initSigmask
1825 if gp
:= getg(); gp
!= nil && gp
.m
!= nil && (gp
.m
.lockedExt
!= 0 || gp
.m
.incgo
) && GOOS
!= "plan9" {
1826 // We're on a locked M or a thread that may have been
1827 // started by C. The kernel state of this thread may
1828 // be strange (the user may have locked it for that
1829 // purpose). We don't want to clone that into another
1830 // thread. Instead, ask a known-good thread to create
1831 // the thread for us.
1833 // This is disabled on Plan 9. See golang.org/issue/22227.
1835 // TODO: This may be unnecessary on Windows, which
1836 // doesn't model thread creation off fork.
1837 lock(&newmHandoff
.lock
)
1838 if newmHandoff
.haveTemplateThread
== 0 {
1839 throw("on a locked thread with no template thread")
1841 mp
.schedlink
= newmHandoff
.newm
1842 newmHandoff
.newm
.set(mp
)
1843 if newmHandoff
.waiting
{
1844 newmHandoff
.waiting
= false
1845 notewakeup(&newmHandoff
.wake
)
1847 unlock(&newmHandoff
.lock
)
1854 execLock
.rlock() // Prevent process clone.
1859 // startTemplateThread starts the template thread if it is not already
1862 // The calling thread must itself be in a known-good state.
1863 func startTemplateThread() {
1864 if GOARCH
== "wasm" { // no threads on wasm yet
1867 if !atomic
.Cas(&newmHandoff
.haveTemplateThread
, 0, 1) {
1870 newm(templateThread
, nil)
1873 // templateThread is a thread in a known-good state that exists solely
1874 // to start new threads in known-good states when the calling thread
1875 // may not be a a good state.
1877 // Many programs never need this, so templateThread is started lazily
1878 // when we first enter a state that might lead to running on a thread
1879 // in an unknown state.
1881 // templateThread runs on an M without a P, so it must not have write
1884 //go:nowritebarrierrec
1885 func templateThread() {
1892 lock(&newmHandoff
.lock
)
1893 for newmHandoff
.newm
!= 0 {
1894 newm
:= newmHandoff
.newm
.ptr()
1895 newmHandoff
.newm
= 0
1896 unlock(&newmHandoff
.lock
)
1898 next
:= newm
.schedlink
.ptr()
1903 lock(&newmHandoff
.lock
)
1905 newmHandoff
.waiting
= true
1906 noteclear(&newmHandoff
.wake
)
1907 unlock(&newmHandoff
.lock
)
1908 notesleep(&newmHandoff
.wake
)
1912 // Stops execution of the current m until new work is available.
1913 // Returns with acquired P.
1917 if _g_
.m
.locks
!= 0 {
1918 throw("stopm holding locks")
1921 throw("stopm holding p")
1924 throw("stopm spinning")
1931 notesleep(&_g_
.m
.park
)
1932 noteclear(&_g_
.m
.park
)
1933 if _g_
.m
.helpgc
!= 0 {
1934 // helpgc() set _g_.m.p and _g_.m.mcache, so we have a P.
1936 // Undo the effects of helpgc().
1942 acquirep(_g_
.m
.nextp
.ptr())
1947 // startm's caller incremented nmspinning. Set the new M's spinning.
1948 getg().m
.spinning
= true
1951 // Schedules some M to run the p (creates an M if necessary).
1952 // If p==nil, tries to get an idle P, if no idle P's does nothing.
1953 // May run with m.p==nil, so write barriers are not allowed.
1954 // If spinning is set, the caller has incremented nmspinning and startm will
1955 // either decrement nmspinning or set m.spinning in the newly started M.
1956 //go:nowritebarrierrec
1957 func startm(_p_
*p
, spinning
bool) {
1964 // The caller incremented nmspinning, but there are no idle Ps,
1965 // so it's okay to just undo the increment and give up.
1966 if int32(atomic
.Xadd(&sched
.nmspinning
, -1)) < 0 {
1967 throw("startm: negative nmspinning")
1978 // The caller incremented nmspinning, so set m.spinning in the new M.
1985 throw("startm: m is spinning")
1988 throw("startm: m has p")
1990 if spinning
&& !runqempty(_p_
) {
1991 throw("startm: p has runnable gs")
1993 // The caller incremented nmspinning, so set m.spinning in the new M.
1994 mp
.spinning
= spinning
1996 notewakeup(&mp
.park
)
1999 // Hands off P from syscall or locked M.
2000 // Always runs without a P, so write barriers are not allowed.
2001 //go:nowritebarrierrec
2002 func handoffp(_p_
*p
) {
2003 // handoffp must start an M in any situation where
2004 // findrunnable would return a G to run on _p_.
2006 // if it has local work, start it straight away
2007 if !runqempty(_p_
) || sched
.runqsize
!= 0 {
2011 // if it has GC work, start it straight away
2012 if gcBlackenEnabled
!= 0 && gcMarkWorkAvailable(_p_
) {
2016 // no local work, check that there are no spinning/idle M's,
2017 // otherwise our help is not required
2018 if atomic
.Load(&sched
.nmspinning
)+atomic
.Load(&sched
.npidle
) == 0 && atomic
.Cas(&sched
.nmspinning
, 0, 1) { // TODO: fast atomic
2023 if sched
.gcwaiting
!= 0 {
2024 _p_
.status
= _Pgcstop
2026 if sched
.stopwait
== 0 {
2027 notewakeup(&sched
.stopnote
)
2032 if _p_
.runSafePointFn
!= 0 && atomic
.Cas(&_p_
.runSafePointFn
, 1, 0) {
2033 sched
.safePointFn(_p_
)
2034 sched
.safePointWait
--
2035 if sched
.safePointWait
== 0 {
2036 notewakeup(&sched
.safePointNote
)
2039 if sched
.runqsize
!= 0 {
2044 // If this is the last running P and nobody is polling network,
2045 // need to wakeup another M to poll network.
2046 if sched
.npidle
== uint32(gomaxprocs
-1) && atomic
.Load64(&sched
.lastpoll
) != 0 {
2055 // Tries to add one more P to execute G's.
2056 // Called when a G is made runnable (newproc, ready).
2058 // be conservative about spinning threads
2059 if !atomic
.Cas(&sched
.nmspinning
, 0, 1) {
2065 // Stops execution of the current m that is locked to a g until the g is runnable again.
2066 // Returns with acquired P.
2067 func stoplockedm() {
2070 if _g_
.m
.lockedg
== 0 || _g_
.m
.lockedg
.ptr().lockedm
.ptr() != _g_
.m
{
2071 throw("stoplockedm: inconsistent locking")
2074 // Schedule another M to run this p.
2079 // Wait until another thread schedules lockedg again.
2080 notesleep(&_g_
.m
.park
)
2081 noteclear(&_g_
.m
.park
)
2082 status
:= readgstatus(_g_
.m
.lockedg
.ptr())
2083 if status
&^_Gscan
!= _Grunnable
{
2084 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
2086 throw("stoplockedm: not runnable")
2088 acquirep(_g_
.m
.nextp
.ptr())
2092 // Schedules the locked m to run the locked gp.
2093 // May run during STW, so write barriers are not allowed.
2094 //go:nowritebarrierrec
2095 func startlockedm(gp
*g
) {
2098 mp
:= gp
.lockedm
.ptr()
2100 throw("startlockedm: locked to me")
2103 throw("startlockedm: m has p")
2105 // directly handoff current P to the locked m
2109 notewakeup(&mp
.park
)
2113 // Stops the current m for stopTheWorld.
2114 // Returns when the world is restarted.
2118 if sched
.gcwaiting
== 0 {
2119 throw("gcstopm: not waiting for gc")
2122 _g_
.m
.spinning
= false
2123 // OK to just drop nmspinning here,
2124 // startTheWorld will unpark threads as necessary.
2125 if int32(atomic
.Xadd(&sched
.nmspinning
, -1)) < 0 {
2126 throw("gcstopm: negative nmspinning")
2131 _p_
.status
= _Pgcstop
2133 if sched
.stopwait
== 0 {
2134 notewakeup(&sched
.stopnote
)
2140 // Schedules gp to run on the current M.
2141 // If inheritTime is true, gp inherits the remaining time in the
2142 // current time slice. Otherwise, it starts a new time slice.
2145 // Write barriers are allowed because this is called immediately after
2146 // acquiring a P in several places.
2148 //go:yeswritebarrierrec
2149 func execute(gp
*g
, inheritTime
bool) {
2152 casgstatus(gp
, _Grunnable
, _Grunning
)
2156 _g_
.m
.p
.ptr().schedtick
++
2161 // Check whether the profiler needs to be turned on or off.
2162 hz
:= sched
.profilehz
2163 if _g_
.m
.profilehz
!= hz
{
2164 setThreadCPUProfiler(hz
)
2168 // GoSysExit has to happen when we have a P, but before GoStart.
2169 // So we emit it here.
2170 if gp
.syscallsp
!= 0 && gp
.sysblocktraced
{
2171 traceGoSysExit(gp
.sysexitticks
)
2179 // Finds a runnable goroutine to execute.
2180 // Tries to steal from other P's, get g from global queue, poll network.
2181 func findrunnable() (gp
*g
, inheritTime
bool) {
2184 // The conditions here and in handoffp must agree: if
2185 // findrunnable would return a G to run, handoffp must start
2189 _p_
:= _g_
.m
.p
.ptr()
2190 if sched
.gcwaiting
!= 0 {
2194 if _p_
.runSafePointFn
!= 0 {
2197 if fingwait
&& fingwake
{
2198 if gp
:= wakefing(); gp
!= nil {
2202 if *cgo_yield
!= nil {
2203 asmcgocall(*cgo_yield
, nil)
2207 if gp
, inheritTime
:= runqget(_p_
); gp
!= nil {
2208 return gp
, inheritTime
2212 if sched
.runqsize
!= 0 {
2214 gp
:= globrunqget(_p_
, 0)
2222 // This netpoll is only an optimization before we resort to stealing.
2223 // We can safely skip it if there are no waiters or a thread is blocked
2224 // in netpoll already. If there is any kind of logical race with that
2225 // blocked thread (e.g. it has already returned from netpoll, but does
2226 // not set lastpoll yet), this thread will do blocking netpoll below
2228 if netpollinited() && atomic
.Load(&netpollWaiters
) > 0 && atomic
.Load64(&sched
.lastpoll
) != 0 {
2229 if gp
:= netpoll(false); gp
!= nil { // non-blocking
2230 // netpoll returns list of goroutines linked by schedlink.
2231 injectglist(gp
.schedlink
.ptr())
2232 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2234 traceGoUnpark(gp
, 0)
2240 // Steal work from other P's.
2241 procs
:= uint32(gomaxprocs
)
2242 if atomic
.Load(&sched
.npidle
) == procs
-1 {
2243 // Either GOMAXPROCS=1 or everybody, except for us, is idle already.
2244 // New work can appear from returning syscall/cgocall, network or timers.
2245 // Neither of that submits to local run queues, so no point in stealing.
2248 // If number of spinning M's >= number of busy P's, block.
2249 // This is necessary to prevent excessive CPU consumption
2250 // when GOMAXPROCS>>1 but the program parallelism is low.
2251 if !_g_
.m
.spinning
&& 2*atomic
.Load(&sched
.nmspinning
) >= procs
-atomic
.Load(&sched
.npidle
) {
2254 if !_g_
.m
.spinning
{
2255 _g_
.m
.spinning
= true
2256 atomic
.Xadd(&sched
.nmspinning
, 1)
2258 for i
:= 0; i
< 4; i
++ {
2259 for enum
:= stealOrder
.start(fastrand()); !enum
.done(); enum
.next() {
2260 if sched
.gcwaiting
!= 0 {
2263 stealRunNextG
:= i
> 2 // first look for ready queues with more than 1 g
2264 if gp
:= runqsteal(_p_
, allp
[enum
.position()], stealRunNextG
); gp
!= nil {
2272 // We have nothing to do. If we're in the GC mark phase, can
2273 // safely scan and blacken objects, and have work to do, run
2274 // idle-time marking rather than give up the P.
2275 if gcBlackenEnabled
!= 0 && _p_
.gcBgMarkWorker
!= 0 && gcMarkWorkAvailable(_p_
) {
2276 _p_
.gcMarkWorkerMode
= gcMarkWorkerIdleMode
2277 gp
:= _p_
.gcBgMarkWorker
.ptr()
2278 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2280 traceGoUnpark(gp
, 0)
2286 // Check if a goroutine is waiting for a callback from the WebAssembly host.
2287 // If yes, pause the execution until a callback was triggered.
2288 if pauseSchedulerUntilCallback() {
2289 // A callback was triggered and caused at least one goroutine to wake up.
2293 // Before we drop our P, make a snapshot of the allp slice,
2294 // which can change underfoot once we no longer block
2295 // safe-points. We don't need to snapshot the contents because
2296 // everything up to cap(allp) is immutable.
2297 allpSnapshot
:= allp
2299 // return P and block
2301 if sched
.gcwaiting
!= 0 || _p_
.runSafePointFn
!= 0 {
2305 if sched
.runqsize
!= 0 {
2306 gp
:= globrunqget(_p_
, 0)
2310 if releasep() != _p_
{
2311 throw("findrunnable: wrong p")
2316 // Delicate dance: thread transitions from spinning to non-spinning state,
2317 // potentially concurrently with submission of new goroutines. We must
2318 // drop nmspinning first and then check all per-P queues again (with
2319 // #StoreLoad memory barrier in between). If we do it the other way around,
2320 // another thread can submit a goroutine after we've checked all run queues
2321 // but before we drop nmspinning; as the result nobody will unpark a thread
2322 // to run the goroutine.
2323 // If we discover new work below, we need to restore m.spinning as a signal
2324 // for resetspinning to unpark a new worker thread (because there can be more
2325 // than one starving goroutine). However, if after discovering new work
2326 // we also observe no idle Ps, it is OK to just park the current thread:
2327 // the system is fully loaded so no spinning threads are required.
2328 // Also see "Worker thread parking/unparking" comment at the top of the file.
2329 wasSpinning
:= _g_
.m
.spinning
2331 _g_
.m
.spinning
= false
2332 if int32(atomic
.Xadd(&sched
.nmspinning
, -1)) < 0 {
2333 throw("findrunnable: negative nmspinning")
2337 // check all runqueues once again
2338 for _
, _p_
:= range allpSnapshot
{
2339 if !runqempty(_p_
) {
2346 _g_
.m
.spinning
= true
2347 atomic
.Xadd(&sched
.nmspinning
, 1)
2355 // Check for idle-priority GC work again.
2356 if gcBlackenEnabled
!= 0 && gcMarkWorkAvailable(nil) {
2359 if _p_
!= nil && _p_
.gcBgMarkWorker
== 0 {
2367 _g_
.m
.spinning
= true
2368 atomic
.Xadd(&sched
.nmspinning
, 1)
2370 // Go back to idle GC check.
2376 if netpollinited() && atomic
.Load(&netpollWaiters
) > 0 && atomic
.Xchg64(&sched
.lastpoll
, 0) != 0 {
2378 throw("findrunnable: netpoll with p")
2381 throw("findrunnable: netpoll with spinning")
2383 gp
:= netpoll(true) // block until new work is available
2384 atomic
.Store64(&sched
.lastpoll
, uint64(nanotime()))
2391 injectglist(gp
.schedlink
.ptr())
2392 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2394 traceGoUnpark(gp
, 0)
2405 // pollWork returns true if there is non-background work this P could
2406 // be doing. This is a fairly lightweight check to be used for
2407 // background work loops, like idle GC. It checks a subset of the
2408 // conditions checked by the actual scheduler.
2409 func pollWork() bool {
2410 if sched
.runqsize
!= 0 {
2413 p
:= getg().m
.p
.ptr()
2417 if netpollinited() && atomic
.Load(&netpollWaiters
) > 0 && sched
.lastpoll
!= 0 {
2418 if gp
:= netpoll(false); gp
!= nil {
2426 func resetspinning() {
2428 if !_g_
.m
.spinning
{
2429 throw("resetspinning: not a spinning m")
2431 _g_
.m
.spinning
= false
2432 nmspinning
:= atomic
.Xadd(&sched
.nmspinning
, -1)
2433 if int32(nmspinning
) < 0 {
2434 throw("findrunnable: negative nmspinning")
2436 // M wakeup policy is deliberately somewhat conservative, so check if we
2437 // need to wakeup another P here. See "Worker thread parking/unparking"
2438 // comment at the top of the file for details.
2439 if nmspinning
== 0 && atomic
.Load(&sched
.npidle
) > 0 {
2444 // Injects the list of runnable G's into the scheduler.
2445 // Can run concurrently with GC.
2446 func injectglist(glist
*g
) {
2451 for gp
:= glist
; gp
!= nil; gp
= gp
.schedlink
.ptr() {
2452 traceGoUnpark(gp
, 0)
2457 for n
= 0; glist
!= nil; n
++ {
2459 glist
= gp
.schedlink
.ptr()
2460 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2464 for ; n
!= 0 && sched
.npidle
!= 0; n
-- {
2469 // One round of scheduler: find a runnable goroutine and execute it.
2474 if _g_
.m
.locks
!= 0 {
2475 throw("schedule: holding locks")
2478 if _g_
.m
.lockedg
!= 0 {
2480 execute(_g_
.m
.lockedg
.ptr(), false) // Never returns.
2483 // We should not schedule away from a g that is executing a cgo call,
2484 // since the cgo call is using the m's g0 stack.
2486 throw("schedule: in cgo")
2490 if sched
.gcwaiting
!= 0 {
2494 if _g_
.m
.p
.ptr().runSafePointFn
!= 0 {
2499 var inheritTime
bool
2500 if trace
.enabled || trace
.shutdown
{
2503 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2504 traceGoUnpark(gp
, 0)
2507 if gp
== nil && gcBlackenEnabled
!= 0 {
2508 gp
= gcController
.findRunnableGCWorker(_g_
.m
.p
.ptr())
2511 // Check the global runnable queue once in a while to ensure fairness.
2512 // Otherwise two goroutines can completely occupy the local runqueue
2513 // by constantly respawning each other.
2514 if _g_
.m
.p
.ptr().schedtick%61
== 0 && sched
.runqsize
> 0 {
2516 gp
= globrunqget(_g_
.m
.p
.ptr(), 1)
2521 gp
, inheritTime
= runqget(_g_
.m
.p
.ptr())
2522 if gp
!= nil && _g_
.m
.spinning
{
2523 throw("schedule: spinning with local work")
2526 // Because gccgo does not implement preemption as a stack check,
2527 // we need to check for preemption here for fairness.
2528 // Otherwise goroutines on the local queue may starve
2529 // goroutines on the global queue.
2530 // Since we preempt by storing the goroutine on the global
2531 // queue, this is the only place we need to check preempt.
2532 // This does not call checkPreempt because gp is not running.
2533 if gp
!= nil && gp
.preempt
{
2542 gp
, inheritTime
= findrunnable() // blocks until work is available
2545 // This thread is going to run a goroutine and is not spinning anymore,
2546 // so if it was marked as spinning we need to reset it now and potentially
2547 // start a new spinning M.
2552 if gp
.lockedm
!= 0 {
2553 // Hands off own p to the locked m,
2554 // then blocks waiting for a new p.
2559 execute(gp
, inheritTime
)
2562 // dropg removes the association between m and the current goroutine m->curg (gp for short).
2563 // Typically a caller sets gp's status away from Grunning and then
2564 // immediately calls dropg to finish the job. The caller is also responsible
2565 // for arranging that gp will be restarted using ready at an
2566 // appropriate time. After calling dropg and arranging for gp to be
2567 // readied later, the caller can do other work but eventually should
2568 // call schedule to restart the scheduling of goroutines on this m.
2572 setMNoWB(&_g_
.m
.curg
.m
, nil)
2573 setGNoWB(&_g_
.m
.curg
, nil)
2576 func parkunlock_c(gp
*g
, lock unsafe
.Pointer
) bool {
2577 unlock((*mutex
)(lock
))
2581 // park continuation on g0.
2582 func park_m(gp
*g
) {
2586 traceGoPark(_g_
.m
.waittraceev
, _g_
.m
.waittraceskip
)
2589 casgstatus(gp
, _Grunning
, _Gwaiting
)
2592 if _g_
.m
.waitunlockf
!= nil {
2593 fn
:= *(*func(*g
, unsafe
.Pointer
) bool)(unsafe
.Pointer(&_g_
.m
.waitunlockf
))
2594 ok
:= fn(gp
, _g_
.m
.waitlock
)
2595 _g_
.m
.waitunlockf
= nil
2596 _g_
.m
.waitlock
= nil
2599 traceGoUnpark(gp
, 2)
2601 casgstatus(gp
, _Gwaiting
, _Grunnable
)
2602 execute(gp
, true) // Schedule it back, never returns.
2608 func goschedImpl(gp
*g
) {
2609 status
:= readgstatus(gp
)
2610 if status
&^_Gscan
!= _Grunning
{
2612 throw("bad g status")
2614 casgstatus(gp
, _Grunning
, _Grunnable
)
2623 // Gosched continuation on g0.
2624 func gosched_m(gp
*g
) {
2631 // goschedguarded is a forbidden-states-avoided version of gosched_m
2632 func goschedguarded_m(gp
*g
) {
2634 if gp
.m
.locks
!= 0 || gp
.m
.mallocing
!= 0 || gp
.m
.preemptoff
!= "" || gp
.m
.p
.ptr().status
!= _Prunning
{
2635 gogo(gp
) // never return
2644 func gopreempt_m(gp
*g
) {
2651 // Finishes execution of the current goroutine.
2659 // goexit continuation on g0.
2660 func goexit0(gp
*g
) {
2663 casgstatus(gp
, _Grunning
, _Gdead
)
2664 if isSystemGoroutine(gp
) {
2665 atomic
.Xadd(&sched
.ngsys
, -1)
2666 gp
.isSystemGoroutine
= false
2669 locked
:= gp
.lockedm
!= 0
2673 gp
.paniconfault
= false
2674 gp
._defer
= nil // should be true already but just in case.
2675 gp
._panic
= nil // non-nil for Goexit during panic. points at stack-allocated data.
2682 if gcBlackenEnabled
!= 0 && gp
.gcAssistBytes
> 0 {
2683 // Flush assist credit to the global pool. This gives
2684 // better information to pacing if the application is
2685 // rapidly creating an exiting goroutines.
2686 scanCredit
:= int64(gcController
.assistWorkPerByte
* float64(gp
.gcAssistBytes
))
2687 atomic
.Xaddint64(&gcController
.bgScanCredit
, scanCredit
)
2688 gp
.gcAssistBytes
= 0
2691 // Note that gp's stack scan is now "valid" because it has no
2693 gp
.gcscanvalid
= true
2696 if GOARCH
== "wasm" { // no threads yet on wasm
2697 gfput(_g_
.m
.p
.ptr(), gp
)
2698 schedule() // never returns
2701 if _g_
.m
.lockedInt
!= 0 {
2702 print("invalid m->lockedInt = ", _g_
.m
.lockedInt
, "\n")
2703 throw("internal lockOSThread error")
2706 gfput(_g_
.m
.p
.ptr(), gp
)
2708 // The goroutine may have locked this thread because
2709 // it put it in an unusual kernel state. Kill it
2710 // rather than returning it to the thread pool.
2712 // Return to mstart, which will release the P and exit
2714 if GOOS
!= "plan9" { // See golang.org/issue/22227.
2715 _g_
.m
.exiting
= true
2722 // The goroutine g is about to enter a system call.
2723 // Record that it's not using the cpu anymore.
2724 // This is called only from the go syscall library and cgocall,
2725 // not from the low-level system calls used by the runtime.
2727 // The entersyscall function is written in C, so that it can save the
2728 // current register context so that the GC will see them.
2729 // It calls reentersyscall.
2732 // At the start of a syscall we emit traceGoSysCall to capture the stack trace.
2733 // If the syscall does not block, that is it, we do not emit any other events.
2734 // If the syscall blocks (that is, P is retaken), retaker emits traceGoSysBlock;
2735 // when syscall returns we emit traceGoSysExit and when the goroutine starts running
2736 // (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart.
2737 // To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock,
2738 // we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick),
2739 // whoever emits traceGoSysBlock increments p.syscalltick afterwards;
2740 // and we wait for the increment before emitting traceGoSysExit.
2741 // Note that the increment is done even if tracing is not enabled,
2742 // because tracing can be enabled in the middle of syscall. We don't want the wait to hang.
2746 func reentersyscall(pc
, sp
uintptr) {
2749 // Disable preemption because during this function g is in Gsyscall status,
2750 // but can have inconsistent g->sched, do not let GC observe it.
2755 casgstatus(_g_
, _Grunning
, _Gsyscall
)
2758 systemstack(traceGoSysCall
)
2761 if atomic
.Load(&sched
.sysmonwait
) != 0 {
2762 systemstack(entersyscall_sysmon
)
2765 if _g_
.m
.p
.ptr().runSafePointFn
!= 0 {
2766 // runSafePointFn may stack split if run on this stack
2767 systemstack(runSafePointFn
)
2770 _g_
.m
.syscalltick
= _g_
.m
.p
.ptr().syscalltick
2771 _g_
.sysblocktraced
= true
2774 atomic
.Store(&_g_
.m
.p
.ptr().status
, _Psyscall
)
2775 if sched
.gcwaiting
!= 0 {
2776 systemstack(entersyscall_gcwait
)
2782 func entersyscall_sysmon() {
2784 if atomic
.Load(&sched
.sysmonwait
) != 0 {
2785 atomic
.Store(&sched
.sysmonwait
, 0)
2786 notewakeup(&sched
.sysmonnote
)
2791 func entersyscall_gcwait() {
2793 _p_
:= _g_
.m
.p
.ptr()
2796 if sched
.stopwait
> 0 && atomic
.Cas(&_p_
.status
, _Psyscall
, _Pgcstop
) {
2798 traceGoSysBlock(_p_
)
2802 if sched
.stopwait
--; sched
.stopwait
== 0 {
2803 notewakeup(&sched
.stopnote
)
2809 func reentersyscallblock(pc
, sp
uintptr) {
2812 _g_
.m
.locks
++ // see comment in entersyscall
2813 _g_
.throwsplit
= true
2814 _g_
.m
.syscalltick
= _g_
.m
.p
.ptr().syscalltick
2815 _g_
.sysblocktraced
= true
2816 _g_
.m
.p
.ptr().syscalltick
++
2818 // Leave SP around for GC and traceback.
2821 casgstatus(_g_
, _Grunning
, _Gsyscall
)
2822 systemstack(entersyscallblock_handoff
)
2827 func entersyscallblock_handoff() {
2830 traceGoSysBlock(getg().m
.p
.ptr())
2832 handoffp(releasep())
2835 // The goroutine g exited its system call.
2836 // Arrange for it to run on a cpu again.
2837 // This is called only from the go syscall library, not
2838 // from the low-level system calls used by the runtime.
2840 // Write barriers are not allowed because our P may have been stolen.
2843 //go:nowritebarrierrec
2844 func exitsyscall() {
2847 _g_
.m
.locks
++ // see comment in entersyscall
2850 oldp
:= _g_
.m
.p
.ptr()
2851 if exitsyscallfast() {
2852 if _g_
.m
.mcache
== nil {
2853 throw("lost mcache")
2856 if oldp
!= _g_
.m
.p
.ptr() || _g_
.m
.syscalltick
!= _g_
.m
.p
.ptr().syscalltick
{
2857 systemstack(traceGoStart
)
2860 // There's a cpu for us, so we can run.
2861 _g_
.m
.p
.ptr().syscalltick
++
2862 // We need to cas the status and scan before resuming...
2863 casgstatus(_g_
, _Gsyscall
, _Grunning
)
2865 exitsyscallclear(_g_
)
2867 _g_
.throwsplit
= false
2869 // Check preemption, since unlike gc we don't check on
2878 _g_
.sysexitticks
= 0
2880 // Wait till traceGoSysBlock event is emitted.
2881 // This ensures consistency of the trace (the goroutine is started after it is blocked).
2882 for oldp
!= nil && oldp
.syscalltick
== _g_
.m
.syscalltick
{
2885 // We can't trace syscall exit right now because we don't have a P.
2886 // Tracing code can invoke write barriers that cannot run without a P.
2887 // So instead we remember the syscall exit time and emit the event
2888 // in execute when we have a P.
2889 _g_
.sysexitticks
= cputicks()
2894 // Call the scheduler.
2897 if _g_
.m
.mcache
== nil {
2898 throw("lost mcache")
2901 // Scheduler returned, so we're allowed to run now.
2902 // Delete the syscallsp information that we left for
2903 // the garbage collector during the system call.
2904 // Must wait until now because until gosched returns
2905 // we don't know for sure that the garbage collector
2907 exitsyscallclear(_g_
)
2909 _g_
.m
.p
.ptr().syscalltick
++
2910 _g_
.throwsplit
= false
2914 func exitsyscallfast() bool {
2917 // Freezetheworld sets stopwait but does not retake P's.
2918 if sched
.stopwait
== freezeStopWait
{
2924 // Try to re-acquire the last P.
2925 if _g_
.m
.p
!= 0 && _g_
.m
.p
.ptr().status
== _Psyscall
&& atomic
.Cas(&_g_
.m
.p
.ptr().status
, _Psyscall
, _Prunning
) {
2926 // There's a cpu for us, so we can run.
2927 exitsyscallfast_reacquired()
2931 // Try to get any other idle P.
2932 oldp
:= _g_
.m
.p
.ptr()
2935 if sched
.pidle
!= 0 {
2937 systemstack(func() {
2938 ok
= exitsyscallfast_pidle()
2939 if ok
&& trace
.enabled
{
2941 // Wait till traceGoSysBlock event is emitted.
2942 // This ensures consistency of the trace (the goroutine is started after it is blocked).
2943 for oldp
.syscalltick
== _g_
.m
.syscalltick
{
2957 // exitsyscallfast_reacquired is the exitsyscall path on which this G
2958 // has successfully reacquired the P it was running on before the
2961 // This function is allowed to have write barriers because exitsyscall
2962 // has acquired a P at this point.
2964 //go:yeswritebarrierrec
2966 func exitsyscallfast_reacquired() {
2968 _g_
.m
.mcache
= _g_
.m
.p
.ptr().mcache
2969 _g_
.m
.p
.ptr().m
.set(_g_
.m
)
2970 if _g_
.m
.syscalltick
!= _g_
.m
.p
.ptr().syscalltick
{
2972 // The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed).
2973 // traceGoSysBlock for this syscall was already emitted,
2974 // but here we effectively retake the p from the new syscall running on the same p.
2975 systemstack(func() {
2976 // Denote blocking of the new syscall.
2977 traceGoSysBlock(_g_
.m
.p
.ptr())
2978 // Denote completion of the current syscall.
2982 _g_
.m
.p
.ptr().syscalltick
++
2986 func exitsyscallfast_pidle() bool {
2989 if _p_
!= nil && atomic
.Load(&sched
.sysmonwait
) != 0 {
2990 atomic
.Store(&sched
.sysmonwait
, 0)
2991 notewakeup(&sched
.sysmonnote
)
3001 // exitsyscall slow path on g0.
3002 // Failed to acquire P, enqueue gp as runnable.
3004 //go:nowritebarrierrec
3005 func exitsyscall0(gp
*g
) {
3008 casgstatus(gp
, _Gsyscall
, _Grunnable
)
3014 } else if atomic
.Load(&sched
.sysmonwait
) != 0 {
3015 atomic
.Store(&sched
.sysmonwait
, 0)
3016 notewakeup(&sched
.sysmonnote
)
3021 execute(gp
, false) // Never returns.
3023 if _g_
.m
.lockedg
!= 0 {
3024 // Wait until another thread schedules gp and so m again.
3026 execute(gp
, false) // Never returns.
3029 schedule() // Never returns.
3032 // exitsyscallclear clears GC-related information that we only track
3033 // during a syscall.
3034 func exitsyscallclear(gp
*g
) {
3035 // Garbage collector isn't running (since we are), so okay to
3041 memclrNoHeapPointers(unsafe
.Pointer(&gp
.gcregs
), unsafe
.Sizeof(gp
.gcregs
))
3044 // Code generated by cgo, and some library code, calls syscall.Entersyscall
3045 // and syscall.Exitsyscall.
3047 //go:linkname syscall_entersyscall syscall.Entersyscall
3049 func syscall_entersyscall() {
3053 //go:linkname syscall_exitsyscall syscall.Exitsyscall
3055 func syscall_exitsyscall() {
3062 // Block signals during a fork, so that the child does not run
3063 // a signal handler before exec if a signal is sent to the process
3064 // group. See issue #18600.
3070 // Called from syscall package before fork.
3071 //go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork
3073 func syscall_runtime_BeforeFork() {
3074 systemstack(beforefork
)
3080 msigrestore(gp
.m
.sigmask
)
3085 // Called from syscall package after fork in parent.
3086 //go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork
3088 func syscall_runtime_AfterFork() {
3089 systemstack(afterfork
)
3092 // inForkedChild is true while manipulating signals in the child process.
3093 // This is used to avoid calling libc functions in case we are using vfork.
3094 var inForkedChild
bool
3096 // Called from syscall package after fork in child.
3097 // It resets non-sigignored signals to the default handler, and
3098 // restores the signal mask in preparation for the exec.
3100 // Because this might be called during a vfork, and therefore may be
3101 // temporarily sharing address space with the parent process, this must
3102 // not change any global variables or calling into C code that may do so.
3104 //go:linkname syscall_runtime_AfterForkInChild syscall.runtime_AfterForkInChild
3106 //go:nowritebarrierrec
3107 func syscall_runtime_AfterForkInChild() {
3108 // It's OK to change the global variable inForkedChild here
3109 // because we are going to change it back. There is no race here,
3110 // because if we are sharing address space with the parent process,
3111 // then the parent process can not be running concurrently.
3112 inForkedChild
= true
3114 clearSignalHandlers()
3116 // When we are the child we are the only thread running,
3117 // so we know that nothing else has changed gp.m.sigmask.
3118 msigrestore(getg().m
.sigmask
)
3120 inForkedChild
= false
3123 // Called from syscall package before Exec.
3124 //go:linkname syscall_runtime_BeforeExec syscall.runtime_BeforeExec
3125 func syscall_runtime_BeforeExec() {
3126 // Prevent thread creation during exec.
3130 // Called from syscall package after Exec.
3131 //go:linkname syscall_runtime_AfterExec syscall.runtime_AfterExec
3132 func syscall_runtime_AfterExec() {
3136 // Create a new g running fn passing arg as the single argument.
3137 // Put it on the queue of g's waiting to run.
3138 // The compiler turns a go statement into a call to this.
3139 //go:linkname newproc __go_go
3140 func newproc(fn
uintptr, arg unsafe
.Pointer
) *g
{
3144 _g_
.m
.throwing
= -1 // do not dump full stacks
3145 throw("go of nil func value")
3147 _g_
.m
.locks
++ // disable preemption because it can be holding p in a local var
3149 _p_
:= _g_
.m
.p
.ptr()
3156 newg
= malg(true, false, &sp
, &spsize
)
3157 casgstatus(newg
, _Gidle
, _Gdead
)
3158 allgadd(newg
) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
3160 resetNewG(newg
, &sp
, &spsize
)
3164 if readgstatus(newg
) != _Gdead
{
3165 throw("newproc1: new g is not Gdead")
3168 // Store the C function pointer into entryfn, take the address
3169 // of entryfn, convert it to a Go function value, and store
3172 var entry
func(unsafe
.Pointer
)
3173 *(*unsafe
.Pointer
)(unsafe
.Pointer(&entry
)) = unsafe
.Pointer(&newg
.entryfn
)
3177 newg
.gopc
= getcallerpc()
3179 if _g_
.m
.curg
!= nil {
3180 newg
.labels
= _g_
.m
.curg
.labels
3182 if isSystemGoroutine(newg
) {
3183 atomic
.Xadd(&sched
.ngsys
, +1)
3185 newg
.gcscanvalid
= false
3186 casgstatus(newg
, _Gdead
, _Grunnable
)
3188 if _p_
.goidcache
== _p_
.goidcacheend
{
3189 // Sched.goidgen is the last allocated id,
3190 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
3191 // At startup sched.goidgen=0, so main goroutine receives goid=1.
3192 _p_
.goidcache
= atomic
.Xadd64(&sched
.goidgen
, _GoidCacheBatch
)
3193 _p_
.goidcache
-= _GoidCacheBatch
- 1
3194 _p_
.goidcacheend
= _p_
.goidcache
+ _GoidCacheBatch
3196 newg
.goid
= int64(_p_
.goidcache
)
3199 traceGoCreate(newg
, newg
.startpc
)
3202 makeGContext(newg
, sp
, spsize
)
3204 runqput(_p_
, newg
, true)
3206 if atomic
.Load(&sched
.npidle
) != 0 && atomic
.Load(&sched
.nmspinning
) == 0 && mainStarted
{
3213 // expectedSystemGoroutines counts the number of goroutines expected
3214 // to mark themselves as system goroutines. After they mark themselves
3215 // by calling setSystemGoroutine, this is decremented. NumGoroutines
3216 // uses this to wait for all system goroutines to mark themselves
3217 // before it counts them.
3218 var expectedSystemGoroutines
uint32
3220 // expectSystemGoroutine is called when starting a goroutine that will
3221 // call setSystemGoroutine. It increments expectedSystemGoroutines.
3222 func expectSystemGoroutine() {
3223 atomic
.Xadd(&expectedSystemGoroutines
, +1)
3226 // waitForSystemGoroutines waits for all currently expected system
3227 // goroutines to register themselves.
3228 func waitForSystemGoroutines() {
3229 for atomic
.Load(&expectedSystemGoroutines
) > 0 {
3235 // setSystemGoroutine marks this goroutine as a "system goroutine".
3236 // In the gc toolchain this is done by comparing startpc to a list of
3237 // saved special PCs. In gccgo that approach does not work as startpc
3238 // is often a thunk that invokes the real function with arguments,
3239 // so the thunk address never matches the saved special PCs. Instead,
3240 // since there are only a limited number of "system goroutines",
3241 // we force each one to mark itself as special.
3242 func setSystemGoroutine() {
3243 getg().isSystemGoroutine
= true
3244 atomic
.Xadd(&sched
.ngsys
, +1)
3245 atomic
.Xadd(&expectedSystemGoroutines
, -1)
3248 // saveAncestors copies previous ancestors of the given caller g and
3249 // includes infor for the current caller into a new set of tracebacks for
3250 // a g being created.
3251 func saveAncestors(callergp
*g
) *[]ancestorInfo
{
3252 // Copy all prior info, except for the root goroutine (goid 0).
3253 if debug
.tracebackancestors
<= 0 || callergp
.goid
== 0 {
3256 var callerAncestors
[]ancestorInfo
3257 if callergp
.ancestors
!= nil {
3258 callerAncestors
= *callergp
.ancestors
3260 n
:= int32(len(callerAncestors
)) + 1
3261 if n
> debug
.tracebackancestors
{
3262 n
= debug
.tracebackancestors
3264 ancestors
:= make([]ancestorInfo
, n
)
3265 copy(ancestors
[1:], callerAncestors
)
3267 var pcs
[_TracebackMaxFrames
]uintptr
3268 // FIXME: This should get a traceback of callergp.
3269 // npcs := gcallers(callergp, 0, pcs[:])
3271 ipcs
:= make([]uintptr, npcs
)
3273 ancestors
[0] = ancestorInfo
{
3275 goid
: callergp
.goid
,
3276 gopc
: callergp
.gopc
,
3279 ancestorsp
:= new([]ancestorInfo
)
3280 *ancestorsp
= ancestors
3284 // Put on gfree list.
3285 // If local list is too long, transfer a batch to the global list.
3286 func gfput(_p_
*p
, gp
*g
) {
3287 if readgstatus(gp
) != _Gdead
{
3288 throw("gfput: bad status (not Gdead)")
3291 gp
.schedlink
.set(_p_
.gfree
)
3294 if _p_
.gfreecnt
>= 64 {
3296 for _p_
.gfreecnt
>= 32 {
3299 _p_
.gfree
= gp
.schedlink
.ptr()
3300 gp
.schedlink
.set(sched
.gfree
)
3304 unlock(&sched
.gflock
)
3308 // Get from gfree list.
3309 // If local list is empty, grab a batch from global list.
3310 func gfget(_p_
*p
) *g
{
3313 if gp
== nil && sched
.gfree
!= nil {
3315 for _p_
.gfreecnt
< 32 {
3316 if sched
.gfree
!= nil {
3318 sched
.gfree
= gp
.schedlink
.ptr()
3324 gp
.schedlink
.set(_p_
.gfree
)
3327 unlock(&sched
.gflock
)
3331 _p_
.gfree
= gp
.schedlink
.ptr()
3337 // Purge all cached G's from gfree list to the global list.
3338 func gfpurge(_p_
*p
) {
3340 for _p_
.gfreecnt
!= 0 {
3343 _p_
.gfree
= gp
.schedlink
.ptr()
3344 gp
.schedlink
.set(sched
.gfree
)
3348 unlock(&sched
.gflock
)
3351 // Breakpoint executes a breakpoint trap.
3356 // dolockOSThread is called by LockOSThread and lockOSThread below
3357 // after they modify m.locked. Do not allow preemption during this call,
3358 // or else the m might be different in this function than in the caller.
3360 func dolockOSThread() {
3361 if GOARCH
== "wasm" {
3362 return // no threads on wasm yet
3365 _g_
.m
.lockedg
.set(_g_
)
3366 _g_
.lockedm
.set(_g_
.m
)
3371 // LockOSThread wires the calling goroutine to its current operating system thread.
3372 // The calling goroutine will always execute in that thread,
3373 // and no other goroutine will execute in it,
3374 // until the calling goroutine has made as many calls to
3375 // UnlockOSThread as to LockOSThread.
3376 // If the calling goroutine exits without unlocking the thread,
3377 // the thread will be terminated.
3379 // All init functions are run on the startup thread. Calling LockOSThread
3380 // from an init function will cause the main function to be invoked on
3383 // A goroutine should call LockOSThread before calling OS services or
3384 // non-Go library functions that depend on per-thread state.
3385 func LockOSThread() {
3386 if atomic
.Load(&newmHandoff
.haveTemplateThread
) == 0 && GOOS
!= "plan9" {
3387 // If we need to start a new thread from the locked
3388 // thread, we need the template thread. Start it now
3389 // while we're in a known-good state.
3390 startTemplateThread()
3394 if _g_
.m
.lockedExt
== 0 {
3396 panic("LockOSThread nesting overflow")
3402 func lockOSThread() {
3403 getg().m
.lockedInt
++
3407 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below
3408 // after they update m->locked. Do not allow preemption during this call,
3409 // or else the m might be in different in this function than in the caller.
3411 func dounlockOSThread() {
3412 if GOARCH
== "wasm" {
3413 return // no threads on wasm yet
3416 if _g_
.m
.lockedInt
!= 0 || _g_
.m
.lockedExt
!= 0 {
3425 // UnlockOSThread undoes an earlier call to LockOSThread.
3426 // If this drops the number of active LockOSThread calls on the
3427 // calling goroutine to zero, it unwires the calling goroutine from
3428 // its fixed operating system thread.
3429 // If there are no active LockOSThread calls, this is a no-op.
3431 // Before calling UnlockOSThread, the caller must ensure that the OS
3432 // thread is suitable for running other goroutines. If the caller made
3433 // any permanent changes to the state of the thread that would affect
3434 // other goroutines, it should not call this function and thus leave
3435 // the goroutine locked to the OS thread until the goroutine (and
3436 // hence the thread) exits.
3437 func UnlockOSThread() {
3439 if _g_
.m
.lockedExt
== 0 {
3447 func unlockOSThread() {
3449 if _g_
.m
.lockedInt
== 0 {
3450 systemstack(badunlockosthread
)
3456 func badunlockosthread() {
3457 throw("runtime: internal error: misuse of lockOSThread/unlockOSThread")
3460 func gcount() int32 {
3461 n
:= int32(allglen
) - sched
.ngfree
- int32(atomic
.Load(&sched
.ngsys
))
3462 for _
, _p_
:= range allp
{
3466 // All these variables can be changed concurrently, so the result can be inconsistent.
3467 // But at least the current goroutine is running.
3474 func mcount() int32 {
3475 return int32(sched
.mnext
- sched
.nmfreed
)
3483 func _System() { _System() }
3484 func _ExternalCode() { _ExternalCode() }
3485 func _LostExternalCode() { _LostExternalCode() }
3486 func _GC() { _GC() }
3487 func _LostSIGPROFDuringAtomic64() { _LostSIGPROFDuringAtomic64() }
3488 func _VDSO() { _VDSO() }
3490 // Counts SIGPROFs received while in atomic64 critical section, on mips{,le}
3491 var lostAtomic64Count
uint64
3493 var _SystemPC
= funcPC(_System
)
3494 var _ExternalCodePC
= funcPC(_ExternalCode
)
3495 var _LostExternalCodePC
= funcPC(_LostExternalCode
)
3496 var _GCPC
= funcPC(_GC
)
3497 var _LostSIGPROFDuringAtomic64PC
= funcPC(_LostSIGPROFDuringAtomic64
)
3499 // Called if we receive a SIGPROF signal.
3500 // Called by the signal handler, may run during STW.
3501 //go:nowritebarrierrec
3502 func sigprof(pc
uintptr, gp
*g
, mp
*m
) {
3507 // Profiling runs concurrently with GC, so it must not allocate.
3508 // Set a trap in case the code does allocate.
3509 // Note that on windows, one thread takes profiles of all the
3510 // other threads, so mp is usually not getg().m.
3511 // In fact mp may not even be stopped.
3512 // See golang.org/issue/17165.
3513 getg().m
.mallocing
++
3517 // If SIGPROF arrived while already fetching runtime callers
3518 // we can have trouble on older systems because the unwind
3519 // library calls dl_iterate_phdr which was not reentrant in
3520 // the past. alreadyInCallers checks for that.
3521 if gp
== nil ||
alreadyInCallers() {
3525 var stk
[maxCPUProfStack
]uintptr
3528 var stklocs
[maxCPUProfStack
]location
3529 n
= callers(0, stklocs
[:])
3531 // Issue 26595: the stack trace we've just collected is going
3532 // to include frames that we don't want to report in the CPU
3533 // profile, including signal handler frames. Here is what we
3534 // might typically see at the point of "callers" above for a
3535 // signal delivered to the application routine "interesting"
3536 // called by "main".
3538 // 0: runtime.sigprof
3539 // 1: runtime.sighandler
3540 // 2: runtime.sigtrampgo
3541 // 3: runtime.sigtramp
3542 // 4: <signal handler called>
3543 // 5: main.interesting_routine
3546 // To ensure a sane profile, walk through the frames in
3547 // "stklocs" until we find the "runtime.sigtramp" frame, then
3548 // report only those frames below the frame one down from
3549 // that. If for some reason "runtime.sigtramp" is not present,
3550 // don't make any changes.
3551 framesToDiscard
:= 0
3552 for i
:= 0; i
< n
; i
++ {
3553 if stklocs
[i
].function
== "runtime.sigtramp" && i
+2 < n
{
3554 framesToDiscard
= i
+ 2
3555 n
-= framesToDiscard
3559 for i
:= 0; i
< n
; i
++ {
3560 stk
[i
] = stklocs
[i
+framesToDiscard
].pc
3565 // Normal traceback is impossible or has failed.
3566 // Account it against abstract "System" or "GC".
3569 if mp
.preemptoff
!= "" || mp
.helpgc
!= 0 {
3570 stk
[1] = _GCPC
+ sys
.PCQuantum
3572 stk
[1] = _SystemPC
+ sys
.PCQuantum
3577 if (GOARCH
== "mips" || GOARCH
== "mipsle" || GOARCH
== "arm") && lostAtomic64Count
> 0 {
3578 cpuprof
.addLostAtomic64(lostAtomic64Count
)
3579 lostAtomic64Count
= 0
3581 cpuprof
.add(gp
, stk
[:n
])
3583 getg().m
.mallocing
--
3586 // Use global arrays rather than using up lots of stack space in the
3587 // signal handler. This is safe since while we are executing a SIGPROF
3588 // signal other SIGPROF signals are blocked.
3589 var nonprofGoStklocs
[maxCPUProfStack
]location
3590 var nonprofGoStk
[maxCPUProfStack
]uintptr
3592 // sigprofNonGo is called if we receive a SIGPROF signal on a non-Go thread,
3593 // and the signal handler collected a stack trace in sigprofCallers.
3594 // When this is called, sigprofCallersUse will be non-zero.
3595 // g is nil, and what we can do is very limited.
3597 //go:nowritebarrierrec
3598 func sigprofNonGo(pc
uintptr) {
3600 n
:= callers(0, nonprofGoStklocs
[:])
3602 for i
:= 0; i
< n
; i
++ {
3603 nonprofGoStk
[i
] = nonprofGoStklocs
[i
].pc
3608 nonprofGoStk
[0] = pc
3609 nonprofGoStk
[1] = _ExternalCodePC
+ sys
.PCQuantum
3612 cpuprof
.addNonGo(nonprofGoStk
[:n
])
3616 // sigprofNonGoPC is called when a profiling signal arrived on a
3617 // non-Go thread and we have a single PC value, not a stack trace.
3618 // g is nil, and what we can do is very limited.
3620 //go:nowritebarrierrec
3621 func sigprofNonGoPC(pc
uintptr) {
3625 _ExternalCodePC
+ sys
.PCQuantum
,
3627 cpuprof
.addNonGo(stk
)
3631 // setcpuprofilerate sets the CPU profiling rate to hz times per second.
3632 // If hz <= 0, setcpuprofilerate turns off CPU profiling.
3633 func setcpuprofilerate(hz
int32) {
3634 // Force sane arguments.
3639 // Disable preemption, otherwise we can be rescheduled to another thread
3640 // that has profiling enabled.
3644 // Stop profiler on this thread so that it is safe to lock prof.
3645 // if a profiling signal came in while we had prof locked,
3646 // it would deadlock.
3647 setThreadCPUProfiler(0)
3649 for !atomic
.Cas(&prof
.signalLock
, 0, 1) {
3653 setProcessCPUProfiler(hz
)
3656 atomic
.Store(&prof
.signalLock
, 0)
3659 sched
.profilehz
= hz
3663 setThreadCPUProfiler(hz
)
3669 // Change number of processors. The world is stopped, sched is locked.
3670 // gcworkbufs are not being modified by either the GC or
3671 // the write barrier code.
3672 // Returns list of Ps with local work, they need to be scheduled by the caller.
3673 func procresize(nprocs
int32) *p
{
3675 if old
< 0 || nprocs
<= 0 {
3676 throw("procresize: invalid arg")
3679 traceGomaxprocs(nprocs
)
3682 // update statistics
3684 if sched
.procresizetime
!= 0 {
3685 sched
.totaltime
+= int64(old
) * (now
- sched
.procresizetime
)
3687 sched
.procresizetime
= now
3689 // Grow allp if necessary.
3690 if nprocs
> int32(len(allp
)) {
3691 // Synchronize with retake, which could be running
3692 // concurrently since it doesn't run on a P.
3694 if nprocs
<= int32(cap(allp
)) {
3695 allp
= allp
[:nprocs
]
3697 nallp
:= make([]*p
, nprocs
)
3698 // Copy everything up to allp's cap so we
3699 // never lose old allocated Ps.
3700 copy(nallp
, allp
[:cap(allp
)])
3706 // initialize new P's
3707 for i
:= int32(0); i
< nprocs
; i
++ {
3712 pp
.status
= _Pgcstop
3713 pp
.sudogcache
= pp
.sudogbuf
[:0]
3714 pp
.deferpool
= pp
.deferpoolbuf
[:0]
3716 atomicstorep(unsafe
.Pointer(&allp
[i
]), unsafe
.Pointer(pp
))
3718 if pp
.mcache
== nil {
3719 if old
== 0 && i
== 0 {
3720 if getg().m
.mcache
== nil {
3721 throw("missing mcache?")
3723 pp
.mcache
= getg().m
.mcache
// bootstrap
3725 pp
.mcache
= allocmcache()
3731 for i
:= nprocs
; i
< old
; i
++ {
3733 if trace
.enabled
&& p
== getg().m
.p
.ptr() {
3734 // moving to p[0], pretend that we were descheduled
3735 // and then scheduled again to keep the trace sane.
3739 // move all runnable goroutines to the global queue
3740 for p
.runqhead
!= p
.runqtail
{
3741 // pop from tail of local queue
3743 gp
:= p
.runq
[p
.runqtail%uint
32(len(p
.runq
))].ptr()
3744 // push onto head of global queue
3748 globrunqputhead(p
.runnext
.ptr())
3751 // if there's a background worker, make it runnable and put
3752 // it on the global queue so it can clean itself up
3753 if gp
:= p
.gcBgMarkWorker
.ptr(); gp
!= nil {
3754 casgstatus(gp
, _Gwaiting
, _Grunnable
)
3756 traceGoUnpark(gp
, 0)
3759 // This assignment doesn't race because the
3760 // world is stopped.
3761 p
.gcBgMarkWorker
.set(nil)
3763 // Flush p's write barrier buffer.
3764 if gcphase
!= _GCoff
{
3768 for i
:= range p
.sudogbuf
{
3771 p
.sudogcache
= p
.sudogbuf
[:0]
3772 for i
:= range p
.deferpoolbuf
{
3773 p
.deferpoolbuf
[i
] = nil
3775 p
.deferpool
= p
.deferpoolbuf
[:0]
3776 freemcache(p
.mcache
)
3782 // can't free P itself because it can be referenced by an M in syscall
3786 if int32(len(allp
)) != nprocs
{
3788 allp
= allp
[:nprocs
]
3793 if _g_
.m
.p
!= 0 && _g_
.m
.p
.ptr().id
< nprocs
{
3794 // continue to use the current P
3795 _g_
.m
.p
.ptr().status
= _Prunning
3797 // release the current P and acquire allp[0]
3812 for i
:= nprocs
- 1; i
>= 0; i
-- {
3814 if _g_
.m
.p
.ptr() == p
{
3822 p
.link
.set(runnablePs
)
3826 stealOrder
.reset(uint32(nprocs
))
3827 var int32p
*int32 = &gomaxprocs
// make compiler check that gomaxprocs is an int32
3828 atomic
.Store((*uint32)(unsafe
.Pointer(int32p
)), uint32(nprocs
))
3832 // Associate p and the current m.
3834 // This function is allowed to have write barriers even if the caller
3835 // isn't because it immediately acquires _p_.
3837 //go:yeswritebarrierrec
3838 func acquirep(_p_
*p
) {
3839 // Do the part that isn't allowed to have write barriers.
3842 // have p; write barriers now allowed
3844 _g_
.m
.mcache
= _p_
.mcache
3851 // acquirep1 is the first step of acquirep, which actually acquires
3852 // _p_. This is broken out so we can disallow write barriers for this
3853 // part, since we don't yet have a P.
3855 //go:nowritebarrierrec
3856 func acquirep1(_p_
*p
) {
3859 if _g_
.m
.p
!= 0 || _g_
.m
.mcache
!= nil {
3860 throw("acquirep: already in go")
3862 if _p_
.m
!= 0 || _p_
.status
!= _Pidle
{
3867 print("acquirep: p->m=", _p_
.m
, "(", id
, ") p->status=", _p_
.status
, "\n")
3868 throw("acquirep: invalid p state")
3872 _p_
.status
= _Prunning
3875 // Disassociate p and the current m.
3876 func releasep() *p
{
3879 if _g_
.m
.p
== 0 || _g_
.m
.mcache
== nil {
3880 throw("releasep: invalid arg")
3882 _p_
:= _g_
.m
.p
.ptr()
3883 if _p_
.m
.ptr() != _g_
.m || _p_
.mcache
!= _g_
.m
.mcache || _p_
.status
!= _Prunning
{
3884 print("releasep: m=", _g_
.m
, " m->p=", _g_
.m
.p
.ptr(), " p->m=", _p_
.m
, " m->mcache=", _g_
.m
.mcache
, " p->mcache=", _p_
.mcache
, " p->status=", _p_
.status
, "\n")
3885 throw("releasep: invalid p state")
3888 traceProcStop(_g_
.m
.p
.ptr())
3897 func incidlelocked(v
int32) {
3899 sched
.nmidlelocked
+= v
3906 // Check for deadlock situation.
3907 // The check is based on number of running M's, if 0 -> deadlock.
3908 // sched.lock must be held.
3910 // For -buildmode=c-shared or -buildmode=c-archive it's OK if
3911 // there are no running goroutines. The calling program is
3912 // assumed to be running.
3913 if islibrary || isarchive
{
3917 // If we are dying because of a signal caught on an already idle thread,
3918 // freezetheworld will cause all running threads to block.
3919 // And runtime will essentially enter into deadlock state,
3920 // except that there is a thread that will call exit soon.
3925 // If we are not running under cgo, but we have an extra M then account
3926 // for it. (It is possible to have an extra M on Windows without cgo to
3927 // accommodate callbacks created by syscall.NewCallback. See issue #6751
3930 if !iscgo
&& cgoHasExtraM
{
3934 run
:= mcount() - sched
.nmidle
- sched
.nmidlelocked
- sched
.nmsys
3939 print("runtime: checkdead: nmidle=", sched
.nmidle
, " nmidlelocked=", sched
.nmidlelocked
, " mcount=", mcount(), " nmsys=", sched
.nmsys
, "\n")
3940 throw("checkdead: inconsistent counts")
3945 for i
:= 0; i
< len(allgs
); i
++ {
3947 if isSystemGoroutine(gp
) {
3950 s
:= readgstatus(gp
)
3951 switch s
&^ _Gscan
{
3958 print("runtime: checkdead: find g ", gp
.goid
, " in status ", s
, "\n")
3959 throw("checkdead: runnable g")
3963 if grunning
== 0 { // possible if main goroutine calls runtime·Goexit()
3964 throw("no goroutines (main called runtime.Goexit) - deadlock!")
3967 // Maybe jump time forward for playground.
3970 casgstatus(gp
, _Gwaiting
, _Grunnable
)
3974 throw("checkdead: no p for timer")
3978 // There should always be a free M since
3979 // nothing is running.
3980 throw("checkdead: no m for timer")
3983 notewakeup(&mp
.park
)
3987 getg().m
.throwing
= -1 // do not dump full stacks
3988 throw("all goroutines are asleep - deadlock!")
3991 // forcegcperiod is the maximum time in nanoseconds between garbage
3992 // collections. If we go this long without a garbage collection, one
3993 // is forced to run.
3995 // This is a variable for testing purposes. It normally doesn't change.
3996 var forcegcperiod
int64 = 2 * 60 * 1e9
3998 // Always runs without a P, so write barriers are not allowed.
4000 //go:nowritebarrierrec
4007 // If a heap span goes unused for 5 minutes after a garbage collection,
4008 // we hand it back to the operating system.
4009 scavengelimit
:= int64(5 * 60 * 1e9
)
4011 if debug
.scavenge
> 0 {
4012 // Scavenge-a-lot for testing.
4013 forcegcperiod
= 10 * 1e6
4014 scavengelimit
= 20 * 1e6
4017 lastscavenge
:= nanotime()
4020 lasttrace
:= int64(0)
4021 idle
:= 0 // how many cycles in succession we had not wokeup somebody
4024 if idle
== 0 { // start with 20us sleep...
4026 } else if idle
> 50 { // start doubling the sleep after 1ms...
4029 if delay
> 10*1000 { // up to 10ms
4033 if debug
.schedtrace
<= 0 && (sched
.gcwaiting
!= 0 || atomic
.Load(&sched
.npidle
) == uint32(gomaxprocs
)) {
4035 if atomic
.Load(&sched
.gcwaiting
) != 0 || atomic
.Load(&sched
.npidle
) == uint32(gomaxprocs
) {
4036 atomic
.Store(&sched
.sysmonwait
, 1)
4038 // Make wake-up period small enough
4039 // for the sampling to be correct.
4040 maxsleep
:= forcegcperiod
/ 2
4041 if scavengelimit
< forcegcperiod
{
4042 maxsleep
= scavengelimit
/ 2
4045 if osRelaxMinNS
> 0 {
4046 next
:= timeSleepUntil()
4048 if next
-now
< osRelaxMinNS
{
4055 notetsleep(&sched
.sysmonnote
, maxsleep
)
4060 atomic
.Store(&sched
.sysmonwait
, 0)
4061 noteclear(&sched
.sysmonnote
)
4067 // trigger libc interceptors if needed
4068 if *cgo_yield
!= nil {
4069 asmcgocall(*cgo_yield
, nil)
4071 // poll network if not polled for more than 10ms
4072 lastpoll
:= int64(atomic
.Load64(&sched
.lastpoll
))
4074 if netpollinited() && lastpoll
!= 0 && lastpoll
+10*1000*1000 < now
{
4075 atomic
.Cas64(&sched
.lastpoll
, uint64(lastpoll
), uint64(now
))
4076 gp
:= netpoll(false) // non-blocking - returns list of goroutines
4078 // Need to decrement number of idle locked M's
4079 // (pretending that one more is running) before injectglist.
4080 // Otherwise it can lead to the following situation:
4081 // injectglist grabs all P's but before it starts M's to run the P's,
4082 // another M returns from syscall, finishes running its G,
4083 // observes that there is no work to do and no other running M's
4084 // and reports deadlock.
4090 // retake P's blocked in syscalls
4091 // and preempt long running G's
4092 if retake(now
) != 0 {
4097 // check if we need to force a GC
4098 if t
:= (gcTrigger
{kind
: gcTriggerTime
, now
: now
}); t
.test() && atomic
.Load(&forcegc
.idle
) != 0 {
4101 forcegc
.g
.schedlink
= 0
4102 injectglist(forcegc
.g
)
4103 unlock(&forcegc
.lock
)
4105 // scavenge heap once in a while
4106 if lastscavenge
+scavengelimit
/2 < now
{
4107 mheap_
.scavenge(int32(nscavenge
), uint64(now
), uint64(scavengelimit
))
4111 if debug
.schedtrace
> 0 && lasttrace
+int64(debug
.schedtrace
)*1000000 <= now
{
4113 schedtrace(debug
.scheddetail
> 0)
4118 type sysmontick
struct {
4125 // forcePreemptNS is the time slice given to a G before it is
4127 const forcePreemptNS
= 10 * 1000 * 1000 // 10ms
4129 func retake(now
int64) uint32 {
4131 // Prevent allp slice changes. This lock will be completely
4132 // uncontended unless we're already stopping the world.
4134 // We can't use a range loop over allp because we may
4135 // temporarily drop the allpLock. Hence, we need to re-fetch
4136 // allp each time around the loop.
4137 for i
:= 0; i
< len(allp
); i
++ {
4140 // This can happen if procresize has grown
4141 // allp but not yet created new Ps.
4144 pd
:= &_p_
.sysmontick
4147 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us).
4148 t
:= int64(_p_
.syscalltick
)
4149 if int64(pd
.syscalltick
) != t
{
4150 pd
.syscalltick
= uint32(t
)
4151 pd
.syscallwhen
= now
4154 // On the one hand we don't want to retake Ps if there is no other work to do,
4155 // but on the other hand we want to retake them eventually
4156 // because they can prevent the sysmon thread from deep sleep.
4157 if runqempty(_p_
) && atomic
.Load(&sched
.nmspinning
)+atomic
.Load(&sched
.npidle
) > 0 && pd
.syscallwhen
+10*1000*1000 > now
{
4160 // Drop allpLock so we can take sched.lock.
4162 // Need to decrement number of idle locked M's
4163 // (pretending that one more is running) before the CAS.
4164 // Otherwise the M from which we retake can exit the syscall,
4165 // increment nmidle and report deadlock.
4167 if atomic
.Cas(&_p_
.status
, s
, _Pidle
) {
4169 traceGoSysBlock(_p_
)
4178 } else if s
== _Prunning
{
4179 // Preempt G if it's running for too long.
4180 t
:= int64(_p_
.schedtick
)
4181 if int64(pd
.schedtick
) != t
{
4182 pd
.schedtick
= uint32(t
)
4186 if pd
.schedwhen
+forcePreemptNS
> now
{
4196 // Tell all goroutines that they have been preempted and they should stop.
4197 // This function is purely best-effort. It can fail to inform a goroutine if a
4198 // processor just started running it.
4199 // No locks need to be held.
4200 // Returns true if preemption request was issued to at least one goroutine.
4201 func preemptall() bool {
4203 for _
, _p_
:= range allp
{
4204 if _p_
.status
!= _Prunning
{
4207 if preemptone(_p_
) {
4214 // Tell the goroutine running on processor P to stop.
4215 // This function is purely best-effort. It can incorrectly fail to inform the
4216 // goroutine. It can send inform the wrong goroutine. Even if it informs the
4217 // correct goroutine, that goroutine might ignore the request if it is
4218 // simultaneously executing newstack.
4219 // No lock needs to be held.
4220 // Returns true if preemption request was issued.
4221 // The actual preemption will happen at some point in the future
4222 // and will be indicated by the gp->status no longer being
4224 func preemptone(_p_
*p
) bool {
4226 if mp
== nil || mp
== getg().m
{
4230 if gp
== nil || gp
== mp
.g0
{
4236 // At this point the gc implementation sets gp.stackguard0 to
4237 // a value that causes the goroutine to suspend itself.
4238 // gccgo has no support for this, and it's hard to support.
4239 // The split stack code reads a value from its TCB.
4240 // We have no way to set a value in the TCB of a different thread.
4241 // And, of course, not all systems support split stack anyhow.
4242 // Checking the field in the g is expensive, since it requires
4243 // loading the g from TLS. The best mechanism is likely to be
4244 // setting a global variable and figuring out a way to efficiently
4245 // check that global variable.
4247 // For now we check gp.preempt in schedule, mallocgc, selectgo,
4248 // and a few other places, which is at least better than doing
4256 func schedtrace(detailed
bool) {
4263 print("SCHED ", (now
-starttime
)/1e6
, "ms: gomaxprocs=", gomaxprocs
, " idleprocs=", sched
.npidle
, " threads=", mcount(), " spinningthreads=", sched
.nmspinning
, " idlethreads=", sched
.nmidle
, " runqueue=", sched
.runqsize
)
4265 print(" gcwaiting=", sched
.gcwaiting
, " nmidlelocked=", sched
.nmidlelocked
, " stopwait=", sched
.stopwait
, " sysmonwait=", sched
.sysmonwait
, "\n")
4267 // We must be careful while reading data from P's, M's and G's.
4268 // Even if we hold schedlock, most data can be changed concurrently.
4269 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
4270 for i
, _p_
:= range allp
{
4272 h
:= atomic
.Load(&_p_
.runqhead
)
4273 t
:= atomic
.Load(&_p_
.runqtail
)
4279 print(" P", i
, ": status=", _p_
.status
, " schedtick=", _p_
.schedtick
, " syscalltick=", _p_
.syscalltick
, " m=", id
, " runqsize=", t
-h
, " gfreecnt=", _p_
.gfreecnt
, "\n")
4281 // In non-detailed mode format lengths of per-P run queues as:
4282 // [len1 len2 len3 len4]
4288 if i
== len(allp
)-1 {
4299 for mp
:= allm
; mp
!= nil; mp
= mp
.alllink
{
4302 lockedg
:= mp
.lockedg
.ptr()
4315 print(" M", mp
.id
, ": p=", id1
, " curg=", id2
, " mallocing=", mp
.mallocing
, " throwing=", mp
.throwing
, " preemptoff=", mp
.preemptoff
, ""+" locks=", mp
.locks
, " dying=", mp
.dying
, " helpgc=", mp
.helpgc
, " spinning=", mp
.spinning
, " blocked=", mp
.blocked
, " lockedg=", id3
, "\n")
4319 for gi
:= 0; gi
< len(allgs
); gi
++ {
4322 lockedm
:= gp
.lockedm
.ptr()
4331 print(" G", gp
.goid
, ": status=", readgstatus(gp
), "(", gp
.waitreason
.String(), ") m=", id1
, " lockedm=", id2
, "\n")
4337 // Put mp on midle list.
4338 // Sched must be locked.
4339 // May run during STW, so write barriers are not allowed.
4340 //go:nowritebarrierrec
4342 mp
.schedlink
= sched
.midle
4348 // Try to get an m from midle list.
4349 // Sched must be locked.
4350 // May run during STW, so write barriers are not allowed.
4351 //go:nowritebarrierrec
4353 mp
:= sched
.midle
.ptr()
4355 sched
.midle
= mp
.schedlink
4361 // Put gp on the global runnable queue.
4362 // Sched must be locked.
4363 // May run during STW, so write barriers are not allowed.
4364 //go:nowritebarrierrec
4365 func globrunqput(gp
*g
) {
4367 if sched
.runqtail
!= 0 {
4368 sched
.runqtail
.ptr().schedlink
.set(gp
)
4370 sched
.runqhead
.set(gp
)
4372 sched
.runqtail
.set(gp
)
4376 // Put gp at the head of the global runnable queue.
4377 // Sched must be locked.
4378 // May run during STW, so write barriers are not allowed.
4379 //go:nowritebarrierrec
4380 func globrunqputhead(gp
*g
) {
4381 gp
.schedlink
= sched
.runqhead
4382 sched
.runqhead
.set(gp
)
4383 if sched
.runqtail
== 0 {
4384 sched
.runqtail
.set(gp
)
4389 // Put a batch of runnable goroutines on the global runnable queue.
4390 // Sched must be locked.
4391 func globrunqputbatch(ghead
*g
, gtail
*g
, n
int32) {
4393 if sched
.runqtail
!= 0 {
4394 sched
.runqtail
.ptr().schedlink
.set(ghead
)
4396 sched
.runqhead
.set(ghead
)
4398 sched
.runqtail
.set(gtail
)
4402 // Try get a batch of G's from the global runnable queue.
4403 // Sched must be locked.
4404 func globrunqget(_p_
*p
, max
int32) *g
{
4405 if sched
.runqsize
== 0 {
4409 n
:= sched
.runqsize
/gomaxprocs
+ 1
4410 if n
> sched
.runqsize
{
4413 if max
> 0 && n
> max
{
4416 if n
> int32(len(_p_
.runq
))/2 {
4417 n
= int32(len(_p_
.runq
)) / 2
4421 if sched
.runqsize
== 0 {
4425 gp
:= sched
.runqhead
.ptr()
4426 sched
.runqhead
= gp
.schedlink
4429 gp1
:= sched
.runqhead
.ptr()
4430 sched
.runqhead
= gp1
.schedlink
4431 runqput(_p_
, gp1
, false)
4436 // Put p to on _Pidle list.
4437 // Sched must be locked.
4438 // May run during STW, so write barriers are not allowed.
4439 //go:nowritebarrierrec
4440 func pidleput(_p_
*p
) {
4441 if !runqempty(_p_
) {
4442 throw("pidleput: P has non-empty run queue")
4444 _p_
.link
= sched
.pidle
4445 sched
.pidle
.set(_p_
)
4446 atomic
.Xadd(&sched
.npidle
, 1) // TODO: fast atomic
4449 // Try get a p from _Pidle list.
4450 // Sched must be locked.
4451 // May run during STW, so write barriers are not allowed.
4452 //go:nowritebarrierrec
4453 func pidleget() *p
{
4454 _p_
:= sched
.pidle
.ptr()
4456 sched
.pidle
= _p_
.link
4457 atomic
.Xadd(&sched
.npidle
, -1) // TODO: fast atomic
4462 // runqempty returns true if _p_ has no Gs on its local run queue.
4463 // It never returns true spuriously.
4464 func runqempty(_p_
*p
) bool {
4465 // Defend against a race where 1) _p_ has G1 in runqnext but runqhead == runqtail,
4466 // 2) runqput on _p_ kicks G1 to the runq, 3) runqget on _p_ empties runqnext.
4467 // Simply observing that runqhead == runqtail and then observing that runqnext == nil
4468 // does not mean the queue is empty.
4470 head
:= atomic
.Load(&_p_
.runqhead
)
4471 tail
:= atomic
.Load(&_p_
.runqtail
)
4472 runnext
:= atomic
.Loaduintptr((*uintptr)(unsafe
.Pointer(&_p_
.runnext
)))
4473 if tail
== atomic
.Load(&_p_
.runqtail
) {
4474 return head
== tail
&& runnext
== 0
4479 // To shake out latent assumptions about scheduling order,
4480 // we introduce some randomness into scheduling decisions
4481 // when running with the race detector.
4482 // The need for this was made obvious by changing the
4483 // (deterministic) scheduling order in Go 1.5 and breaking
4484 // many poorly-written tests.
4485 // With the randomness here, as long as the tests pass
4486 // consistently with -race, they shouldn't have latent scheduling
4488 const randomizeScheduler
= raceenabled
4490 // runqput tries to put g on the local runnable queue.
4491 // If next is false, runqput adds g to the tail of the runnable queue.
4492 // If next is true, runqput puts g in the _p_.runnext slot.
4493 // If the run queue is full, runnext puts g on the global queue.
4494 // Executed only by the owner P.
4495 func runqput(_p_
*p
, gp
*g
, next
bool) {
4496 if randomizeScheduler
&& next
&& fastrand()%2
== 0 {
4502 oldnext
:= _p_
.runnext
4503 if !_p_
.runnext
.cas(oldnext
, guintptr(unsafe
.Pointer(gp
))) {
4509 // Kick the old runnext out to the regular run queue.
4514 h
:= atomic
.Load(&_p_
.runqhead
) // load-acquire, synchronize with consumers
4516 if t
-h
< uint32(len(_p_
.runq
)) {
4517 _p_
.runq
[t%uint
32(len(_p_
.runq
))].set(gp
)
4518 atomic
.Store(&_p_
.runqtail
, t
+1) // store-release, makes the item available for consumption
4521 if runqputslow(_p_
, gp
, h
, t
) {
4524 // the queue is not full, now the put above must succeed
4528 // Put g and a batch of work from local runnable queue on global queue.
4529 // Executed only by the owner P.
4530 func runqputslow(_p_
*p
, gp
*g
, h
, t
uint32) bool {
4531 var batch
[len(_p_
.runq
)/2 + 1]*g
4533 // First, grab a batch from local queue.
4536 if n
!= uint32(len(_p_
.runq
)/2) {
4537 throw("runqputslow: queue is not full")
4539 for i
:= uint32(0); i
< n
; i
++ {
4540 batch
[i
] = _p_
.runq
[(h
+i
)%uint
32(len(_p_
.runq
))].ptr()
4542 if !atomic
.Cas(&_p_
.runqhead
, h
, h
+n
) { // cas-release, commits consume
4547 if randomizeScheduler
{
4548 for i
:= uint32(1); i
<= n
; i
++ {
4549 j
:= fastrandn(i
+ 1)
4550 batch
[i
], batch
[j
] = batch
[j
], batch
[i
]
4554 // Link the goroutines.
4555 for i
:= uint32(0); i
< n
; i
++ {
4556 batch
[i
].schedlink
.set(batch
[i
+1])
4559 // Now put the batch on global queue.
4561 globrunqputbatch(batch
[0], batch
[n
], int32(n
+1))
4566 // Get g from local runnable queue.
4567 // If inheritTime is true, gp should inherit the remaining time in the
4568 // current time slice. Otherwise, it should start a new time slice.
4569 // Executed only by the owner P.
4570 func runqget(_p_
*p
) (gp
*g
, inheritTime
bool) {
4571 // If there's a runnext, it's the next G to run.
4577 if _p_
.runnext
.cas(next
, 0) {
4578 return next
.ptr(), true
4583 h
:= atomic
.Load(&_p_
.runqhead
) // load-acquire, synchronize with other consumers
4588 gp
:= _p_
.runq
[h%uint
32(len(_p_
.runq
))].ptr()
4589 if atomic
.Cas(&_p_
.runqhead
, h
, h
+1) { // cas-release, commits consume
4595 // Grabs a batch of goroutines from _p_'s runnable queue into batch.
4596 // Batch is a ring buffer starting at batchHead.
4597 // Returns number of grabbed goroutines.
4598 // Can be executed by any P.
4599 func runqgrab(_p_
*p
, batch
*[256]guintptr
, batchHead
uint32, stealRunNextG
bool) uint32 {
4601 h
:= atomic
.Load(&_p_
.runqhead
) // load-acquire, synchronize with other consumers
4602 t
:= atomic
.Load(&_p_
.runqtail
) // load-acquire, synchronize with the producer
4607 // Try to steal from _p_.runnext.
4608 if next
:= _p_
.runnext
; next
!= 0 {
4609 if _p_
.status
== _Prunning
{
4610 // Sleep to ensure that _p_ isn't about to run the g
4611 // we are about to steal.
4612 // The important use case here is when the g running
4613 // on _p_ ready()s another g and then almost
4614 // immediately blocks. Instead of stealing runnext
4615 // in this window, back off to give _p_ a chance to
4616 // schedule runnext. This will avoid thrashing gs
4617 // between different Ps.
4618 // A sync chan send/recv takes ~50ns as of time of
4619 // writing, so 3us gives ~50x overshoot.
4620 if GOOS
!= "windows" {
4623 // On windows system timer granularity is
4624 // 1-15ms, which is way too much for this
4625 // optimization. So just yield.
4629 if !_p_
.runnext
.cas(next
, 0) {
4632 batch
[batchHead%uint
32(len(batch
))] = next
4638 if n
> uint32(len(_p_
.runq
)/2) { // read inconsistent h and t
4641 for i
:= uint32(0); i
< n
; i
++ {
4642 g
:= _p_
.runq
[(h
+i
)%uint
32(len(_p_
.runq
))]
4643 batch
[(batchHead
+i
)%uint
32(len(batch
))] = g
4645 if atomic
.Cas(&_p_
.runqhead
, h
, h
+n
) { // cas-release, commits consume
4651 // Steal half of elements from local runnable queue of p2
4652 // and put onto local runnable queue of p.
4653 // Returns one of the stolen elements (or nil if failed).
4654 func runqsteal(_p_
, p2
*p
, stealRunNextG
bool) *g
{
4656 n
:= runqgrab(p2
, &_p_
.runq
, t
, stealRunNextG
)
4661 gp
:= _p_
.runq
[(t
+n
)%uint
32(len(_p_
.runq
))].ptr()
4665 h
:= atomic
.Load(&_p_
.runqhead
) // load-acquire, synchronize with consumers
4666 if t
-h
+n
>= uint32(len(_p_
.runq
)) {
4667 throw("runqsteal: runq overflow")
4669 atomic
.Store(&_p_
.runqtail
, t
+n
) // store-release, makes the item available for consumption
4673 //go:linkname setMaxThreads runtime..z2fdebug.setMaxThreads
4674 func setMaxThreads(in
int) (out
int) {
4676 out
= int(sched
.maxmcount
)
4677 if in
> 0x7fffffff { // MaxInt32
4678 sched
.maxmcount
= 0x7fffffff
4680 sched
.maxmcount
= int32(in
)
4687 func haveexperiment(name
string) bool {
4688 // The gofrontend does not support experiments.
4693 func procPin() int {
4698 return int(mp
.p
.ptr().id
)
4707 //go:linkname sync_runtime_procPin sync.runtime_procPin
4709 func sync_runtime_procPin() int {
4713 //go:linkname sync_runtime_procUnpin sync.runtime_procUnpin
4715 func sync_runtime_procUnpin() {
4719 //go:linkname sync_atomic_runtime_procPin sync..z2fatomic.runtime_procPin
4721 func sync_atomic_runtime_procPin() int {
4725 //go:linkname sync_atomic_runtime_procUnpin sync..z2fatomic.runtime_procUnpin
4727 func sync_atomic_runtime_procUnpin() {
4731 // Active spinning for sync.Mutex.
4732 //go:linkname sync_runtime_canSpin sync.runtime_canSpin
4734 func sync_runtime_canSpin(i
int) bool {
4735 // sync.Mutex is cooperative, so we are conservative with spinning.
4736 // Spin only few times and only if running on a multicore machine and
4737 // GOMAXPROCS>1 and there is at least one other running P and local runq is empty.
4738 // As opposed to runtime mutex we don't do passive spinning here,
4739 // because there can be work on global runq or on other Ps.
4740 if i
>= active_spin || ncpu
<= 1 || gomaxprocs
<= int32(sched
.npidle
+sched
.nmspinning
)+1 {
4743 if p
:= getg().m
.p
.ptr(); !runqempty(p
) {
4749 //go:linkname sync_runtime_doSpin sync.runtime_doSpin
4751 func sync_runtime_doSpin() {
4752 procyield(active_spin_cnt
)
4755 var stealOrder randomOrder
4757 // randomOrder/randomEnum are helper types for randomized work stealing.
4758 // They allow to enumerate all Ps in different pseudo-random orders without repetitions.
4759 // The algorithm is based on the fact that if we have X such that X and GOMAXPROCS
4760 // are coprime, then a sequences of (i + X) % GOMAXPROCS gives the required enumeration.
4761 type randomOrder
struct {
4766 type randomEnum
struct {
4773 func (ord
*randomOrder
) reset(count
uint32) {
4775 ord
.coprimes
= ord
.coprimes
[:0]
4776 for i
:= uint32(1); i
<= count
; i
++ {
4777 if gcd(i
, count
) == 1 {
4778 ord
.coprimes
= append(ord
.coprimes
, i
)
4783 func (ord
*randomOrder
) start(i
uint32) randomEnum
{
4787 inc
: ord
.coprimes
[i%uint
32(len(ord
.coprimes
))],
4791 func (enum
*randomEnum
) done() bool {
4792 return enum
.i
== enum
.count
4795 func (enum
*randomEnum
) next() {
4797 enum
.pos
= (enum
.pos
+ enum
.inc
) % enum
.count
4800 func (enum
*randomEnum
) position() uint32 {
4804 func gcd(a
, b
uint32) uint32 {