1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
13 #ifdef HAVE_DL_ITERATE_PHDR
25 #ifdef USING_SPLIT_STACK
27 /* FIXME: These are not declared anywhere. */
29 extern void __splitstack_getcontext(void *context
[10]);
31 extern void __splitstack_setcontext(void *context
[10]);
33 extern void *__splitstack_makecontext(size_t, void *context
[10], size_t *);
35 extern void * __splitstack_resetcontext(void *context
[10], size_t *);
37 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
40 extern void __splitstack_block_signals (int *, int *);
42 extern void __splitstack_block_signals_context (void *context
[10], int *,
47 #ifndef PTHREAD_STACK_MIN
48 # define PTHREAD_STACK_MIN 8192
51 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
52 # define StackMin PTHREAD_STACK_MIN
54 # define StackMin 2 * 1024 * 1024
57 uintptr runtime_stacks_sys
;
59 static void schedule(G
*);
61 static void gtraceback(G
*);
63 typedef struct Sched Sched
;
66 G runtime_g0
; // idle goroutine for m0
75 #ifndef SETCONTEXT_CLOBBERS_TLS
83 fixcontext(ucontext_t
*c
__attribute__ ((unused
)))
89 # if defined(__x86_64__) && defined(__sun__)
91 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
92 // register to that of the thread which called getcontext. The effect
93 // is that the address of all __thread variables changes. This bug
94 // also affects pthread_self() and pthread_getspecific. We work
95 // around it by clobbering the context field directly to keep %fs the
98 static __thread greg_t fs
;
106 fs
= c
.uc_mcontext
.gregs
[REG_FSBASE
];
110 fixcontext(ucontext_t
* c
)
112 c
->uc_mcontext
.gregs
[REG_FSBASE
] = fs
;
115 # elif defined(__NetBSD__)
117 // NetBSD has a bug: setcontext clobbers tlsbase, we need to save
118 // and restore it ourselves.
120 static __thread __greg_t tlsbase
;
128 tlsbase
= c
.uc_mcontext
._mc_tlsbase
;
132 fixcontext(ucontext_t
* c
)
134 c
->uc_mcontext
._mc_tlsbase
= tlsbase
;
139 # error unknown case for SETCONTEXT_CLOBBERS_TLS
145 // We can not always refer to the TLS variables directly. The
146 // compiler will call tls_get_addr to get the address of the variable,
147 // and it may hold it in a register across a call to schedule. When
148 // we get back from the call we may be running in a different thread,
149 // in which case the register now points to the TLS variable for a
150 // different thread. We use non-inlinable functions to avoid this
153 G
* runtime_g(void) __attribute__ ((noinline
, no_split_stack
));
161 M
* runtime_m(void) __attribute__ ((noinline
, no_split_stack
));
169 int32 runtime_gcwaiting
;
178 // The static TLS size. See runtime_newm.
181 #ifdef HAVE_DL_ITERATE_PHDR
183 // Called via dl_iterate_phdr.
186 addtls(struct dl_phdr_info
* info
, size_t size
__attribute__ ((unused
)), void *data
)
188 size_t *total
= (size_t *)data
;
191 for(i
= 0; i
< info
->dlpi_phnum
; ++i
) {
192 if(info
->dlpi_phdr
[i
].p_type
== PT_TLS
)
193 *total
+= info
->dlpi_phdr
[i
].p_memsz
;
198 // Set the total TLS size.
205 dl_iterate_phdr(addtls
, (void *)&total
);
220 // The go scheduler's job is to match ready-to-run goroutines (`g's)
221 // with waiting-for-work schedulers (`m's). If there are ready g's
222 // and no waiting m's, ready() will start a new m running in a new
223 // OS thread, so that all ready g's can run simultaneously, up to a limit.
224 // For now, m's never go away.
226 // By default, Go keeps only one kernel thread (m) running user code
227 // at a single time; other threads may be blocked in the operating system.
228 // Setting the environment variable $GOMAXPROCS or calling
229 // runtime.GOMAXPROCS() will change the number of user threads
230 // allowed to execute simultaneously. $GOMAXPROCS is thus an
231 // approximation of the maximum number of cores to use.
233 // Even a program that can run without deadlock in a single process
234 // might use more m's if given the chance. For example, the prime
235 // sieve will use as many m's as there are primes (up to runtime_sched.mmax),
236 // allowing different stages of the pipeline to execute in parallel.
237 // We could revisit this choice, only kicking off new m's for blocking
238 // system calls, but that would limit the amount of parallel computation
239 // that go would try to do.
241 // In general, one could imagine all sorts of refinements to the
242 // scheduler, but the goal now is just to get something working on
248 G
*gfree
; // available g's (status == Gdead)
251 G
*ghead
; // g's waiting to run
253 int32 gwait
; // number of g's waiting to run
254 int32 gcount
; // number of g's that are alive
255 int32 grunning
; // number of g's running on cpu or in syscall
257 M
*mhead
; // m's waiting for work
258 int32 mwait
; // number of m's waiting for work
259 int32 mcount
; // number of m's that have been created
261 volatile uint32 atomic
; // atomic scheduling word (see below)
263 int32 profilehz
; // cpu profiling rate
265 bool init
; // running initialization
266 bool lockmain
; // init called runtime.LockOSThread
268 Note stopped
; // one g can set waitstop and wait here for m's to stop
271 // The atomic word in sched is an atomic uint32 that
272 // holds these fields.
274 // [15 bits] mcpu number of m's executing on cpu
275 // [15 bits] mcpumax max number of m's allowed on cpu
276 // [1 bit] waitstop some g is waiting on stopped
277 // [1 bit] gwaiting gwait != 0
279 // These fields are the information needed by entersyscall
280 // and exitsyscall to decide whether to coordinate with the
281 // scheduler. Packing them into a single machine word lets
282 // them use a fast path with a single atomic read/write and
283 // no lock/unlock. This greatly reduces contention in
284 // syscall- or cgo-heavy multithreaded programs.
286 // Except for entersyscall and exitsyscall, the manipulations
287 // to these fields only happen while holding the schedlock,
288 // so the routines holding schedlock only need to worry about
289 // what entersyscall and exitsyscall do, not the other routines
290 // (which also use the schedlock).
292 // In particular, entersyscall and exitsyscall only read mcpumax,
293 // waitstop, and gwaiting. They never write them. Thus, writes to those
294 // fields can be done (holding schedlock) without fear of write conflicts.
295 // There may still be logic conflicts: for example, the set of waitstop must
296 // be conditioned on mcpu >= mcpumax or else the wait may be a
297 // spurious sleep. The Promela model in proc.p verifies these accesses.
300 mcpuMask
= (1<<mcpuWidth
) - 1,
302 mcpumaxShift
= mcpuShift
+ mcpuWidth
,
303 waitstopShift
= mcpumaxShift
+ mcpuWidth
,
304 gwaitingShift
= waitstopShift
+1,
306 // The max value of GOMAXPROCS is constrained
307 // by the max value we can store in the bit fields
308 // of the atomic word. Reserve a few high values
309 // so that we can detect accidental decrement
311 maxgomaxprocs
= mcpuMask
- 10,
314 #define atomic_mcpu(v) (((v)>>mcpuShift)&mcpuMask)
315 #define atomic_mcpumax(v) (((v)>>mcpumaxShift)&mcpuMask)
316 #define atomic_waitstop(v) (((v)>>waitstopShift)&1)
317 #define atomic_gwaiting(v) (((v)>>gwaitingShift)&1)
320 int32 runtime_gomaxprocs
;
321 bool runtime_singleproc
;
323 static bool canaddmcpu(void);
325 // An m that is waiting for notewakeup(&m->havenextg). This may
326 // only be accessed while the scheduler lock is held. This is used to
327 // minimize the number of times we call notewakeup while the scheduler
328 // lock is held, since the m will normally move quickly to lock the
329 // scheduler itself, producing lock contention.
332 // Scheduling helpers. Sched must be locked.
333 static void gput(G
*); // put/get on ghead/gtail
334 static G
* gget(void);
335 static void mput(M
*); // put/get on mhead
337 static void gfput(G
*); // put/get on gfree
338 static G
* gfget(void);
339 static void matchmg(void); // match m's to g's
340 static void readylocked(G
*); // ready, but sched is locked
341 static void mnextg(M
*, G
*);
342 static void mcommoninit(M
*);
350 v
= runtime_sched
.atomic
;
352 w
&= ~(mcpuMask
<<mcpumaxShift
);
353 w
|= n
<<mcpumaxShift
;
354 if(runtime_cas(&runtime_sched
.atomic
, v
, w
))
359 // First function run by a new goroutine. This replaces gogocall.
365 if(g
->traceback
!= nil
)
368 fn
= (void (*)(void*))(g
->entry
);
373 // Switch context to a different goroutine. This is like longjmp.
374 static void runtime_gogo(G
*) __attribute__ ((noinline
));
376 runtime_gogo(G
* newg
)
378 #ifdef USING_SPLIT_STACK
379 __splitstack_setcontext(&newg
->stack_context
[0]);
382 newg
->fromgogo
= true;
383 fixcontext(&newg
->context
);
384 setcontext(&newg
->context
);
385 runtime_throw("gogo setcontext returned");
388 // Save context and call fn passing g as a parameter. This is like
389 // setjmp. Because getcontext always returns 0, unlike setjmp, we use
390 // g->fromgogo as a code. It will be true if we got here via
391 // setcontext. g == nil the first time this is called in a new m.
392 static void runtime_mcall(void (*)(G
*)) __attribute__ ((noinline
));
394 runtime_mcall(void (*pfn
)(G
*))
398 #ifndef USING_SPLIT_STACK
402 // Ensure that all registers are on the stack for the garbage
404 __builtin_unwind_init();
409 runtime_throw("runtime: mcall called on m->g0 stack");
413 #ifdef USING_SPLIT_STACK
414 __splitstack_getcontext(&g
->stack_context
[0]);
418 gp
->fromgogo
= false;
419 getcontext(&gp
->context
);
421 // When we return from getcontext, we may be running
422 // in a new thread. That means that m and g may have
423 // changed. They are global variables so we will
424 // reload them, but the addresses of m and g may be
425 // cached in our local stack frame, and those
426 // addresses may be wrong. Call functions to reload
427 // the values for this thread.
431 if(gp
->traceback
!= nil
)
434 if (gp
== nil
|| !gp
->fromgogo
) {
435 #ifdef USING_SPLIT_STACK
436 __splitstack_setcontext(&mp
->g0
->stack_context
[0]);
438 mp
->g0
->entry
= (byte
*)pfn
;
441 // It's OK to set g directly here because this case
442 // can not occur if we got here via a setcontext to
443 // the getcontext call just above.
446 fixcontext(&mp
->g0
->context
);
447 setcontext(&mp
->g0
->context
);
448 runtime_throw("runtime: mcall function returned");
452 // Keep trace of scavenger's goroutine for deadlock detection.
455 // The bootstrap sequence is:
459 // make & queue new G
460 // call runtime_mstart
462 // The new G calls runtime_main.
464 runtime_schedinit(void)
479 runtime_mallocinit();
486 // Allocate internal symbol table representation now,
487 // so that we don't need to call malloc when we crash.
488 // runtime_findfunc(0);
490 runtime_gomaxprocs
= 1;
491 p
= runtime_getenv("GOMAXPROCS");
492 if(p
!= nil
&& (n
= runtime_atoi(p
)) != 0) {
493 if(n
> maxgomaxprocs
)
495 runtime_gomaxprocs
= n
;
497 // wait for the main goroutine to start before taking
498 // GOMAXPROCS into account.
500 runtime_singleproc
= runtime_gomaxprocs
== 1;
502 canaddmcpu(); // mcpu++ to account for bootstrap m
503 m
->helpgc
= 1; // flag to tell schedule() to mcpu--
504 runtime_sched
.grunning
++;
506 // Can not enable GC until all roots are registered.
507 // mstats.enablegc = 1;
514 extern void main_init(void) __asm__ (GOSYM_PREFIX
"__go_init_main");
515 extern void main_main(void) __asm__ (GOSYM_PREFIX
"main.main");
517 // The main goroutine.
521 // Lock the main goroutine onto this, the main OS thread,
522 // during initialization. Most programs won't care, but a few
523 // do require certain calls to be made by the main thread.
524 // Those can arrange for main.main to run in the main thread
525 // by calling runtime.LockOSThread during initialization
526 // to preserve the lock.
527 runtime_LockOSThread();
528 // From now on, newgoroutines may use non-main threads.
529 setmcpumax(runtime_gomaxprocs
);
530 runtime_sched
.init
= true;
531 scvg
= __go_go(runtime_MHeap_Scavenger
, nil
);
532 scvg
->issystem
= true;
534 runtime_sched
.init
= false;
535 if(!runtime_sched
.lockmain
)
536 runtime_UnlockOSThread();
538 // For gccgo we have to wait until after main is initialized
539 // to enable GC, because initializing main registers the GC
543 // The deadlock detection has false negatives.
544 // Let scvg start up, to eliminate the false negative
545 // for the trivial program func main() { select{} }.
556 // Lock the scheduler.
560 runtime_lock(&runtime_sched
);
563 // Unlock the scheduler.
571 runtime_unlock(&runtime_sched
);
573 runtime_notewakeup(&mp
->havenextg
);
579 g
->status
= Gmoribund
;
584 runtime_goroutineheader(G
*gp
)
603 status
= gp
->waitreason
;
614 runtime_printf("goroutine %D [%s]:\n", gp
->goid
, status
);
618 runtime_goroutinetrailer(G
*g
)
620 if(g
!= nil
&& g
->gopc
!= 0 && g
->goid
!= 1) {
625 if(__go_file_line(g
->gopc
- 1, &fn
, &file
, &line
)) {
626 runtime_printf("created by %S\n", fn
);
627 runtime_printf("\t%S:%D\n", file
, (int64
) line
);
635 Location locbuf
[100];
640 runtime_tracebackothers(G
* volatile me
)
647 traceback
= runtime_gotraceback();
648 for(gp
= runtime_allg
; gp
!= nil
; gp
= gp
->alllink
) {
649 if(gp
== me
|| gp
->status
== Gdead
)
651 if(gp
->issystem
&& traceback
< 2)
653 runtime_printf("\n");
654 runtime_goroutineheader(gp
);
656 // Our only mechanism for doing a stack trace is
657 // _Unwind_Backtrace. And that only works for the
658 // current thread, not for other random goroutines.
659 // So we need to switch context to the goroutine, get
660 // the backtrace, and then switch back.
662 // This means that if g is running or in a syscall, we
663 // can't reliably print a stack trace. FIXME.
664 if(gp
->status
== Gsyscall
|| gp
->status
== Grunning
) {
665 runtime_printf("no stack trace available\n");
666 runtime_goroutinetrailer(gp
);
672 #ifdef USING_SPLIT_STACK
673 __splitstack_getcontext(&me
->stack_context
[0]);
675 getcontext(&me
->context
);
677 if(gp
->traceback
!= nil
) {
681 runtime_printtrace(tb
.locbuf
, tb
.c
, false);
682 runtime_goroutinetrailer(gp
);
686 // Do a stack trace of gp, and then restore the context to
692 Traceback
* traceback
;
694 traceback
= gp
->traceback
;
696 traceback
->c
= runtime_callers(1, traceback
->locbuf
,
697 sizeof traceback
->locbuf
/ sizeof traceback
->locbuf
[0]);
698 runtime_gogo(traceback
->gp
);
701 // Mark this g as m's idle goroutine.
702 // This functionality might be used in environments where programs
703 // are limited to a single thread, to simulate a select-driven
704 // network server. It is not exposed via the standard runtime API.
706 runtime_idlegoroutine(void)
709 runtime_throw("g is already an idle goroutine");
716 mp
->id
= runtime_sched
.mcount
++;
717 mp
->fastrand
= 0x49f6428aUL
+ mp
->id
+ runtime_cputicks();
719 if(mp
->mcache
== nil
)
720 mp
->mcache
= runtime_allocmcache();
722 runtime_callers(1, mp
->createstack
, nelem(mp
->createstack
));
724 // Add to runtime_allm so garbage collector doesn't free m
725 // when it is just in a register or thread-local storage.
726 mp
->alllink
= runtime_allm
;
727 // runtime_NumCgoCall() iterates over allm w/o schedlock,
728 // so we need to publish it safely.
729 runtime_atomicstorep(&runtime_allm
, mp
);
732 // Try to increment mcpu. Report whether succeeded.
739 v
= runtime_sched
.atomic
;
740 if(atomic_mcpu(v
) >= atomic_mcpumax(v
))
742 if(runtime_cas(&runtime_sched
.atomic
, v
, v
+(1<<mcpuShift
)))
747 // Put on `g' queue. Sched must be locked.
753 // If g is wired, hand it off directly.
754 if((mp
= gp
->lockedm
) != nil
&& canaddmcpu()) {
759 // If g is the idle goroutine for an m, hand it off.
760 if(gp
->idlem
!= nil
) {
761 if(gp
->idlem
->idleg
!= nil
) {
762 runtime_printf("m%d idle out of sync: g%D g%D\n",
764 gp
->idlem
->idleg
->goid
, gp
->goid
);
765 runtime_throw("runtime: double idle");
767 gp
->idlem
->idleg
= gp
;
772 if(runtime_sched
.ghead
== nil
)
773 runtime_sched
.ghead
= gp
;
775 runtime_sched
.gtail
->schedlink
= gp
;
776 runtime_sched
.gtail
= gp
;
779 // if it transitions to nonzero, set atomic gwaiting bit.
780 if(runtime_sched
.gwait
++ == 0)
781 runtime_xadd(&runtime_sched
.atomic
, 1<<gwaitingShift
);
784 // Report whether gget would return something.
788 return runtime_sched
.ghead
!= nil
|| m
->idleg
!= nil
;
791 // Get from `g' queue. Sched must be locked.
797 gp
= runtime_sched
.ghead
;
799 runtime_sched
.ghead
= gp
->schedlink
;
800 if(runtime_sched
.ghead
== nil
)
801 runtime_sched
.gtail
= nil
;
803 // if it transitions to zero, clear atomic gwaiting bit.
804 if(--runtime_sched
.gwait
== 0)
805 runtime_xadd(&runtime_sched
.atomic
, -1<<gwaitingShift
);
806 } else if(m
->idleg
!= nil
) {
813 // Put on `m' list. Sched must be locked.
817 mp
->schedlink
= runtime_sched
.mhead
;
818 runtime_sched
.mhead
= mp
;
819 runtime_sched
.mwait
++;
822 // Get an `m' to run `g'. Sched must be locked.
828 // if g has its own m, use it.
829 if(gp
&& (mp
= gp
->lockedm
) != nil
)
832 // otherwise use general m pool.
833 if((mp
= runtime_sched
.mhead
) != nil
) {
834 runtime_sched
.mhead
= mp
->schedlink
;
835 runtime_sched
.mwait
--;
840 // Mark g ready to run.
849 // Mark g ready to run. Sched is already locked.
850 // G might be running already and about to stop.
851 // The sched lock protects g->status from changing underfoot.
856 // Running on another machine.
857 // Ready it when it stops.
863 if(gp
->status
== Grunnable
|| gp
->status
== Grunning
) {
864 runtime_printf("goroutine %D has status %d\n", gp
->goid
, gp
->status
);
865 runtime_throw("bad g->status in ready");
867 gp
->status
= Grunnable
;
873 // Same as readylocked but a different symbol so that
874 // debuggers can set a breakpoint here and catch all
877 newprocreadylocked(G
*gp
)
882 // Pass g to m for running.
883 // Caller has already incremented mcpu.
887 runtime_sched
.grunning
++;
892 runtime_notewakeup(&mwakeup
->havenextg
);
897 // Get the next goroutine that m should run.
898 // Sched must be locked on entry, is unlocked on exit.
899 // Makes sure that at most $GOMAXPROCS g's are
900 // running on cpus (not in system calls) at any given time.
908 if(atomic_mcpu(runtime_sched
.atomic
) >= maxgomaxprocs
)
909 runtime_throw("negative mcpu");
911 // If there is a g waiting as m->nextg, the mcpu++
912 // happened before it was passed to mnextg.
913 if(m
->nextg
!= nil
) {
920 if(m
->lockedg
!= nil
) {
921 // We can only run one g, and it's not available.
922 // Make sure some other cpu is running to handle
923 // the ordinary run queue.
924 if(runtime_sched
.gwait
!= 0) {
926 // m->lockedg might have been on the queue.
927 if(m
->nextg
!= nil
) {
935 // Look for work on global queue.
936 while(haveg() && canaddmcpu()) {
939 runtime_throw("gget inconsistency");
942 mnextg(gp
->lockedm
, gp
);
945 runtime_sched
.grunning
++;
950 // The while loop ended either because the g queue is empty
951 // or because we have maxed out our m procs running go
952 // code (mcpu >= mcpumax). We need to check that
953 // concurrent actions by entersyscall/exitsyscall cannot
954 // invalidate the decision to end the loop.
956 // We hold the sched lock, so no one else is manipulating the
957 // g queue or changing mcpumax. Entersyscall can decrement
958 // mcpu, but if does so when there is something on the g queue,
959 // the gwait bit will be set, so entersyscall will take the slow path
960 // and use the sched lock. So it cannot invalidate our decision.
962 // Wait on global m queue.
966 // Look for deadlock situation.
967 // There is a race with the scavenger that causes false negatives:
968 // if the scavenger is just starting, then we have
969 // scvg != nil && grunning == 0 && gwait == 0
970 // and we do not detect a deadlock. It is possible that we should
971 // add that case to the if statement here, but it is too close to Go 1
972 // to make such a subtle change. Instead, we work around the
973 // false negative in trivial programs by calling runtime.gosched
974 // from the main goroutine just before main.main.
975 // See runtime_main above.
977 // On a related note, it is also possible that the scvg == nil case is
978 // wrong and should include gwait, but that does not happen in
979 // standard Go programs, which all start the scavenger.
981 if((scvg
== nil
&& runtime_sched
.grunning
== 0) ||
982 (scvg
!= nil
&& runtime_sched
.grunning
== 1 && runtime_sched
.gwait
== 0 &&
983 (scvg
->status
== Grunning
|| scvg
->status
== Gsyscall
))) {
984 m
->throwing
= -1; // do not dump full stacks
985 runtime_throw("all goroutines are asleep - deadlock!");
990 runtime_noteclear(&m
->havenextg
);
992 // Stoptheworld is waiting for all but its cpu to go to stop.
993 // Entersyscall might have decremented mcpu too, but if so
994 // it will see the waitstop and take the slow path.
995 // Exitsyscall never increments mcpu beyond mcpumax.
996 v
= runtime_atomicload(&runtime_sched
.atomic
);
997 if(atomic_waitstop(v
) && atomic_mcpu(v
) <= atomic_mcpumax(v
)) {
998 // set waitstop = 0 (known to be 1)
999 runtime_xadd(&runtime_sched
.atomic
, -1<<waitstopShift
);
1000 runtime_notewakeup(&runtime_sched
.stopped
);
1004 runtime_notesleep(&m
->havenextg
);
1008 runtime_lock(&runtime_sched
);
1011 if((gp
= m
->nextg
) == nil
)
1012 runtime_throw("bad m->nextg in nextgoroutine");
1018 runtime_gcprocs(void)
1022 // Figure out how many CPUs to use during GC.
1023 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
1024 n
= runtime_gomaxprocs
;
1025 if(n
> runtime_ncpu
)
1026 n
= runtime_ncpu
> 0 ? runtime_ncpu
: 1;
1029 if(n
> runtime_sched
.mwait
+1) // one M is currently running
1030 n
= runtime_sched
.mwait
+1;
1035 runtime_helpgc(int32 nproc
)
1040 runtime_lock(&runtime_sched
);
1041 for(n
= 1; n
< nproc
; n
++) { // one M is currently running
1044 runtime_throw("runtime_gcprocs inconsistency");
1047 runtime_notewakeup(&mp
->havenextg
);
1049 runtime_unlock(&runtime_sched
);
1053 runtime_stoptheworld(void)
1058 runtime_gcwaiting
= 1;
1064 v
= runtime_sched
.atomic
;
1065 if(atomic_mcpu(v
) <= 1)
1068 // It would be unsafe for multiple threads to be using
1069 // the stopped note at once, but there is only
1070 // ever one thread doing garbage collection.
1071 runtime_noteclear(&runtime_sched
.stopped
);
1072 if(atomic_waitstop(v
))
1073 runtime_throw("invalid waitstop");
1075 // atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
1076 // still being true.
1077 if(!runtime_cas(&runtime_sched
.atomic
, v
, v
+(1<<waitstopShift
)))
1081 runtime_notesleep(&runtime_sched
.stopped
);
1084 runtime_singleproc
= runtime_gomaxprocs
== 1;
1089 runtime_starttheworld(void)
1094 // Figure out how many CPUs GC could possibly use.
1095 max
= runtime_gomaxprocs
;
1096 if(max
> runtime_ncpu
)
1097 max
= runtime_ncpu
> 0 ? runtime_ncpu
: 1;
1102 runtime_gcwaiting
= 0;
1103 setmcpumax(runtime_gomaxprocs
);
1105 if(runtime_gcprocs() < max
&& canaddmcpu()) {
1106 // If GC could have used another helper proc, start one now,
1107 // in the hope that it will be available next time.
1108 // It would have been even better to start it before the collection,
1109 // but doing so requires allocating memory, so it's tricky to
1110 // coordinate. This lazy approach works out in practice:
1111 // we don't mind if the first couple gc rounds don't have quite
1112 // the maximum number of procs.
1113 // canaddmcpu above did mcpu++
1114 // (necessary, because m will be doing various
1115 // initialization work so is definitely running),
1116 // but m is not running a specific goroutine,
1117 // so set the helpgc flag as a signal to m's
1118 // first schedule(nil) to mcpu-- and grunning--.
1119 mp
= runtime_newm();
1121 runtime_sched
.grunning
++;
1126 // Called to start an M.
1128 runtime_mstart(void* mp
)
1138 // Record top of stack for use by mcall.
1139 // Once we call schedule we're never coming back,
1140 // so other calls can reuse this stack space.
1141 #ifdef USING_SPLIT_STACK
1142 __splitstack_getcontext(&g
->stack_context
[0]);
1144 g
->gcinitial_sp
= &mp
;
1145 // Setting gcstack_size to 0 is a marker meaning that gcinitial_sp
1146 // is the top of the stack, not the bottom.
1147 g
->gcstack_size
= 0;
1150 getcontext(&g
->context
);
1152 if(g
->entry
!= nil
) {
1153 // Got here from mcall.
1154 void (*pfn
)(G
*) = (void (*)(G
*))g
->entry
;
1155 G
* gp
= (G
*)g
->param
;
1161 #ifdef USING_SPLIT_STACK
1163 int dont_block_signals
= 0;
1164 __splitstack_block_signals(&dont_block_signals
, nil
);
1168 // Install signal handlers; after minit so that minit can
1169 // prepare the thread to be able to handle the signals.
1170 if(m
== &runtime_m0
)
1175 // TODO(brainman): This point is never reached, because scheduler
1176 // does not release os threads at the moment. But once this path
1177 // is enabled, we must remove our seh here.
1182 typedef struct CgoThreadStart CgoThreadStart
;
1183 struct CgoThreadStart
1190 // Kick off new m's as needed (up to mcpumax).
1198 if(m
->mallocing
|| m
->gcing
)
1201 while(haveg() && canaddmcpu()) {
1204 runtime_throw("gget inconsistency");
1206 // Find the m that will run gp.
1207 if((mp
= mget(gp
)) == nil
)
1208 mp
= runtime_newm();
1213 // Create a new m. It will start off with a call to runtime_mstart.
1218 pthread_attr_t attr
;
1226 static const Type
*mtype
; // The Go type M
1229 runtime_gc_m_ptr(&e
);
1230 mtype
= ((const PtrType
*)e
.__type_descriptor
)->__element_type
;
1234 mp
= runtime_mal(sizeof *mp
);
1236 mp
->g0
= runtime_malg(-1, nil
, nil
);
1238 if(pthread_attr_init(&attr
) != 0)
1239 runtime_throw("pthread_attr_init");
1240 if(pthread_attr_setdetachstate(&attr
, PTHREAD_CREATE_DETACHED
) != 0)
1241 runtime_throw("pthread_attr_setdetachstate");
1243 stacksize
= PTHREAD_STACK_MIN
;
1245 // With glibc before version 2.16 the static TLS size is taken
1246 // out of the stack size, and we get an error or a crash if
1247 // there is not enough stack space left. Add it back in if we
1248 // can, in case the program uses a lot of TLS space. FIXME:
1249 // This can be disabled in glibc 2.16 and later, if the bug is
1250 // indeed fixed then.
1251 stacksize
+= tlssize
;
1253 if(pthread_attr_setstacksize(&attr
, stacksize
) != 0)
1254 runtime_throw("pthread_attr_setstacksize");
1256 // Block signals during pthread_create so that the new thread
1257 // starts with signals disabled. It will enable them in minit.
1260 sigprocmask(SIG_BLOCK
, &clear
, &old
);
1261 ret
= pthread_create(&tid
, &attr
, runtime_mstart
, mp
);
1262 sigprocmask(SIG_SETMASK
, &old
, nil
);
1265 runtime_throw("pthread_create");
1270 // One round of scheduler: find a goroutine and run it.
1271 // The argument is the goroutine that was running before
1272 // schedule was called, or nil if this is the first call.
1282 // Just finished running gp.
1284 runtime_sched
.grunning
--;
1286 // atomic { mcpu-- }
1287 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1288 if(atomic_mcpu(v
) > maxgomaxprocs
)
1289 runtime_throw("negative mcpu in scheduler");
1291 switch(gp
->status
) {
1294 // Shouldn't have been running!
1295 runtime_throw("bad gp->status in sched");
1297 gp
->status
= Grunnable
;
1302 runtime_racegoend(gp
->goid
);
1309 runtime_memclr(&gp
->context
, sizeof gp
->context
);
1311 if(--runtime_sched
.gcount
== 0)
1315 if(gp
->readyonstop
) {
1316 gp
->readyonstop
= 0;
1319 } else if(m
->helpgc
) {
1320 // Bootstrap m or new m started by starttheworld.
1321 // atomic { mcpu-- }
1322 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1323 if(atomic_mcpu(v
) > maxgomaxprocs
)
1324 runtime_throw("negative mcpu in scheduler");
1325 // Compensate for increment in starttheworld().
1326 runtime_sched
.grunning
--;
1328 } else if(m
->nextg
!= nil
) {
1329 // New m started by matchmg.
1331 runtime_throw("invalid m state in scheduler");
1334 // Find (or wait for) g to run. Unlocks runtime_sched.
1335 gp
= nextgandunlock();
1336 gp
->readyonstop
= 0;
1337 gp
->status
= Grunning
;
1341 // Check whether the profiler needs to be turned on or off.
1342 hz
= runtime_sched
.profilehz
;
1343 if(m
->profilehz
!= hz
)
1344 runtime_resetcpuprofiler(hz
);
1349 // Enter scheduler. If g->status is Grunning,
1350 // re-queues g and runs everyone else who is waiting
1351 // before running g again. If g->status is Gmoribund,
1354 runtime_gosched(void)
1357 runtime_throw("gosched holding locks");
1359 runtime_throw("gosched of g0");
1360 runtime_mcall(schedule
);
1363 // Puts the current goroutine into a waiting state and unlocks the lock.
1364 // The goroutine can be made runnable again by calling runtime_ready(gp).
1366 runtime_park(void (*unlockf
)(Lock
*), Lock
*lock
, const char *reason
)
1368 g
->status
= Gwaiting
;
1369 g
->waitreason
= reason
;
1375 // The goroutine g is about to enter a system call.
1376 // Record that it's not using the cpu anymore.
1377 // This is called only from the go syscall library and cgocall,
1378 // not from the low-level system calls used by the runtime.
1380 // Entersyscall cannot split the stack: the runtime_gosave must
1381 // make g->sched refer to the caller's stack segment, because
1382 // entersyscall is going to return immediately after.
1383 // It's okay to call matchmg and notewakeup even after
1384 // decrementing mcpu, because we haven't released the
1385 // sched lock yet, so the garbage collector cannot be running.
1387 void runtime_entersyscall(void) __attribute__ ((no_split_stack
));
1390 runtime_entersyscall(void)
1394 if(m
->profilehz
> 0)
1395 runtime_setprof(false);
1397 // Leave SP around for gc and traceback.
1398 #ifdef USING_SPLIT_STACK
1399 g
->gcstack
= __splitstack_find(nil
, nil
, &g
->gcstack_size
,
1400 &g
->gcnext_segment
, &g
->gcnext_sp
,
1403 g
->gcnext_sp
= (byte
*) &v
;
1406 // Save the registers in the g structure so that any pointers
1407 // held in registers will be seen by the garbage collector.
1408 getcontext(&g
->gcregs
);
1410 g
->status
= Gsyscall
;
1413 // The slow path inside the schedlock/schedunlock will get
1414 // through without stopping if it does:
1417 // waitstop && mcpu <= mcpumax not true
1418 // If we can do the same with a single atomic add,
1419 // then we can skip the locks.
1420 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1421 if(!atomic_gwaiting(v
) && (!atomic_waitstop(v
) || atomic_mcpu(v
) > atomic_mcpumax(v
)))
1425 v
= runtime_atomicload(&runtime_sched
.atomic
);
1426 if(atomic_gwaiting(v
)) {
1428 v
= runtime_atomicload(&runtime_sched
.atomic
);
1430 if(atomic_waitstop(v
) && atomic_mcpu(v
) <= atomic_mcpumax(v
)) {
1431 runtime_xadd(&runtime_sched
.atomic
, -1<<waitstopShift
);
1432 runtime_notewakeup(&runtime_sched
.stopped
);
1438 // The goroutine g exited its system call.
1439 // Arrange for it to run on a cpu again.
1440 // This is called only from the go syscall library, not
1441 // from the low-level system calls used by the runtime.
1443 runtime_exitsyscall(void)
1449 // If we can do the mcpu++ bookkeeping and
1450 // find that we still have mcpu <= mcpumax, then we can
1451 // start executing Go code immediately, without having to
1452 // schedlock/schedunlock.
1453 // Also do fast return if any locks are held, so that
1454 // panic code can use syscalls to open a file.
1456 v
= runtime_xadd(&runtime_sched
.atomic
, (1<<mcpuShift
));
1457 if((m
->profilehz
== runtime_sched
.profilehz
&& atomic_mcpu(v
) <= atomic_mcpumax(v
)) || m
->locks
> 0) {
1458 // There's a cpu for us, so we can run.
1459 gp
->status
= Grunning
;
1460 // Garbage collector isn't running (since we are),
1461 // so okay to clear gcstack.
1462 #ifdef USING_SPLIT_STACK
1465 gp
->gcnext_sp
= nil
;
1466 runtime_memclr(&gp
->gcregs
, sizeof gp
->gcregs
);
1468 if(m
->profilehz
> 0)
1469 runtime_setprof(true);
1473 // Tell scheduler to put g back on the run queue:
1474 // mostly equivalent to g->status = Grunning,
1475 // but keeps the garbage collector from thinking
1476 // that g is running right now, which it's not.
1477 gp
->readyonstop
= 1;
1479 // All the cpus are taken.
1480 // The scheduler will ready g and put this m to sleep.
1481 // When the scheduler takes g away from m,
1482 // it will undo the runtime_sched.mcpu++ above.
1485 // Gosched returned, so we're allowed to run now.
1486 // Delete the gcstack information that we left for
1487 // the garbage collector during the system call.
1488 // Must wait until now because until gosched returns
1489 // we don't know for sure that the garbage collector
1491 #ifdef USING_SPLIT_STACK
1494 gp
->gcnext_sp
= nil
;
1495 runtime_memclr(&gp
->gcregs
, sizeof gp
->gcregs
);
1498 // Allocate a new g, with a stack big enough for stacksize bytes.
1500 runtime_malg(int32 stacksize
, byte
** ret_stack
, size_t* ret_stacksize
)
1504 newg
= runtime_malloc(sizeof(G
));
1505 if(stacksize
>= 0) {
1506 #if USING_SPLIT_STACK
1507 int dont_block_signals
= 0;
1509 *ret_stack
= __splitstack_makecontext(stacksize
,
1510 &newg
->stack_context
[0],
1512 __splitstack_block_signals_context(&newg
->stack_context
[0],
1513 &dont_block_signals
, nil
);
1515 *ret_stack
= runtime_mallocgc(stacksize
, FlagNoProfiling
|FlagNoGC
, 0, 0);
1516 *ret_stacksize
= stacksize
;
1517 newg
->gcinitial_sp
= *ret_stack
;
1518 newg
->gcstack_size
= stacksize
;
1519 runtime_xadd(&runtime_stacks_sys
, stacksize
);
1525 /* For runtime package testing. */
1527 void runtime_testing_entersyscall(void)
1528 __asm__ (GOSYM_PREFIX
"runtime.entersyscall");
1531 runtime_testing_entersyscall()
1533 runtime_entersyscall();
1536 void runtime_testing_exitsyscall(void)
1537 __asm__ (GOSYM_PREFIX
"runtime.exitsyscall");
1540 runtime_testing_exitsyscall()
1542 runtime_exitsyscall();
1546 __go_go(void (*fn
)(void*), void* arg
)
1553 goid
= runtime_xadd64((uint64
*)&runtime_sched
.goidgen
, 1);
1555 runtime_racegostart(goid
, runtime_getcallerpc(&fn
));
1559 if((newg
= gfget()) != nil
) {
1560 #ifdef USING_SPLIT_STACK
1561 int dont_block_signals
= 0;
1563 sp
= __splitstack_resetcontext(&newg
->stack_context
[0],
1565 __splitstack_block_signals_context(&newg
->stack_context
[0],
1566 &dont_block_signals
, nil
);
1568 sp
= newg
->gcinitial_sp
;
1569 spsize
= newg
->gcstack_size
;
1571 runtime_throw("bad spsize in __go_go");
1572 newg
->gcnext_sp
= sp
;
1575 newg
= runtime_malg(StackMin
, &sp
, &spsize
);
1576 if(runtime_lastg
== nil
)
1577 runtime_allg
= newg
;
1579 runtime_lastg
->alllink
= newg
;
1580 runtime_lastg
= newg
;
1582 newg
->status
= Gwaiting
;
1583 newg
->waitreason
= "new goroutine";
1585 newg
->entry
= (byte
*)fn
;
1587 newg
->gopc
= (uintptr
)__builtin_return_address(0);
1589 runtime_sched
.gcount
++;
1593 runtime_throw("nil g->stack0");
1596 // Avoid warnings about variables clobbered by
1598 byte
* volatile vsp
= sp
;
1599 size_t volatile vspsize
= spsize
;
1600 G
* volatile vnewg
= newg
;
1602 getcontext(&vnewg
->context
);
1603 vnewg
->context
.uc_stack
.ss_sp
= vsp
;
1604 #ifdef MAKECONTEXT_STACK_TOP
1605 vnewg
->context
.uc_stack
.ss_sp
+= vspsize
;
1607 vnewg
->context
.uc_stack
.ss_size
= vspsize
;
1608 makecontext(&vnewg
->context
, kickoff
, 0);
1610 newprocreadylocked(vnewg
);
1617 // Put on gfree list. Sched must be locked.
1621 gp
->schedlink
= runtime_sched
.gfree
;
1622 runtime_sched
.gfree
= gp
;
1625 // Get from gfree list. Sched must be locked.
1631 gp
= runtime_sched
.gfree
;
1633 runtime_sched
.gfree
= gp
->schedlink
;
1637 void runtime_Gosched (void) __asm__ (GOSYM_PREFIX
"runtime.Gosched");
1640 runtime_Gosched(void)
1645 // Implementation of runtime.GOMAXPROCS.
1646 // delete when scheduler is stronger
1648 runtime_gomaxprocsfunc(int32 n
)
1654 ret
= runtime_gomaxprocs
;
1657 if(n
> maxgomaxprocs
)
1659 runtime_gomaxprocs
= n
;
1660 if(runtime_gomaxprocs
> 1)
1661 runtime_singleproc
= false;
1662 if(runtime_gcwaiting
!= 0) {
1663 if(atomic_mcpumax(runtime_sched
.atomic
) != 1)
1664 runtime_throw("invalid mcpumax during gc");
1671 // If there are now fewer allowed procs
1672 // than procs running, stop.
1673 v
= runtime_atomicload(&runtime_sched
.atomic
);
1674 if((int32
)atomic_mcpu(v
) > n
) {
1679 // handle more procs
1686 runtime_LockOSThread(void)
1688 if(m
== &runtime_m0
&& runtime_sched
.init
) {
1689 runtime_sched
.lockmain
= true;
1697 runtime_UnlockOSThread(void)
1699 if(m
== &runtime_m0
&& runtime_sched
.init
) {
1700 runtime_sched
.lockmain
= false;
1708 runtime_lockedOSThread(void)
1710 return g
->lockedm
!= nil
&& m
->lockedg
!= nil
;
1713 // for testing of callbacks
1715 _Bool
runtime_golockedOSThread(void)
1716 __asm__ (GOSYM_PREFIX
"runtime.golockedOSThread");
1719 runtime_golockedOSThread(void)
1721 return runtime_lockedOSThread();
1724 // for testing of wire, unwire
1731 intgo
runtime_NumGoroutine (void)
1732 __asm__ (GOSYM_PREFIX
"runtime.NumGoroutine");
1735 runtime_NumGoroutine()
1737 return runtime_sched
.gcount
;
1741 runtime_gcount(void)
1743 return runtime_sched
.gcount
;
1747 runtime_mcount(void)
1749 return runtime_sched
.mcount
;
1754 void (*fn
)(uintptr
*, int32
);
1757 Location locbuf
[100];
1760 // Called if we receive a SIGPROF signal.
1766 if(prof
.fn
== nil
|| prof
.hz
== 0)
1769 runtime_lock(&prof
);
1770 if(prof
.fn
== nil
) {
1771 runtime_unlock(&prof
);
1774 n
= runtime_callers(0, prof
.locbuf
, nelem(prof
.locbuf
));
1775 for(i
= 0; i
< n
; i
++)
1776 prof
.pcbuf
[i
] = prof
.locbuf
[i
].pc
;
1778 prof
.fn(prof
.pcbuf
, n
);
1779 runtime_unlock(&prof
);
1782 // Arrange to call fn with a traceback hz times a second.
1784 runtime_setcpuprofilerate(void (*fn
)(uintptr
*, int32
), int32 hz
)
1786 // Force sane arguments.
1794 // Stop profiler on this cpu so that it is safe to lock prof.
1795 // if a profiling signal came in while we had prof locked,
1796 // it would deadlock.
1797 runtime_resetcpuprofiler(0);
1799 runtime_lock(&prof
);
1802 runtime_unlock(&prof
);
1803 runtime_lock(&runtime_sched
);
1804 runtime_sched
.profilehz
= hz
;
1805 runtime_unlock(&runtime_sched
);
1808 runtime_resetcpuprofiler(hz
);