1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 #ifdef HAVE_DL_ITERATE_PHDR
23 #ifdef USING_SPLIT_STACK
25 /* FIXME: These are not declared anywhere. */
27 extern void __splitstack_getcontext(void *context
[10]);
29 extern void __splitstack_setcontext(void *context
[10]);
31 extern void *__splitstack_makecontext(size_t, void *context
[10], size_t *);
33 extern void * __splitstack_resetcontext(void *context
[10], size_t *);
35 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
38 extern void __splitstack_block_signals (int *, int *);
40 extern void __splitstack_block_signals_context (void *context
[10], int *,
45 #ifndef PTHREAD_STACK_MIN
46 # define PTHREAD_STACK_MIN 8192
49 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
50 # define StackMin PTHREAD_STACK_MIN
52 # define StackMin ((sizeof(char *) < 8) ? 2 * 1024 * 1024 : 4 * 1024 * 1024)
55 uintptr runtime_stacks_sys
;
58 __asm__(GOSYM_PREFIX
"runtime.gtraceback");
66 #ifndef SETCONTEXT_CLOBBERS_TLS
74 fixcontext(ucontext_t
*c
__attribute__ ((unused
)))
80 # if defined(__x86_64__) && defined(__sun__)
82 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
83 // register to that of the thread which called getcontext. The effect
84 // is that the address of all __thread variables changes. This bug
85 // also affects pthread_self() and pthread_getspecific. We work
86 // around it by clobbering the context field directly to keep %fs the
89 static __thread greg_t fs
;
97 fs
= c
.uc_mcontext
.gregs
[REG_FSBASE
];
101 fixcontext(ucontext_t
* c
)
103 c
->uc_mcontext
.gregs
[REG_FSBASE
] = fs
;
106 # elif defined(__NetBSD__)
108 // NetBSD has a bug: setcontext clobbers tlsbase, we need to save
109 // and restore it ourselves.
111 static __thread __greg_t tlsbase
;
119 tlsbase
= c
.uc_mcontext
._mc_tlsbase
;
123 fixcontext(ucontext_t
* c
)
125 c
->uc_mcontext
._mc_tlsbase
= tlsbase
;
128 # elif defined(__sparc__)
136 fixcontext(ucontext_t
*c
)
139 register unsigned long thread __asm__("%g7");
140 c->uc_mcontext.gregs[REG_G7] = thread;
142 error: variable ‘thread’ might be clobbered by \
143 ‘longjmp’ or ‘vfork’ [-Werror=clobbered]
144 which ought to be false, as %g7 is a fixed register. */
146 if (sizeof (c
->uc_mcontext
.gregs
[REG_G7
]) == 8)
147 asm ("stx %%g7, %0" : "=m"(c
->uc_mcontext
.gregs
[REG_G7
]));
149 asm ("st %%g7, %0" : "=m"(c
->uc_mcontext
.gregs
[REG_G7
]));
160 fixcontext(ucontext_t
* c
)
162 // Thread pointer is in r13, per 64-bit ABI.
163 if (sizeof (c
->uc_mcontext
.jmp_context
.gpr
[13]) == 8)
164 asm ("std 13, %0" : "=m"(c
->uc_mcontext
.jmp_context
.gpr
[13]));
169 # error unknown case for SETCONTEXT_CLOBBERS_TLS
175 // ucontext_arg returns a properly aligned ucontext_t value. On some
176 // systems a ucontext_t value must be aligned to a 16-byte boundary.
177 // The g structure that has fields of type ucontext_t is defined in
178 // Go, and Go has no simple way to align a field to such a boundary.
179 // So we make the field larger in runtime2.go and pick an appropriate
180 // offset within the field here.
182 ucontext_arg(uintptr_t* go_ucontext
)
184 uintptr_t p
= (uintptr_t)go_ucontext
;
185 size_t align
= __alignof__(ucontext_t
);
187 // We only ensured space for up to a 16 byte alignment
188 // in libgo/go/runtime/runtime2.go.
189 runtime_throw("required alignment of ucontext_t too large");
191 p
= (p
+ align
- 1) &~ (uintptr_t)(align
- 1);
192 return (ucontext_t
*)p
;
195 // We can not always refer to the TLS variables directly. The
196 // compiler will call tls_get_addr to get the address of the variable,
197 // and it may hold it in a register across a call to schedule. When
198 // we get back from the call we may be running in a different thread,
199 // in which case the register now points to the TLS variable for a
200 // different thread. We use non-inlinable functions to avoid this
203 G
* runtime_g(void) __attribute__ ((noinline
, no_split_stack
));
211 M
* runtime_m(void) __attribute__ ((noinline
, no_split_stack
));
228 void runtime_newosproc(M
*)
229 __asm__(GOSYM_PREFIX
"runtime.newosproc");
231 // Start a new thread.
233 runtime_newosproc(M
*mp
)
241 if(pthread_attr_init(&attr
) != 0)
242 runtime_throw("pthread_attr_init");
243 if(pthread_attr_setdetachstate(&attr
, PTHREAD_CREATE_DETACHED
) != 0)
244 runtime_throw("pthread_attr_setdetachstate");
246 // Block signals during pthread_create so that the new thread
247 // starts with signals disabled. It will enable them in minit.
251 // Blocking SIGTRAP reportedly breaks gdb on Alpha GNU/Linux.
252 sigdelset(&clear
, SIGTRAP
);
256 pthread_sigmask(SIG_BLOCK
, &clear
, &old
);
258 for (tries
= 0; tries
< 20; tries
++) {
259 ret
= pthread_create(&tid
, &attr
, runtime_mstart
, mp
);
263 runtime_usleep((tries
+ 1) * 1000); // Milliseconds.
266 pthread_sigmask(SIG_SETMASK
, &old
, nil
);
269 runtime_printf("pthread_create failed: %d\n", ret
);
270 runtime_throw("pthread_create");
274 // Switch context to a different goroutine. This is like longjmp.
275 void runtime_gogo(G
*) __attribute__ ((noinline
));
277 runtime_gogo(G
* newg
)
279 #ifdef USING_SPLIT_STACK
280 __splitstack_setcontext((void*)(&newg
->stackcontext
[0]));
283 newg
->fromgogo
= true;
284 fixcontext(ucontext_arg(&newg
->context
[0]));
285 setcontext(ucontext_arg(&newg
->context
[0]));
286 runtime_throw("gogo setcontext returned");
289 // Save context and call fn passing g as a parameter. This is like
290 // setjmp. Because getcontext always returns 0, unlike setjmp, we use
291 // g->fromgogo as a code. It will be true if we got here via
292 // setcontext. g == nil the first time this is called in a new m.
293 void runtime_mcall(FuncVal
*) __attribute__ ((noinline
));
295 runtime_mcall(FuncVal
*fv
)
299 #ifndef USING_SPLIT_STACK
303 // Ensure that all registers are on the stack for the garbage
305 __builtin_unwind_init();
310 runtime_throw("runtime: mcall called on m->g0 stack");
314 #ifdef USING_SPLIT_STACK
315 __splitstack_getcontext((void*)(&g
->stackcontext
[0]));
317 // We have to point to an address on the stack that is
318 // below the saved registers.
319 gp
->gcnextsp
= (uintptr
)(&afterregs
);
321 gp
->fromgogo
= false;
322 getcontext(ucontext_arg(&gp
->context
[0]));
324 // When we return from getcontext, we may be running
325 // in a new thread. That means that g may have
326 // changed. It is a global variables so we will
327 // reload it, but the address of g may be cached in
328 // our local stack frame, and that address may be
329 // wrong. Call the function to reload the value for
334 if(gp
->traceback
!= nil
)
337 if (gp
== nil
|| !gp
->fromgogo
) {
338 #ifdef USING_SPLIT_STACK
339 __splitstack_setcontext((void*)(&mp
->g0
->stackcontext
[0]));
344 // It's OK to set g directly here because this case
345 // can not occur if we got here via a setcontext to
346 // the getcontext call just above.
349 fixcontext(ucontext_arg(&mp
->g0
->context
[0]));
350 setcontext(ucontext_arg(&mp
->g0
->context
[0]));
351 runtime_throw("runtime: mcall function returned");
355 // Goroutine scheduler
356 // The scheduler's job is to distribute ready-to-run goroutines over worker threads.
358 // The main concepts are:
360 // M - worker thread, or machine.
361 // P - processor, a resource that is required to execute Go code.
362 // M must have an associated P to execute Go code, however it can be
363 // blocked or in a syscall w/o an associated P.
365 // Design doc at http://golang.org/s/go11sched.
367 extern bool* runtime_getCgoHasExtraM()
368 __asm__ (GOSYM_PREFIX
"runtime.getCgoHasExtraM");
369 extern G
* allocg(void)
370 __asm__ (GOSYM_PREFIX
"runtime.allocg");
372 Sched
* runtime_sched
;
374 bool runtime_isarchive
;
376 extern void kickoff(void)
377 __asm__(GOSYM_PREFIX
"runtime.kickoff");
378 extern void minit(void)
379 __asm__(GOSYM_PREFIX
"runtime.minit");
380 extern void mstart1(void)
381 __asm__(GOSYM_PREFIX
"runtime.mstart1");
382 extern void stopm(void)
383 __asm__(GOSYM_PREFIX
"runtime.stopm");
384 extern void handoffp(P
*)
385 __asm__(GOSYM_PREFIX
"runtime.handoffp");
386 extern void wakep(void)
387 __asm__(GOSYM_PREFIX
"runtime.wakep");
388 extern void stoplockedm(void)
389 __asm__(GOSYM_PREFIX
"runtime.stoplockedm");
390 extern void schedule(void)
391 __asm__(GOSYM_PREFIX
"runtime.schedule");
392 extern void execute(G
*, bool)
393 __asm__(GOSYM_PREFIX
"runtime.execute");
394 extern void reentersyscall(uintptr
, uintptr
)
395 __asm__(GOSYM_PREFIX
"runtime.reentersyscall");
396 extern void reentersyscallblock(uintptr
, uintptr
)
397 __asm__(GOSYM_PREFIX
"runtime.reentersyscallblock");
399 __asm__(GOSYM_PREFIX
"runtime.gfget");
400 extern void acquirep(P
*)
401 __asm__(GOSYM_PREFIX
"runtime.acquirep");
402 extern P
* releasep(void)
403 __asm__(GOSYM_PREFIX
"runtime.releasep");
404 extern void incidlelocked(int32
)
405 __asm__(GOSYM_PREFIX
"runtime.incidlelocked");
406 extern void globrunqput(G
*)
407 __asm__(GOSYM_PREFIX
"runtime.globrunqput");
408 extern P
* pidleget(void)
409 __asm__(GOSYM_PREFIX
"runtime.pidleget");
410 extern struct mstats
* getMemstats(void)
411 __asm__(GOSYM_PREFIX
"runtime.getMemstats");
413 bool runtime_isstarted
;
415 // Used to determine the field alignment.
423 void getTraceback(G
*, G
*) __asm__(GOSYM_PREFIX
"runtime.getTraceback");
425 // getTraceback stores a traceback of gp in the g's traceback field
426 // and then returns to me. We expect that gp's traceback is not nil.
427 // It works by saving me's current context, and checking gp's traceback field.
428 // If gp's traceback field is not nil, it starts running gp.
429 // In places where we call getcontext, we check the traceback field.
430 // If it is not nil, we collect a traceback, and then return to the
431 // goroutine stored in the traceback field, which is me.
432 void getTraceback(G
* me
, G
* gp
)
434 #ifdef USING_SPLIT_STACK
435 __splitstack_getcontext((void*)(&me
->stackcontext
[0]));
437 getcontext(ucontext_arg(&me
->context
[0]));
439 if (gp
->traceback
!= nil
) {
444 // Do a stack trace of gp, and then restore the context to
445 // gp->traceback->gp.
450 Traceback
* traceback
;
453 traceback
= gp
->traceback
;
456 if(holdm
!= nil
&& holdm
!= g
->m
)
457 runtime_throw("gtraceback: m is not nil");
458 gp
->m
= traceback
->gp
->m
;
459 traceback
->c
= runtime_callers(1, traceback
->locbuf
,
460 sizeof traceback
->locbuf
/ sizeof traceback
->locbuf
[0], false);
462 runtime_gogo(traceback
->gp
);
465 // Called by pthread_create to start an M.
467 runtime_mstart(void *arg
)
481 // We have to call minit before we call getcontext,
482 // because getcontext will copy the signal mask.
487 // Record top of stack for use by mcall.
488 // Once we call schedule we're never coming back,
489 // so other calls can reuse this stack space.
490 #ifdef USING_SPLIT_STACK
491 __splitstack_getcontext((void*)(&gp
->stackcontext
[0]));
493 gp
->gcinitialsp
= &arg
;
494 // Setting gcstacksize to 0 is a marker meaning that gcinitialsp
495 // is the top of the stack, not the bottom.
497 gp
->gcnextsp
= (uintptr
)(&arg
);
500 // Save the currently active context. This will return
501 // multiple times via the setcontext call in mcall.
502 getcontext(ucontext_arg(&gp
->context
[0]));
504 if(gp
->traceback
!= nil
) {
505 // Got here from getTraceback.
506 // I'm not sure this ever actually happens--getTraceback
507 // may always go to the getcontext call in mcall.
511 if(gp
->entry
!= nil
) {
512 // Got here from mcall.
513 FuncVal
*fv
= gp
->entry
;
514 void (*pfn
)(G
*) = (void (*)(G
*))fv
->fn
;
515 G
* gp1
= (G
*)gp
->param
;
518 __builtin_call_with_static_chain(pfn(gp1
), fv
);
522 // Initial call to getcontext--starting thread.
524 #ifdef USING_SPLIT_STACK
526 int dont_block_signals
= 0;
527 __splitstack_block_signals(&dont_block_signals
, nil
);
533 // mstart1 does not return, but we need a return statement
534 // here to avoid a compiler warning.
538 typedef struct CgoThreadStart CgoThreadStart
;
539 struct CgoThreadStart
547 void setGContext(void) __asm__ (GOSYM_PREFIX
"runtime.setGContext");
549 // setGContext sets up a new goroutine context for the current g.
560 #ifdef USING_SPLIT_STACK
561 __splitstack_getcontext((void*)(&gp
->stackcontext
[0]));
563 __splitstack_block_signals(&val
, nil
);
565 gp
->gcinitialsp
= &val
;
568 gp
->gcnextsp
= (uintptr
)(&val
);
570 getcontext(ucontext_arg(&gp
->context
[0]));
572 if(gp
->entry
!= nil
) {
573 // Got here from mcall.
574 FuncVal
*fv
= gp
->entry
;
575 void (*pfn
)(G
*) = (void (*)(G
*))fv
->fn
;
576 G
* gp1
= (G
*)gp
->param
;
579 __builtin_call_with_static_chain(pfn(gp1
), fv
);
584 void makeGContext(G
*, byte
*, uintptr
)
585 __asm__(GOSYM_PREFIX
"runtime.makeGContext");
587 // makeGContext makes a new context for a g.
589 makeGContext(G
* gp
, byte
* sp
, uintptr spsize
) {
592 uc
= ucontext_arg(&gp
->context
[0]);
594 uc
->uc_stack
.ss_sp
= sp
;
595 uc
->uc_stack
.ss_size
= (size_t)spsize
;
596 makecontext(uc
, kickoff
, 0);
599 // The goroutine g is about to enter a system call.
600 // Record that it's not using the cpu anymore.
601 // This is called only from the go syscall library and cgocall,
602 // not from the low-level system calls used by the runtime.
604 // Entersyscall cannot split the stack: the runtime_gosave must
605 // make g->sched refer to the caller's stack segment, because
606 // entersyscall is going to return immediately after.
608 void runtime_entersyscall(int32
) __attribute__ ((no_split_stack
));
609 static void doentersyscall(uintptr
, uintptr
)
610 __attribute__ ((no_split_stack
, noinline
));
613 runtime_entersyscall(int32 dummy
__attribute__ ((unused
)))
615 // Save the registers in the g structure so that any pointers
616 // held in registers will be seen by the garbage collector.
617 getcontext(ucontext_arg(&g
->gcregs
[0]));
619 // Note that if this function does save any registers itself,
620 // we might store the wrong value in the call to getcontext.
621 // FIXME: This assumes that we do not need to save any
622 // callee-saved registers to access the TLS variable g. We
623 // don't want to put the ucontext_t on the stack because it is
624 // large and we can not split the stack here.
625 doentersyscall((uintptr
)runtime_getcallerpc(&dummy
),
626 (uintptr
)runtime_getcallersp(&dummy
));
630 doentersyscall(uintptr pc
, uintptr sp
)
632 // Leave SP around for GC and traceback.
633 #ifdef USING_SPLIT_STACK
636 g
->gcstack
= (uintptr
)(__splitstack_find(nil
, nil
, &gcstacksize
,
637 (void**)(&g
->gcnextsegment
),
638 (void**)(&g
->gcnextsp
),
640 g
->gcstacksize
= (uintptr
)gcstacksize
;
646 g
->gcnextsp
= (uintptr
)(&v
);
650 reentersyscall(pc
, sp
);
653 static void doentersyscallblock(uintptr
, uintptr
)
654 __attribute__ ((no_split_stack
, noinline
));
656 // The same as runtime_entersyscall(), but with a hint that the syscall is blocking.
658 runtime_entersyscallblock(int32 dummy
__attribute__ ((unused
)))
660 // Save the registers in the g structure so that any pointers
661 // held in registers will be seen by the garbage collector.
662 getcontext(ucontext_arg(&g
->gcregs
[0]));
664 // See comment in runtime_entersyscall.
665 doentersyscallblock((uintptr
)runtime_getcallerpc(&dummy
),
666 (uintptr
)runtime_getcallersp(&dummy
));
670 doentersyscallblock(uintptr pc
, uintptr sp
)
672 // Leave SP around for GC and traceback.
673 #ifdef USING_SPLIT_STACK
676 g
->gcstack
= (uintptr
)(__splitstack_find(nil
, nil
, &gcstacksize
,
677 (void**)(&g
->gcnextsegment
),
678 (void**)(&g
->gcnextsp
),
680 g
->gcstacksize
= (uintptr
)gcstacksize
;
686 g
->gcnextsp
= (uintptr
)(&v
);
690 reentersyscallblock(pc
, sp
);
693 // Allocate a new g, with a stack big enough for stacksize bytes.
695 runtime_malg(bool allocatestack
, bool signalstack
, byte
** ret_stack
, uintptr
* ret_stacksize
)
700 uintptr unused_stacksize
;
701 #if USING_SPLIT_STACK
702 int dont_block_signals
= 0;
706 if (ret_stack
== nil
) {
707 ret_stack
= &unused_stack
;
709 if (ret_stacksize
== nil
) {
710 ret_stacksize
= &unused_stacksize
;
714 stacksize
= StackMin
;
716 stacksize
= 32 * 1024; // OS X wants >= 8K, GNU/Linux >= 2K
718 if(stacksize
< SIGSTKSZ
)
719 stacksize
= SIGSTKSZ
;
723 #if USING_SPLIT_STACK
724 *ret_stack
= __splitstack_makecontext(stacksize
,
725 (void*)(&newg
->stackcontext
[0]),
727 *ret_stacksize
= (uintptr
)ss_stacksize
;
728 __splitstack_block_signals_context((void*)(&newg
->stackcontext
[0]),
729 &dont_block_signals
, nil
);
731 // In 64-bit mode, the maximum Go allocation space is
732 // 128G. Our stack size is 4M, which only permits 32K
733 // goroutines. In order to not limit ourselves,
734 // allocate the stacks out of separate memory. In
735 // 32-bit mode, the Go allocation space is all of
737 if(sizeof(void*) == 8) {
738 void *p
= runtime_sysAlloc(stacksize
, &getMemstats()->stacks_sys
);
740 runtime_throw("runtime: cannot allocate memory for goroutine stack");
741 *ret_stack
= (byte
*)p
;
743 *ret_stack
= runtime_mallocgc(stacksize
, nil
, false);
744 runtime_xadd(&runtime_stacks_sys
, stacksize
);
746 *ret_stacksize
= (uintptr
)stacksize
;
747 newg
->gcinitialsp
= *ret_stack
;
748 newg
->gcstacksize
= (uintptr
)stacksize
;
754 void resetNewG(G
*, void **, uintptr
*)
755 __asm__(GOSYM_PREFIX
"runtime.resetNewG");
757 // Reset stack information for g pulled out of the cache to start a
760 resetNewG(G
*newg
, void **sp
, uintptr
*spsize
)
762 #ifdef USING_SPLIT_STACK
763 int dont_block_signals
= 0;
766 *sp
= __splitstack_resetcontext((void*)(&newg
->stackcontext
[0]), &ss_spsize
);
768 __splitstack_block_signals_context((void*)(&newg
->stackcontext
[0]),
769 &dont_block_signals
, nil
);
771 *sp
= newg
->gcinitialsp
;
772 *spsize
= newg
->gcstacksize
;
774 runtime_throw("bad spsize in resetNewG");
775 newg
->gcnextsp
= (uintptr
)(*sp
);
779 // Return whether we are waiting for a GC. This gc toolchain uses
780 // preemption instead.
782 runtime_gcwaiting(void)
784 return runtime_sched
->gcwaiting
;