1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
8 "runtime/internal/atomic"
13 // Should be a built-in for unsafe.Pointer?
15 func add(p unsafe
.Pointer
, x
uintptr) unsafe
.Pointer
{
16 return unsafe
.Pointer(uintptr(p
) + x
)
19 // getg returns the pointer to the current g.
20 // The compiler rewrites calls to this function into instructions
21 // that fetch the g directly (from TLS or from the dedicated register).
24 // mcall switches from the g to the g0 stack and invokes fn(g),
25 // where g is the goroutine that made the call.
26 // mcall saves g's current PC/SP in g->sched so that it can be restored later.
27 // It is up to fn to arrange for that later execution, typically by recording
28 // g in a data structure, causing something to call ready(g) later.
29 // mcall returns to the original goroutine g later, when g has been rescheduled.
30 // fn must not return at all; typically it ends by calling schedule, to let the m
31 // run other goroutines.
33 // mcall can only be called from g stacks (not g0, not gsignal).
35 // This must NOT be go:noescape: if fn is a stack-allocated closure,
36 // fn puts g on a run queue, and g executes before fn returns, the
37 // closure will be invalidated while it is still executing.
38 func mcall(fn
func(*g
))
40 // systemstack runs fn on a system stack.
42 // It is common to use a func literal as the argument, in order
43 // to share inputs and outputs with the code around the call
47 // systemstack(func() {
52 // For the gc toolchain this permits running a function that requires
53 // additional stack space in a context where the stack can not be
54 // split. For gccgo, however, stack splitting is not managed by the
55 // Go runtime. In effect, all stacks are system stacks. So this gccgo
56 // version just runs the function.
57 func systemstack(fn
func()) {
61 func badsystemstack() {
62 throw("systemstack called from unexpected goroutine")
65 // memclrNoHeapPointers clears n bytes starting at ptr.
67 // Usually you should use typedmemclr. memclrNoHeapPointers should be
68 // used only when the caller knows that *ptr contains no heap pointers
71 // 1. *ptr is initialized memory and its type is pointer-free.
73 // 2. *ptr is uninitialized memory (e.g., memory that's being reused
74 // for a new allocation) and hence contains only "junk".
78 func memclrNoHeapPointers(ptr unsafe
.Pointer
, n
uintptr)
80 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
81 func reflect_memclrNoHeapPointers(ptr unsafe
.Pointer
, n
uintptr) {
82 memclrNoHeapPointers(ptr
, n
)
85 // memmove copies n bytes from "from" to "to".
87 func memmove(to
, from unsafe
.Pointer
, n
uintptr)
89 //go:linkname reflect_memmove reflect.memmove
90 func reflect_memmove(to
, from unsafe
.Pointer
, n
uintptr) {
95 //extern __builtin_memcmp
96 func memcmp(a
, b unsafe
.Pointer
, size
uintptr) int32
98 // exported value for testing
99 var hashLoad
= loadFactor
102 func fastrand() uint32
104 //go:linkname sync_fastrand sync.fastrand
105 func sync_fastrand() uint32 { return fastrand() }
109 func memequal(a
, b unsafe
.Pointer
, size
uintptr) bool
111 // noescape hides a pointer from escape analysis. noescape is
112 // the identity function but escape analysis doesn't think the
113 // output depends on the input. noescape is inlined and currently
114 // compiles down to zero instructions.
117 func noescape(p unsafe
.Pointer
) unsafe
.Pointer
{
119 return unsafe
.Pointer(x
^ 0)
123 func mincore(addr unsafe
.Pointer
, n
uintptr, dst
*byte) int32
126 func jmpdefer(fv
*funcval
, argp
uintptr)
127 func exit1(code
int32)
132 // reflectcall calls fn with a copy of the n argument bytes pointed at by arg.
133 // After fn returns, reflectcall copies n-retoffset result bytes
134 // back into arg+retoffset before returning. If copying result bytes back,
135 // the caller should pass the argument frame type as argtype, so that
136 // call can execute appropriate write barriers during the copy.
137 // Package reflect passes a frame type. In package runtime, there is only
138 // one call that copies results back, in cgocallbackg1, and it does NOT pass a
139 // frame type, meaning there are no write barriers invoked. See that call
140 // site for justification.
141 func reflectcall(argtype
*_type
, fn
, arg unsafe
.Pointer
, argsize
uint32, retoffset
uint32)
143 func procyield(cycles
uint32)
145 type neverCallThisFunction
struct{}
147 // goexit is the return stub at the top of every goroutine call stack.
148 // Each goroutine stack is constructed as if goexit called the
149 // goroutine's entry point function, so that when the entry point
150 // function returns, it will return to goexit, which will call goexit1
151 // to perform the actual exit.
153 // This function must never be called directly. Call goexit1 instead.
154 // gentraceback assumes that goexit terminates the stack. A direct
155 // call on the stack will cause gentraceback to stop walking the stack
156 // prematurely and if there are leftover stack barriers it may panic.
157 func goexit(neverCallThisFunction
)
159 // publicationBarrier performs a store/store barrier (a "publication"
160 // or "export" barrier). Some form of synchronization is required
161 // between initializing an object and making that object accessible to
162 // another processor. Without synchronization, the initialization
163 // writes and the "publication" write may be reordered, allowing the
164 // other processor to follow the pointer and observe an uninitialized
165 // object. In general, higher-level synchronization should be used,
166 // such as locking or an atomic pointer write. publicationBarrier is
167 // for when those aren't an option, such as in the implementation of
168 // the memory manager.
170 // There's no corresponding barrier for the read side because the read
171 // side naturally has a data dependency order. All architectures that
172 // Go supports or seems likely to ever support automatically enforce
173 // data dependency ordering.
174 func publicationBarrier()
177 func setcallerpc(argp unsafe
.Pointer
, pc
uintptr)
179 // getcallerpc returns the program counter (PC) of its caller's caller.
180 // getcallersp returns the stack pointer (SP) of its caller's caller.
181 // For both, the argp must be a pointer to the caller's first function argument.
182 // The implementation may or may not use argp, depending on
187 // func f(arg1, arg2, arg3 int) {
188 // pc := getcallerpc(unsafe.Pointer(&arg1))
189 // sp := getcallersp(unsafe.Pointer(&arg1))
192 // These two lines find the PC and SP immediately following
193 // the call to f (where f will return).
195 // The call to getcallerpc and getcallersp must be done in the
196 // frame being asked about. It would not be correct for f to pass &arg1
197 // to another function g and let g call getcallerpc/getcallersp.
198 // The call inside g might return information about g's caller or
199 // information about f's caller or complete garbage.
201 // The result of getcallersp is correct at the time of the return,
202 // but it may be invalidated by any subsequent call to a function
203 // that might relocate the stack in order to grow or shrink it.
204 // A general rule is that the result of getcallersp should be used
205 // immediately and can only be passed to nosplit functions.
208 func getcallerpc(argp unsafe
.Pointer
) uintptr
211 func getcallersp(argp unsafe
.Pointer
) uintptr
213 // argp used in Defer structs when there is no argp.
214 const _NoArgs
= ^uintptr(0)
216 //go:linkname time_now time.now
217 func time_now() (sec
int64, nsec
int32)
219 // For gccgo, expose this for C callers.
220 //go:linkname unixnanotime runtime.unixnanotime
221 func unixnanotime() int64 {
222 sec
, nsec
:= time_now()
223 return sec
*1e9
+ int64(nsec
)
226 // round n up to a multiple of a. a must be a power of 2.
227 func round(n
, a
uintptr) uintptr {
228 return (n
+ a
- 1) &^ (a
- 1)
231 // checkASM returns whether assembly runtime checks have passed.
232 func checkASM() bool {
236 func eqstring(x
, y
string) bool {
237 a
:= stringStructOf(&x
)
238 b
:= stringStructOf(&y
)
245 return memequal(a
.str
, b
.str
, uintptr(a
.len))
248 // For gccgo this is in the C code.
251 // For gccgo this can be called directly.
253 func syscall(trap
uintptr, a1
, a2
, a3
, a4
, a5
, a6
uintptr) uintptr
255 // newobject allocates a new object.
256 // For gccgo unless and until we port malloc.go.
257 func newobject(*_type
) unsafe
.Pointer
259 // newarray allocates a new array of objects.
260 // For gccgo unless and until we port malloc.go.
261 func newarray(*_type
, int) unsafe
.Pointer
263 // For gccgo, to communicate from the C code to the Go code.
264 //go:linkname setIsCgo runtime.setIsCgo
269 // Temporary for gccgo until we port proc.go.
270 //go:linkname makeMainInitDone runtime.makeMainInitDone
271 func makeMainInitDone() {
272 main_init_done
= make(chan bool)
275 // Temporary for gccgo until we port proc.go.
276 //go:linkname closeMainInitDone runtime.closeMainInitDone
277 func closeMainInitDone() {
278 close(main_init_done
)
281 // For gccgo, to communicate from the C code to the Go code.
282 //go:linkname setCpuidECX runtime.setCpuidECX
283 func setCpuidECX(v
uint32) {
287 // For gccgo, to communicate from the C code to the Go code.
288 //go:linkname setSupportAES runtime.setSupportAES
289 func setSupportAES(v
bool) {
293 // typedmemmove copies a typed value.
294 // For gccgo for now.
295 //go:linkname typedmemmove runtime.typedmemmove
297 func typedmemmove(typ
*_type
, dst
, src unsafe
.Pointer
) {
298 memmove(dst
, src
, typ
.size
)
301 // Temporary for gccgo until we port mbarrier.go.
302 //go:linkname reflect_typedmemmove reflect.typedmemmove
303 func reflect_typedmemmove(typ
*_type
, dst
, src unsafe
.Pointer
) {
304 typedmemmove(typ
, dst
, src
)
307 // Temporary for gccgo until we port mbarrier.go.
309 func typedmemclr(typ
*_type
, ptr unsafe
.Pointer
) {
310 memclrNoHeapPointers(ptr
, typ
.size
)
313 // Temporary for gccgo until we port mbarrier.go.
315 func memclrHasPointers(ptr unsafe
.Pointer
, n
uintptr) {
316 memclrNoHeapPointers(ptr
, n
)
319 // Temporary for gccgo until we port mbarrier.go.
320 //go:linkname typedslicecopy runtime.typedslicecopy
321 func typedslicecopy(typ
*_type
, dst
, src slice
) int {
329 memmove(dst
.array
, src
.array
, uintptr(n
)*typ
.size
)
333 // Temporary for gccgo until we port mbarrier.go.
334 //go:linkname reflect_typedslicecopy reflect.typedslicecopy
335 func reflect_typedslicecopy(elemType
*_type
, dst
, src slice
) int {
336 return typedslicecopy(elemType
, dst
, src
)
339 // Here for gccgo until we port malloc.go.
341 _64bit
= 1 << (^uintptr(0) >> 63) / 2
342 _MHeapMap_TotalBits
= (_64bit
*sys
.GoosWindows
)*35 + (_64bit
*(1-sys
.GoosWindows
)*(1-sys
.GoosDarwin
*sys
.GoarchArm64
))*39 + sys
.GoosDarwin
*sys
.GoarchArm64
*31 + (1-_64bit
)*32
343 _MaxMem
= uintptr(1<<_MHeapMap_TotalBits
- 1)
347 // Here for gccgo until we port malloc.go.
348 //extern runtime_mallocgc
349 func c_mallocgc(size
uintptr, typ
uintptr, flag
uint32) unsafe
.Pointer
350 func mallocgc(size
uintptr, typ
*_type
, needzero
bool) unsafe
.Pointer
{
355 return c_mallocgc(size
, uintptr(unsafe
.Pointer(typ
)), flag
)
358 // Here for gccgo until we port mgc.go.
359 var writeBarrier
struct {
360 enabled
bool // compiler emits a check of this before calling write barrier
361 needed
bool // whether we need a write barrier for current GC phase
362 cgo
bool // whether we need a write barrier for a cgo check
363 alignme
uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load
366 func queueRescan(*g
) {
369 // Here for gccgo until we port atomic_pointer.go and mgc.go.
371 func casp(ptr
*unsafe
.Pointer
, old
, new unsafe
.Pointer
) bool {
372 if !atomic
.Casp1((*unsafe
.Pointer
)(noescape(unsafe
.Pointer(ptr
))), noescape(old
), new) {
378 // Here for gccgo until we port lock_*.go.
380 func unlock(l
*mutex
)
382 // Here for gccgo for netpoll and Solaris.
385 // Temporary for gccgo until we port proc.go.
386 func entersyscall(int32)
387 func entersyscallblock(int32)
388 func exitsyscall(int32)
389 func gopark(func(*g
, unsafe
.Pointer
) bool, unsafe
.Pointer
, string, byte, int)
390 func goparkunlock(*mutex
, string, byte, int)
392 // Temporary hack for gccgo until we port the garbage collector.
393 func typeBitsBulkBarrier(typ
*_type
, dst
, src
, size
uintptr) {}
395 // Here for gccgo until we port msize.go.
396 func roundupsize(uintptr) uintptr
398 // Here for gccgo until we port mgc.go.
401 // For gccgo to call from C code.
402 //go:linkname acquireWorldsema runtime.acquireWorldsema
403 func acquireWorldsema() {
404 semacquire(&worldsema
, 0)
407 // For gccgo to call from C code.
408 //go:linkname releaseWorldsema runtime.releaseWorldsema
409 func releaseWorldsema() {
410 semrelease(&worldsema
)
413 // For gccgo to call from C code, so that the C code and the Go code
414 // can share the memstats variable for now.
415 //go:linkname getMstats runtime.getMstats
416 func getMstats() *mstats
{
420 // Temporary for gccgo until we port proc.go.
421 func setcpuprofilerate_m(hz
int32)
423 // Temporary for gccgo until we port mem_GOOS.go.
424 func sysAlloc(n
uintptr, sysStat
*uint64) unsafe
.Pointer
425 func sysFree(v unsafe
.Pointer
, n
uintptr, sysStat
*uint64)
427 // Temporary for gccgo until we port proc.go, so that the C signal
428 // handler can call into cpuprof.
429 //go:linkname cpuprofAdd runtime.cpuprofAdd
430 func cpuprofAdd(stk
[]uintptr) {
434 // For gccgo until we port proc.go.
437 func UnlockOSThread()
439 func unlockOSThread()
441 // Temporary for gccgo until we port malloc.go
442 func persistentalloc(size
, align
uintptr, sysStat
*uint64) unsafe
.Pointer
444 // Temporary for gccgo until we port mheap.go
445 func setprofilebucket(p unsafe
.Pointer
, b
*bucket
)
447 // Temporary for gccgo until we port mgc.go.
448 func setgcpercent(int32) int32
450 //go:linkname setGCPercent runtime_debug.setGCPercent
451 func setGCPercent(in
int32) (out
int32) {
452 return setgcpercent(in
)
455 // Temporary for gccgo until we port atomic_pointer.go.
457 func atomicstorep(ptr unsafe
.Pointer
, new unsafe
.Pointer
) {
458 atomic
.StorepNoWB(noescape(ptr
), new)
461 // Temporary for gccgo until we port mbarrier.go
462 func writebarrierptr(dst
*uintptr, src
uintptr) {
466 // Temporary for gccgo until we port malloc.go
469 //go:linkname getZerobase runtime.getZerobase
470 func getZerobase() *uintptr {
474 // Temporary for gccgo until we port proc.go.
478 // Get signal trampoline, written in C.
479 func getSigtramp() uintptr
481 // The sa_handler field is generally hidden in a union, so use C accessors.
482 func getSigactionHandler(*_sigaction
) uintptr
483 func setSigactionHandler(*_sigaction
, uintptr)
485 // Retrieve fields from the siginfo_t and ucontext_t pointers passed
486 // to a signal handler using C, as they are often hidden in a union.
487 // Returns and, if available, PC where signal occurred.
488 func getSiginfo(*_siginfo_t
, unsafe
.Pointer
) (sigaddr
uintptr, sigpc
uintptr)
490 // Implemented in C for gccgo.
491 func dumpregs(*_siginfo_t
, unsafe
.Pointer
)
493 // Temporary for gccgo until we port proc.go.
494 //go:linkname getsched runtime.getsched
495 func getsched() *schedt
{
499 // Temporary for gccgo until we port proc.go.
500 //go:linkname getCgoHasExtraM runtime.getCgoHasExtraM
501 func getCgoHasExtraM() *bool {
505 // Temporary for gccgo until we port proc.go.
506 //go:linkname getAllP runtime.getAllP
511 // Temporary for gccgo until we port proc.go.
512 //go:linkname allocg runtime.allocg
517 // Temporary for gccgo until we port the garbage collector.
518 //go:linkname getallglen runtime.getallglen
519 func getallglen() uintptr {
523 // Temporary for gccgo until we port the garbage collector.
524 //go:linkname getallg runtime.getallg
525 func getallg(i
int) *g
{
529 // Temporary for gccgo until we port the garbage collector.
530 //go:linkname getallm runtime.getallm
535 // Throw and rethrow an exception.
536 func throwException()
537 func rethrowException()
539 // Fetch the size and required alignment of the _Unwind_Exception type
540 // used by the stack unwinder.
541 func unwindExceptionSize() uintptr
543 // Temporary for gccgo until C code no longer needs it.
545 //go:linkname getPanicking runtime.getPanicking
546 func getPanicking() uint32 {
550 // Temporary for gccgo until we port mcache.go.
551 func allocmcache() *mcache
552 func freemcache(*mcache
)
554 // Temporary for gccgo until we port mgc.go.
555 // This is just so that allgadd will compile.
563 // Temporary for gccgo until we port mgc.go.
564 var gcBlackenEnabled
uint32
566 // Temporary for gccgo until we port mgc.go.
567 func gcMarkWorkAvailable(p
*p
) bool {
571 // Temporary for gccgo until we port mgc.go.
572 var gcController gcControllerState
574 // Temporary for gccgo until we port mgc.go.
575 type gcControllerState
struct {
578 // Temporary for gccgo until we port mgc.go.
579 func (c
*gcControllerState
) findRunnableGCWorker(_p_
*p
) *g
{
583 // Temporary for gccgo until we port mgc.go.
586 // Temporary for gccgo until we port mgc.go.
593 // Temporary for gccgo until we port mgc.go.
594 type gcMarkWorkerMode
int
596 // Temporary for gccgo until we port mgc.go.
598 gcMarkWorkerDedicatedMode gcMarkWorkerMode
= iota
599 gcMarkWorkerFractionalMode
603 // Temporary for gccgo until we port mheap.go.
607 // Temporary for gccgo until we port mheap.go.
610 // Temporary for gccgo until we port mheap.go.
611 func (h
*mheap
) scavenge(k
int32, now
, limit
uint64) {
614 // Temporary for gccgo until we initialize ncpu in Go.
615 //go:linkname setncpu runtime.setncpu
616 func setncpu(n
int32) {
620 // Temporary for gccgo until we port malloc.go.
621 var physPageSize
uintptr
623 // Temporary for gccgo until we reliably initialize physPageSize in Go.
624 //go:linkname setpagesize runtime.setpagesize
625 func setpagesize(s
uintptr) {
626 if physPageSize
== 0 {
631 // Temporary for gccgo until we port more of proc.go.
632 func sigprofNonGoPC(pc
uintptr) {
635 // Temporary for gccgo until we port mgc.go.
636 // gcMarkWorkerModeStrings are the strings labels of gcMarkWorkerModes
637 // to use in execution traces.
638 var gcMarkWorkerModeStrings
= [...]string{