1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
9 "runtime/internal/math"
13 // Should be a built-in for unsafe.Pointer?
15 func add(p unsafe
.Pointer
, x
uintptr) unsafe
.Pointer
{
16 return unsafe
.Pointer(uintptr(p
) + x
)
19 // getg returns the pointer to the current g.
20 // The compiler rewrites calls to this function into instructions
21 // that fetch the g directly (from TLS or from the dedicated register).
24 // mcall switches from the g to the g0 stack and invokes fn(g),
25 // where g is the goroutine that made the call.
26 // mcall saves g's current PC/SP in g->sched so that it can be restored later.
27 // It is up to fn to arrange for that later execution, typically by recording
28 // g in a data structure, causing something to call ready(g) later.
29 // mcall returns to the original goroutine g later, when g has been rescheduled.
30 // fn must not return at all; typically it ends by calling schedule, to let the m
31 // run other goroutines.
33 // mcall can only be called from g stacks (not g0, not gsignal).
35 // This must NOT be go:noescape: if fn is a stack-allocated closure,
36 // fn puts g on a run queue, and g executes before fn returns, the
37 // closure will be invalidated while it is still executing.
38 func mcall(fn
func(*g
))
40 // systemstack runs fn on a system stack.
42 // It is common to use a func literal as the argument, in order
43 // to share inputs and outputs with the code around the call
47 // systemstack(func() {
52 // For the gc toolchain this permits running a function that requires
53 // additional stack space in a context where the stack can not be
54 // split. We don't really need additional stack space in gccgo, since
55 // stack splitting is handled separately. But to keep things looking
56 // the same, we do switch to the g0 stack here if necessary.
57 func systemstack(fn
func()) {
60 if gp
== mp
.g0 || gp
== mp
.gsignal
{
62 } else if gp
== mp
.curg
{
63 fn1
:= func(origg
*g
) {
67 mcall(*(*func(*g
))(noescape(unsafe
.Pointer(&fn1
))))
73 var badsystemstackMsg
= "fatal: systemstack called from unexpected goroutine"
76 //go:nowritebarrierrec
77 func badsystemstack() {
78 sp
:= stringStructOf(&badsystemstackMsg
)
79 write(2, sp
.str
, int32(sp
.len))
82 // memclrNoHeapPointers clears n bytes starting at ptr.
84 // Usually you should use typedmemclr. memclrNoHeapPointers should be
85 // used only when the caller knows that *ptr contains no heap pointers
88 // *ptr is initialized memory and its type is pointer-free, or
90 // *ptr is uninitialized memory (e.g., memory that's being reused
91 // for a new allocation) and hence contains only "junk".
93 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
94 // is a multiple of the pointer size, then any pointer-aligned,
95 // pointer-sized portion is cleared atomically. Despite the function
96 // name, this is necessary because this function is the underlying
97 // implementation of typedmemclr and memclrHasPointers. See the doc of
98 // memmove for more details.
100 // The (CPU-specific) implementations of this function are in memclr_*.s.
103 func memclrNoHeapPointers(ptr unsafe
.Pointer
, n
uintptr)
105 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
106 func reflect_memclrNoHeapPointers(ptr unsafe
.Pointer
, n
uintptr) {
107 memclrNoHeapPointers(ptr
, n
)
111 func memmove(to
, from unsafe
.Pointer
, n
uintptr)
113 //go:linkname reflect_memmove reflect.memmove
114 func reflect_memmove(to
, from unsafe
.Pointer
, n
uintptr) {
119 //extern __builtin_memcmp
120 func memcmp(a
, b unsafe
.Pointer
, size
uintptr) int32
122 // exported value for testing
123 const hashLoad
= float32(loadFactorNum
) / float32(loadFactorDen
)
126 func fastrand() uint32 {
128 // Implement wyrand: https://github.com/wangyi-fudan/wyhash
129 // Only the platform that math.Mul64 can be lowered
130 // by the compiler should be in this list.
131 if goarch
.IsAmd64|goarch
.IsArm64|goarch
.IsPpc64|
132 goarch
.IsPpc64le|goarch
.IsMips64|goarch
.IsMips64le|
133 goarch
.IsS390x|goarch
.IsRiscv64
== 1 {
134 mp
.fastrand
+= 0xa0761d6478bd642f
135 hi
, lo
:= math
.Mul64(mp
.fastrand
, mp
.fastrand
^0xe7037ed1a0b428db)
136 return uint32(hi
^ lo
)
139 // Implement xorshift64+: 2 32-bit xorshift sequences added together.
140 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
141 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
142 // This generator passes the SmallCrush suite, part of TestU01 framework:
143 // http://simul.iro.umontreal.ca/testu01/tu01.html
144 t
:= (*[2]uint32)(unsafe
.Pointer(&mp
.fastrand
))
147 s1
= s1
^ s0
^ s1
>>7 ^ s0
>>16
153 func fastrandn(n
uint32) uint32 {
154 // This is similar to fastrand() % n, but faster.
155 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
156 return uint32(uint64(fastrand()) * uint64(n
) >> 32)
159 //go:linkname sync_fastrandn sync.fastrandn
160 func sync_fastrandn(n
uint32) uint32 { return fastrandn(n
) }
162 //go:linkname net_fastrand net.fastrand
163 func net_fastrand() uint32 { return fastrand() }
165 //go:linkname os_fastrand os.fastrand
166 func os_fastrand() uint32 { return fastrand() }
168 // in internal/bytealg/equal_*.s
170 func memequal(a
, b unsafe
.Pointer
, size
uintptr) bool
172 // noescape hides a pointer from escape analysis. noescape is
173 // the identity function but escape analysis doesn't think the
174 // output depends on the input. noescape is inlined and currently
175 // compiles down to zero instructions.
178 func noescape(p unsafe
.Pointer
) unsafe
.Pointer
{
180 return unsafe
.Pointer(x
^ 0)
184 func jmpdefer(fv
*funcval
, argp
uintptr)
185 func exit1(code
int32)
188 //extern __builtin_trap
194 func reflectcall(fntype
*functype
, fn
*funcval
, isInterface
, isMethod
bool, params
, results
*unsafe
.Pointer
)
196 func procyield(cycles
uint32)
198 type neverCallThisFunction
struct{}
200 // goexit is the return stub at the top of every goroutine call stack.
201 // Each goroutine stack is constructed as if goexit called the
202 // goroutine's entry point function, so that when the entry point
203 // function returns, it will return to goexit, which will call goexit1
204 // to perform the actual exit.
206 // This function must never be called directly. Call goexit1 instead.
207 // gentraceback assumes that goexit terminates the stack. A direct
208 // call on the stack will cause gentraceback to stop walking the stack
209 // prematurely and if there is leftover state it may panic.
210 func goexit(neverCallThisFunction
)
212 // publicationBarrier performs a store/store barrier (a "publication"
213 // or "export" barrier). Some form of synchronization is required
214 // between initializing an object and making that object accessible to
215 // another processor. Without synchronization, the initialization
216 // writes and the "publication" write may be reordered, allowing the
217 // other processor to follow the pointer and observe an uninitialized
218 // object. In general, higher-level synchronization should be used,
219 // such as locking or an atomic pointer write. publicationBarrier is
220 // for when those aren't an option, such as in the implementation of
221 // the memory manager.
223 // There's no corresponding barrier for the read side because the read
224 // side naturally has a data dependency order. All architectures that
225 // Go supports or seems likely to ever support automatically enforce
226 // data dependency ordering.
227 func publicationBarrier()
229 // getcallerpc returns the program counter (PC) of its caller's caller.
230 // getcallersp returns the stack pointer (SP) of its caller's caller.
231 // The implementation may be a compiler intrinsic; there is not
232 // necessarily code implementing this on every platform.
236 // func f(arg1, arg2, arg3 int) {
237 // pc := getcallerpc()
238 // sp := getcallersp()
241 // These two lines find the PC and SP immediately following
242 // the call to f (where f will return).
244 // The call to getcallerpc and getcallersp must be done in the
245 // frame being asked about.
247 // The result of getcallersp is correct at the time of the return,
248 // but it may be invalidated by any subsequent call to a function
249 // that might relocate the stack in order to grow or shrink it.
250 // A general rule is that the result of getcallersp should be used
251 // immediately and can only be passed to nosplit functions.
254 func getcallerpc() uintptr
257 func getcallersp() uintptr // implemented as an intrinsic on all platforms
259 // getsp returns the stack pointer (SP) of the caller of getsp.
261 func getsp() uintptr { return getcallersp() }
263 func asmcgocall(fn
, arg unsafe
.Pointer
) int32 {
268 // alignUp rounds n up to a multiple of a. a must be a power of 2.
269 func alignUp(n
, a
uintptr) uintptr {
270 return (n
+ a
- 1) &^ (a
- 1)
273 // alignDown rounds n down to a multiple of a. a must be a power of 2.
274 func alignDown(n
, a
uintptr) uintptr {
278 // divRoundUp returns ceil(n / a).
279 func divRoundUp(n
, a
uintptr) uintptr {
280 // a is generally a power of two. This will get inlined and
281 // the compiler will optimize the division.
282 return (n
+ a
- 1) / a
285 // checkASM returns whether assembly runtime checks have passed.
286 func checkASM() bool {
290 //extern __go_syscall6
291 func syscall(trap
uintptr, a1
, a2
, a3
, a4
, a5
, a6
uintptr) uintptr
293 // For gccgo, to communicate from the C code to the Go code.
294 //go:linkname setIsCgo
299 // For gccgo, to communicate from the C code to the Go code.
300 //go:linkname setSupportAES
301 func setSupportAES(v
bool) {
308 // For gccgo these are written in C.
310 func entersyscallblock()
312 // Get signal trampoline, written in C.
313 func getSigtramp() uintptr
315 // The sa_handler field is generally hidden in a union, so use C accessors.
317 func getSigactionHandler(*_sigaction
) uintptr
320 func setSigactionHandler(*_sigaction
, uintptr)
322 // Get signal code, written in C.
324 func getSiginfoCode(*_siginfo_t
) uintptr
326 // Retrieve fields from the siginfo_t and ucontext_t pointers passed
327 // to a signal handler using C, as they are often hidden in a union.
328 // Returns and, if available, PC where signal occurred.
329 func getSiginfo(*_siginfo_t
, unsafe
.Pointer
) (sigaddr
uintptr, sigpc
uintptr)
331 // Implemented in C for gccgo.
332 func dumpregs(*_siginfo_t
, unsafe
.Pointer
)
334 // Implemented in C for gccgo.
335 func setRandomNumber(uint32)
337 // Called by gccgo's proc.c.
343 // Throw and rethrow an exception.
344 func throwException()
345 func rethrowException()
347 // Fetch the size and required alignment of the _Unwind_Exception type
348 // used by the stack unwinder.
349 func unwindExceptionSize() uintptr
351 const uintptrMask
= 1<<(8*goarch
.PtrSize
) - 1
353 type bitvector
struct {
358 // ptrbit returns the i'th bit in bv.
359 // ptrbit is less efficient than iterating directly over bitvector bits,
360 // and should only be used in non-performance-critical code.
361 // See adjustpointers for an example of a high-efficiency walk of a bitvector.
362 func (bv
*bitvector
) ptrbit(i
uintptr) uint8 {
363 b
:= *(addb(bv
.bytedata
, i
/8))
364 return (b
>> (i
% 8)) & 1
367 // bool2int returns 0 if x is false or 1 if x is true.
368 func bool2int(x
bool) int {
375 // abort crashes the runtime in situations where even throw might not
376 // work. In general it should do something a debugger will recognize
377 // (e.g., an INT3 on x86). A crash in abort is recognized by the
378 // signal handler, which will attempt to tear down the runtime
382 // usestackmaps is true if stack map (precise stack scan) is enabled.
383 var usestackmaps
bool
385 // probestackmaps detects whether there are stack maps.
386 func probestackmaps() bool
388 // For the math/bits packages for gccgo.
389 //go:linkname getDivideError
390 func getDivideError() error
{
394 // For the math/bits packages for gccgo.
395 //go:linkname getOverflowError
396 func getOverflowError() error
{