1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Garbage collector (GC).
9 // - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
10 // - parallel (up to MaxGcproc threads)
11 // - partially concurrent (mark is stop-the-world, while sweep is concurrent)
12 // - non-moving/non-compacting
13 // - full (non-partial)
16 // Next GC is after we've allocated an extra amount of memory proportional to
17 // the amount already in use. The proportion is controlled by GOGC environment variable
18 // (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
19 // (this mark is tracked in next_gc variable). This keeps the GC cost in linear
20 // proportion to the allocation cost. Adjusting GOGC just changes the linear constant
21 // (and also the amount of extra memory used).
24 // The sweep phase proceeds concurrently with normal program execution.
25 // The heap is swept span-by-span both lazily (when a goroutine needs another span)
26 // and concurrently in a background goroutine (this helps programs that are not CPU bound).
27 // However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
28 // and so next_gc calculation is tricky and happens as follows.
29 // At the end of the stop-the-world phase next_gc is conservatively set based on total
30 // heap size; all spans are marked as "needs sweeping".
31 // Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
32 // The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
33 // closer to the target value. However, this is not enough to avoid over-allocating memory.
34 // Consider that a goroutine wants to allocate a new span for a large object and
35 // there are no free swept spans, but there are small-object unswept spans.
36 // If the goroutine naively allocates a new span, it can surpass the yet-unknown
37 // target next_gc value. In order to prevent such cases (1) when a goroutine needs
38 // to allocate a new small-object span, it sweeps small-object spans for the same
39 // object size until it frees at least one object; (2) when a goroutine needs to
40 // allocate large-object span from heap, it sweeps spans until it frees at least
41 // that many pages into heap. Together these two measures ensure that we don't surpass
42 // target next_gc value by a large margin. There is an exception: if a goroutine sweeps
43 // and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
44 // but there can still be other one-page unswept spans which could be combined into a two-page span.
45 // It's critical to ensure that no operations proceed on unswept spans (that would corrupt
46 // mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
47 // so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
48 // When a goroutine explicitly frees an object or sets a finalizer, it ensures that
49 // the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
50 // The finalizer goroutine is kicked off only when all spans are swept.
51 // When the next GC starts, it sweeps all not-yet-swept spans (if any).
62 // Map gccgo field names to gc field names.
63 // Slice aka __go_open_array.
64 #define array __values
65 #define cap __capacity
66 // Iface aka __go_interface
69 typedef struct __go_map Hmap
;
70 // Type aka __go_type_descriptor
71 #define string __reflection
72 #define KindPtr GO_PTR
73 #define KindNoPointers GO_NO_POINTERS
74 #define kindMask GO_CODE_MASK
75 // PtrType aka __go_ptr_type
76 #define elem __element_type
78 #ifdef USING_SPLIT_STACK
80 extern void * __splitstack_find (void *, void *, size_t *, void **, void **,
83 extern void * __splitstack_find_context (void *context
[10], size_t *, void **,
93 WorkbufSize
= 16*1024,
94 FinBlockSize
= 4*1024,
97 IntermediateBufferCapacity
= 64,
99 // Bits in type information
102 PC_BITS
= PRECISE
| LOOP
,
112 #define GcpercentUnknown (-2)
114 // Initialized from $GOGC. GOGC=off means no gc.
115 static int32 gcpercent
= GcpercentUnknown
;
117 static FuncVal
* poolcleanup
;
119 void sync_runtime_registerPoolCleanup(FuncVal
*)
120 __asm__ (GOSYM_PREFIX
"sync.runtime_registerPoolCleanup");
123 sync_runtime_registerPoolCleanup(FuncVal
*f
)
135 if(poolcleanup
!= nil
) {
136 __builtin_call_with_static_chain(poolcleanup
->fn(),
140 for(pp
=runtime_allp
; (p
=*pp
) != nil
; pp
++) {
141 // clear tinyalloc pool
152 // Holding worldsema grants an M the right to try to stop the world.
155 // runtime_semacquire(&runtime_worldsema);
157 // runtime_stoptheworld();
162 // runtime_semrelease(&runtime_worldsema);
163 // runtime_starttheworld();
165 uint32 runtime_worldsema
= 1;
167 typedef struct Workbuf Workbuf
;
170 #define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr))
171 LFNode node
; // must be first
173 Obj obj
[SIZE
/sizeof(Obj
) - 1];
174 uint8 _padding
[SIZE
%sizeof(Obj
) + sizeof(Obj
)];
178 typedef struct Finalizer Finalizer
;
183 const struct __go_func_type
*ft
;
187 typedef struct FinBlock FinBlock
;
197 static Lock finlock
; // protects the following variables
198 static FinBlock
*finq
; // list of finalizers that are to be executed
199 static FinBlock
*finc
; // cache of free blocks
200 static FinBlock
*allfin
; // list of all blocks
201 bool runtime_fingwait
;
202 bool runtime_fingwake
;
207 static void runfinq(void*);
208 static void bgsweep(void*);
209 static Workbuf
* getempty(Workbuf
*);
210 static Workbuf
* getfull(Workbuf
*);
211 static void putempty(Workbuf
*);
212 static Workbuf
* handoff(Workbuf
*);
213 static void gchelperstart(void);
214 static void flushallmcaches(void);
215 static void addstackroots(G
*gp
, Workbuf
**wbufp
);
218 uint64 full
; // lock-free list of full blocks
219 uint64 empty
; // lock-free list of empty blocks
220 byte pad0
[CacheLineSize
]; // prevents false-sharing between full/empty and nproc/nwait
223 volatile uint32 nwait
;
224 volatile uint32 ndone
;
231 } work
__attribute__((aligned(8)));
234 GC_DEFAULT_PTR
= GC_NUM_INSTR
,
254 uint64 instr
[GC_NUM_INSTR2
];
271 // markonly marks an object. It returns true if the object
272 // has been marked by this function, false otherwise.
273 // This function doesn't append the object to any buffer.
275 markonly(const void *obj
)
278 uintptr
*bitp
, bits
, shift
, x
, xbits
, off
, j
;
282 // Words outside the arena cannot be pointers.
283 if((const byte
*)obj
< runtime_mheap
.arena_start
|| (const byte
*)obj
>= runtime_mheap
.arena_used
)
286 // obj may be a pointer to a live object.
287 // Try to find the beginning of the object.
289 // Round down to word boundary.
290 obj
= (const void*)((uintptr
)obj
& ~((uintptr
)PtrSize
-1));
292 // Find bits for this word.
293 off
= (const uintptr
*)obj
- (uintptr
*)runtime_mheap
.arena_start
;
294 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
295 shift
= off
% wordsPerBitmapWord
;
297 bits
= xbits
>> shift
;
299 // Pointing at the beginning of a block?
300 if((bits
& (bitAllocated
|bitBlockBoundary
)) != 0) {
302 runtime_xadd64(&gcstats
.markonly
.foundbit
, 1);
306 // Pointing just past the beginning?
307 // Scan backward a little to find a block boundary.
308 for(j
=shift
; j
-->0; ) {
309 if(((xbits
>>j
) & (bitAllocated
|bitBlockBoundary
)) != 0) {
313 runtime_xadd64(&gcstats
.markonly
.foundword
, 1);
318 // Otherwise consult span table to find beginning.
319 // (Manually inlined copy of MHeap_LookupMaybe.)
320 k
= (uintptr
)obj
>>PageShift
;
322 x
-= (uintptr
)runtime_mheap
.arena_start
>>PageShift
;
323 s
= runtime_mheap
.spans
[x
];
324 if(s
== nil
|| k
< s
->start
|| (const byte
*)obj
>= s
->limit
|| s
->state
!= MSpanInUse
)
326 p
= (byte
*)((uintptr
)s
->start
<<PageShift
);
327 if(s
->sizeclass
== 0) {
330 uintptr size
= s
->elemsize
;
331 int32 i
= ((const byte
*)obj
- p
)/size
;
335 // Now that we know the object header, reload bits.
336 off
= (const uintptr
*)obj
- (uintptr
*)runtime_mheap
.arena_start
;
337 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
338 shift
= off
% wordsPerBitmapWord
;
340 bits
= xbits
>> shift
;
342 runtime_xadd64(&gcstats
.markonly
.foundspan
, 1);
345 // Now we have bits, bitp, and shift correct for
346 // obj pointing at the base of the object.
347 // Only care about allocated and not marked.
348 if((bits
& (bitAllocated
|bitMarked
)) != bitAllocated
)
351 *bitp
|= bitMarked
<<shift
;
355 if(x
& (bitMarked
<<shift
))
357 if(runtime_casp((void**)bitp
, (void*)x
, (void*)(x
|(bitMarked
<<shift
))))
362 // The object is now marked
366 // PtrTarget is a structure used by intermediate buffers.
367 // The intermediate buffers hold GC data before it
368 // is moved/flushed to the work buffer (Workbuf).
369 // The size of an intermediate buffer is very small,
370 // such as 32 or 64 elements.
371 typedef struct PtrTarget PtrTarget
;
378 typedef struct Scanbuf Scanbuf
;
396 typedef struct BufferList BufferList
;
399 PtrTarget ptrtarget
[IntermediateBufferCapacity
];
400 Obj obj
[IntermediateBufferCapacity
];
402 byte pad
[CacheLineSize
];
404 static BufferList bufferList
[MaxGcproc
];
406 static void enqueue(Obj obj
, Workbuf
**_wbuf
, Obj
**_wp
, uintptr
*_nobj
);
408 // flushptrbuf moves data from the PtrTarget buffer to the work buffer.
409 // The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
410 // while the work buffer contains blocks which have been marked
411 // and are prepared to be scanned by the garbage collector.
413 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
415 // A simplified drawing explaining how the todo-list moves from a structure to another:
419 // Obj ------> PtrTarget (pointer targets)
424 // (find block start, mark and enqueue)
426 flushptrbuf(Scanbuf
*sbuf
)
428 byte
*p
, *arena_start
, *obj
;
429 uintptr size
, *bitp
, bits
, shift
, j
, x
, xbits
, off
, nobj
, ti
, n
;
435 PtrTarget
*ptrbuf_end
;
437 arena_start
= runtime_mheap
.arena_start
;
443 ptrbuf
= sbuf
->ptr
.begin
;
444 ptrbuf_end
= sbuf
->ptr
.pos
;
445 n
= ptrbuf_end
- sbuf
->ptr
.begin
;
446 sbuf
->ptr
.pos
= sbuf
->ptr
.begin
;
449 runtime_xadd64(&gcstats
.ptr
.sum
, n
);
450 runtime_xadd64(&gcstats
.ptr
.cnt
, 1);
453 // If buffer is nearly full, get a new one.
454 if(wbuf
== nil
|| nobj
+n
>= nelem(wbuf
->obj
)) {
457 wbuf
= getempty(wbuf
);
461 if(n
>= nelem(wbuf
->obj
))
462 runtime_throw("ptrbuf has to be smaller than WorkBuf");
465 while(ptrbuf
< ptrbuf_end
) {
470 // obj belongs to interval [mheap.arena_start, mheap.arena_used).
472 if(obj
< runtime_mheap
.arena_start
|| obj
>= runtime_mheap
.arena_used
)
473 runtime_throw("object is outside of mheap");
476 // obj may be a pointer to a live object.
477 // Try to find the beginning of the object.
479 // Round down to word boundary.
480 if(((uintptr
)obj
& ((uintptr
)PtrSize
-1)) != 0) {
481 obj
= (void*)((uintptr
)obj
& ~((uintptr
)PtrSize
-1));
485 // Find bits for this word.
486 off
= (uintptr
*)obj
- (uintptr
*)arena_start
;
487 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
488 shift
= off
% wordsPerBitmapWord
;
490 bits
= xbits
>> shift
;
492 // Pointing at the beginning of a block?
493 if((bits
& (bitAllocated
|bitBlockBoundary
)) != 0) {
495 runtime_xadd64(&gcstats
.flushptrbuf
.foundbit
, 1);
501 // Pointing just past the beginning?
502 // Scan backward a little to find a block boundary.
503 for(j
=shift
; j
-->0; ) {
504 if(((xbits
>>j
) & (bitAllocated
|bitBlockBoundary
)) != 0) {
505 obj
= (byte
*)obj
- (shift
-j
)*PtrSize
;
509 runtime_xadd64(&gcstats
.flushptrbuf
.foundword
, 1);
514 // Otherwise consult span table to find beginning.
515 // (Manually inlined copy of MHeap_LookupMaybe.)
516 k
= (uintptr
)obj
>>PageShift
;
518 x
-= (uintptr
)arena_start
>>PageShift
;
519 s
= runtime_mheap
.spans
[x
];
520 if(s
== nil
|| k
< s
->start
|| obj
>= s
->limit
|| s
->state
!= MSpanInUse
)
522 p
= (byte
*)((uintptr
)s
->start
<<PageShift
);
523 if(s
->sizeclass
== 0) {
527 int32 i
= ((byte
*)obj
- p
)/size
;
531 // Now that we know the object header, reload bits.
532 off
= (uintptr
*)obj
- (uintptr
*)arena_start
;
533 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
534 shift
= off
% wordsPerBitmapWord
;
536 bits
= xbits
>> shift
;
538 runtime_xadd64(&gcstats
.flushptrbuf
.foundspan
, 1);
541 // Now we have bits, bitp, and shift correct for
542 // obj pointing at the base of the object.
543 // Only care about allocated and not marked.
544 if((bits
& (bitAllocated
|bitMarked
)) != bitAllocated
)
547 *bitp
|= bitMarked
<<shift
;
551 if(x
& (bitMarked
<<shift
))
553 if(runtime_casp((void**)bitp
, (void*)x
, (void*)(x
|(bitMarked
<<shift
))))
558 // If object has no pointers, don't need to scan further.
559 if((bits
& bitScan
) == 0)
562 // Ask span about size class.
563 // (Manually inlined copy of MHeap_Lookup.)
564 x
= (uintptr
)obj
>> PageShift
;
565 x
-= (uintptr
)arena_start
>>PageShift
;
566 s
= runtime_mheap
.spans
[x
];
570 *wp
= (Obj
){obj
, s
->elemsize
, ti
};
576 // If another proc wants a pointer, give it some.
577 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
579 wbuf
= handoff(wbuf
);
581 wp
= wbuf
->obj
+ nobj
;
590 flushobjbuf(Scanbuf
*sbuf
)
602 objbuf
= sbuf
->obj
.begin
;
603 objbuf_end
= sbuf
->obj
.pos
;
604 sbuf
->obj
.pos
= sbuf
->obj
.begin
;
606 while(objbuf
< objbuf_end
) {
609 // Align obj.b to a word boundary.
610 off
= (uintptr
)obj
.p
& (PtrSize
-1);
612 obj
.p
+= PtrSize
- off
;
613 obj
.n
-= PtrSize
- off
;
617 if(obj
.p
== nil
|| obj
.n
== 0)
620 // If buffer is full, get a new one.
621 if(wbuf
== nil
|| nobj
>= nelem(wbuf
->obj
)) {
624 wbuf
= getempty(wbuf
);
634 // If another proc wants a pointer, give it some.
635 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
637 wbuf
= handoff(wbuf
);
639 wp
= wbuf
->obj
+ nobj
;
647 // Program that scans the whole block and treats every block element as a potential pointer
648 static uintptr defaultProg
[2] = {PtrSize
, GC_DEFAULT_PTR
};
651 static uintptr chanProg
[2] = {0, GC_CHAN
};
653 // Local variables of a program fragment or loop
654 typedef struct Frame Frame
;
656 uintptr count
, elemsize
, b
;
657 const uintptr
*loop_or_ret
;
660 // Sanity check for the derived type info objti.
662 checkptr(void *obj
, uintptr objti
)
664 uintptr
*pc1
, type
, tisize
, i
, j
, x
;
671 runtime_throw("checkptr is debug only");
673 if((byte
*)obj
< runtime_mheap
.arena_start
|| (byte
*)obj
>= runtime_mheap
.arena_used
)
675 type
= runtime_gettype(obj
);
676 t
= (Type
*)(type
& ~(uintptr
)(PtrSize
-1));
679 x
= (uintptr
)obj
>> PageShift
;
680 x
-= (uintptr
)(runtime_mheap
.arena_start
)>>PageShift
;
681 s
= runtime_mheap
.spans
[x
];
682 objstart
= (byte
*)((uintptr
)s
->start
<<PageShift
);
683 if(s
->sizeclass
!= 0) {
684 i
= ((byte
*)obj
- objstart
)/s
->elemsize
;
685 objstart
+= i
*s
->elemsize
;
687 tisize
= *(uintptr
*)objti
;
688 // Sanity check for object size: it should fit into the memory block.
689 if((byte
*)obj
+ tisize
> objstart
+ s
->elemsize
) {
690 runtime_printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
691 *t
->string
, obj
, tisize
, objstart
, s
->elemsize
);
692 runtime_throw("invalid gc type info");
696 // If obj points to the beginning of the memory block,
697 // check type info as well.
698 if(t
->string
== nil
||
699 // Gob allocates unsafe pointers for indirection.
700 (runtime_strcmp((const char *)t
->string
->str
, (const char*)"unsafe.Pointer") &&
701 // Runtime and gc think differently about closures.
702 runtime_strstr((const char *)t
->string
->str
, (const char*)"struct { F uintptr") != (const char *)t
->string
->str
)) {
703 pc1
= (uintptr
*)objti
;
704 pc2
= (const uintptr
*)t
->__gc
;
705 // A simple best-effort check until first GC_END.
706 for(j
= 1; pc1
[j
] != GC_END
&& pc2
[j
] != GC_END
; j
++) {
707 if(pc1
[j
] != pc2
[j
]) {
708 runtime_printf("invalid gc type info for '%s', type info %p [%d]=%p, block info %p [%d]=%p\n",
709 t
->string
? (const int8
*)t
->string
->str
: (const int8
*)"?", pc1
, (int32
)j
, pc1
[j
], pc2
, (int32
)j
, pc2
[j
]);
710 runtime_throw("invalid gc type info");
716 // scanblock scans a block of n bytes starting at pointer b for references
717 // to other objects, scanning any it finds recursively until there are no
718 // unscanned objects left. Instead of using an explicit recursion, it keeps
719 // a work list in the Workbuf* structures and loops in the main function
720 // body. Keeping an explicit work list is easier on the stack allocator and
723 scanblock(Workbuf
*wbuf
, bool keepworking
)
725 byte
*b
, *arena_start
, *arena_used
;
726 uintptr n
, i
, end_b
, elemsize
, size
, ti
, objti
, count
, type
, nobj
;
727 uintptr precise_type
, nominal_size
;
728 const uintptr
*pc
, *chan_ret
;
734 Frame
*stack_ptr
, stack_top
, stack
[GC_STACK_CAPACITY
+4];
735 BufferList
*scanbuffers
;
740 const ChanType
*chantype
;
743 if(sizeof(Workbuf
) % WorkbufSize
!= 0)
744 runtime_throw("scanblock: size of Workbuf is suboptimal");
746 // Memory arena parameters.
747 arena_start
= runtime_mheap
.arena_start
;
748 arena_used
= runtime_mheap
.arena_used
;
750 stack_ptr
= stack
+nelem(stack
)-1;
752 precise_type
= false;
757 wp
= &wbuf
->obj
[nobj
];
764 scanbuffers
= &bufferList
[runtime_m()->helpgc
];
766 sbuf
.ptr
.begin
= sbuf
.ptr
.pos
= &scanbuffers
->ptrtarget
[0];
767 sbuf
.ptr
.end
= sbuf
.ptr
.begin
+ nelem(scanbuffers
->ptrtarget
);
769 sbuf
.obj
.begin
= sbuf
.obj
.pos
= &scanbuffers
->obj
[0];
770 sbuf
.obj
.end
= sbuf
.obj
.begin
+ nelem(scanbuffers
->obj
);
776 // (Silence the compiler)
784 // Each iteration scans the block b of length n, queueing pointers in
788 runtime_xadd64(&gcstats
.nbytes
, n
);
789 runtime_xadd64(&gcstats
.obj
.sum
, sbuf
.nobj
);
790 runtime_xadd64(&gcstats
.obj
.cnt
, 1);
795 runtime_printf("scanblock %p %D ti %p\n", b
, (int64
)n
, ti
);
797 pc
= (uintptr
*)(ti
& ~(uintptr
)PC_BITS
);
798 precise_type
= (ti
& PRECISE
);
799 stack_top
.elemsize
= pc
[0];
801 nominal_size
= pc
[0];
803 stack_top
.count
= 0; // 0 means an infinite number of iterations
804 stack_top
.loop_or_ret
= pc
+1;
809 // Simple sanity check for provided type info ti:
810 // The declared size of the object must be not larger than the actual size
811 // (it can be smaller due to inferior pointers).
812 // It's difficult to make a comprehensive check due to inferior pointers,
813 // reflection, gob, etc.
815 runtime_printf("invalid gc type info: type info size %p, block size %p\n", pc
[0], n
);
816 runtime_throw("invalid gc type info");
819 } else if(UseSpanType
) {
821 runtime_xadd64(&gcstats
.obj
.notype
, 1);
823 type
= runtime_gettype(b
);
826 runtime_xadd64(&gcstats
.obj
.typelookup
, 1);
828 t
= (Type
*)(type
& ~(uintptr
)(PtrSize
-1));
829 switch(type
& (PtrSize
-1)) {
830 case TypeInfo_SingleObject
:
831 pc
= (const uintptr
*)t
->__gc
;
832 precise_type
= true; // type information about 'b' is precise
834 stack_top
.elemsize
= pc
[0];
837 pc
= (const uintptr
*)t
->__gc
;
840 precise_type
= true; // type information about 'b' is precise
841 stack_top
.count
= 0; // 0 means an infinite number of iterations
842 stack_top
.elemsize
= pc
[0];
843 stack_top
.loop_or_ret
= pc
+1;
847 chantype
= (const ChanType
*)t
;
853 runtime_printf("scanblock %p %D type %p %S\n", b
, (int64
)n
, type
, *t
->string
);
854 runtime_throw("scanblock: invalid type");
858 runtime_printf("scanblock %p %D type %p %S pc=%p\n", b
, (int64
)n
, type
, *t
->string
, pc
);
862 runtime_printf("scanblock %p %D unknown type\n", b
, (int64
)n
);
867 runtime_printf("scanblock %p %D no span types\n", b
, (int64
)n
);
874 stack_top
.b
= (uintptr
)b
;
875 end_b
= (uintptr
)b
+ n
- PtrSize
;
879 runtime_xadd64(&gcstats
.instr
[pc
[0]], 1);
885 obj
= *(void**)(stack_top
.b
+ pc
[1]);
888 runtime_printf("gc_ptr @%p: %p ti=%p\n", stack_top
.b
+pc
[1], obj
, objti
);
891 checkptr(obj
, objti
);
895 sliceptr
= (Slice
*)(stack_top
.b
+ pc
[1]);
897 runtime_printf("gc_slice @%p: %p/%D/%D\n", sliceptr
, sliceptr
->array
, (int64
)sliceptr
->__count
, (int64
)sliceptr
->cap
);
898 if(sliceptr
->cap
!= 0) {
899 obj
= sliceptr
->array
;
900 // Can't use slice element type for scanning,
901 // because if it points to an array embedded
902 // in the beginning of a struct,
903 // we will scan the whole struct as the slice.
904 // So just obtain type info from heap.
910 obj
= *(void**)(stack_top
.b
+ pc
[1]);
912 runtime_printf("gc_aptr @%p: %p\n", stack_top
.b
+pc
[1], obj
);
917 stringptr
= (String
*)(stack_top
.b
+ pc
[1]);
919 runtime_printf("gc_string @%p: %p/%D\n", stack_top
.b
+pc
[1], stringptr
->str
, (int64
)stringptr
->len
);
920 if(stringptr
->len
!= 0)
921 markonly(stringptr
->str
);
926 eface
= (Eface
*)(stack_top
.b
+ pc
[1]);
929 runtime_printf("gc_eface @%p: %p %p\n", stack_top
.b
+pc
[1], eface
->__type_descriptor
, eface
->__object
);
930 if(eface
->__type_descriptor
== nil
)
934 t
= eface
->__type_descriptor
;
935 if((const byte
*)t
>= arena_start
&& (const byte
*)t
< arena_used
) {
936 union { const Type
*tc
; Type
*tr
; } u
;
938 *sbuf
.ptr
.pos
++ = (PtrTarget
){u
.tr
, 0};
939 if(sbuf
.ptr
.pos
== sbuf
.ptr
.end
)
944 if((byte
*)eface
->__object
>= arena_start
&& (byte
*)eface
->__object
< arena_used
) {
945 if(__go_is_pointer_type(t
)) {
946 if((t
->__code
& KindNoPointers
))
949 obj
= eface
->__object
;
950 if((t
->__code
& kindMask
) == KindPtr
) {
951 // Only use type information if it is a pointer-containing type.
952 // This matches the GC programs written by cmd/gc/reflect.c's
953 // dgcsym1 in case TPTR32/case TPTR64. See rationale there.
954 et
= ((const PtrType
*)t
)->elem
;
955 if(!(et
->__code
& KindNoPointers
))
956 objti
= (uintptr
)((const PtrType
*)t
)->elem
->__gc
;
959 obj
= eface
->__object
;
960 objti
= (uintptr
)t
->__gc
;
966 iface
= (Iface
*)(stack_top
.b
+ pc
[1]);
969 runtime_printf("gc_iface @%p: %p/%p %p\n", stack_top
.b
+pc
[1], iface
->__methods
[0], nil
, iface
->__object
);
970 if(iface
->tab
== nil
)
974 if((byte
*)iface
->tab
>= arena_start
&& (byte
*)iface
->tab
< arena_used
) {
975 *sbuf
.ptr
.pos
++ = (PtrTarget
){iface
->tab
, 0};
976 if(sbuf
.ptr
.pos
== sbuf
.ptr
.end
)
981 if((byte
*)iface
->__object
>= arena_start
&& (byte
*)iface
->__object
< arena_used
) {
982 t
= (const Type
*)iface
->tab
[0];
983 if(__go_is_pointer_type(t
)) {
984 if((t
->__code
& KindNoPointers
))
987 obj
= iface
->__object
;
988 if((t
->__code
& kindMask
) == KindPtr
) {
989 // Only use type information if it is a pointer-containing type.
990 // This matches the GC programs written by cmd/gc/reflect.c's
991 // dgcsym1 in case TPTR32/case TPTR64. See rationale there.
992 et
= ((const PtrType
*)t
)->elem
;
993 if(!(et
->__code
& KindNoPointers
))
994 objti
= (uintptr
)((const PtrType
*)t
)->elem
->__gc
;
997 obj
= iface
->__object
;
998 objti
= (uintptr
)t
->__gc
;
1003 case GC_DEFAULT_PTR
:
1004 while(stack_top
.b
<= end_b
) {
1005 obj
= *(byte
**)stack_top
.b
;
1007 runtime_printf("gc_default_ptr @%p: %p\n", stack_top
.b
, obj
);
1008 stack_top
.b
+= PtrSize
;
1009 if((byte
*)obj
>= arena_start
&& (byte
*)obj
< arena_used
) {
1010 *sbuf
.ptr
.pos
++ = (PtrTarget
){obj
, 0};
1011 if(sbuf
.ptr
.pos
== sbuf
.ptr
.end
)
1018 if(--stack_top
.count
!= 0) {
1019 // Next iteration of a loop if possible.
1020 stack_top
.b
+= stack_top
.elemsize
;
1021 if(stack_top
.b
+ stack_top
.elemsize
<= end_b
+PtrSize
) {
1022 pc
= stack_top
.loop_or_ret
;
1027 // Stack pop if possible.
1028 if(stack_ptr
+1 < stack
+nelem(stack
)) {
1029 pc
= stack_top
.loop_or_ret
;
1030 stack_top
= *(++stack_ptr
);
1033 i
= (uintptr
)b
+ nominal_size
;
1036 // Quickly scan [b+i,b+n) for possible pointers.
1037 for(; i
<=end_b
; i
+=PtrSize
) {
1038 if(*(byte
**)i
!= nil
) {
1039 // Found a value that may be a pointer.
1040 // Do a rescan of the entire block.
1041 enqueue((Obj
){b
, n
, 0}, &sbuf
.wbuf
, &sbuf
.wp
, &sbuf
.nobj
);
1043 runtime_xadd64(&gcstats
.rescan
, 1);
1044 runtime_xadd64(&gcstats
.rescanbytes
, n
);
1052 case GC_ARRAY_START
:
1053 i
= stack_top
.b
+ pc
[1];
1059 *stack_ptr
-- = stack_top
;
1060 stack_top
= (Frame
){count
, elemsize
, i
, pc
};
1064 if(--stack_top
.count
!= 0) {
1065 stack_top
.b
+= stack_top
.elemsize
;
1066 pc
= stack_top
.loop_or_ret
;
1069 stack_top
= *(++stack_ptr
);
1076 *stack_ptr
-- = stack_top
;
1077 stack_top
= (Frame
){1, 0, stack_top
.b
+ pc
[1], pc
+3 /*return address*/};
1078 pc
= (const uintptr
*)((const byte
*)pc
+ *(const int32
*)(pc
+2)); // target of the CALL instruction
1082 obj
= (void*)(stack_top
.b
+ pc
[1]);
1088 runtime_printf("gc_region @%p: %D %p\n", stack_top
.b
+pc
[1], (int64
)size
, objti
);
1089 *sbuf
.obj
.pos
++ = (Obj
){obj
, size
, objti
};
1090 if(sbuf
.obj
.pos
== sbuf
.obj
.end
)
1095 chan
= *(Hchan
**)(stack_top
.b
+ pc
[1]);
1096 if(Debug
> 2 && chan
!= nil
)
1097 runtime_printf("gc_chan_ptr @%p: %p/%D/%D %p\n", stack_top
.b
+pc
[1], chan
, (int64
)chan
->qcount
, (int64
)chan
->dataqsiz
, pc
[2]);
1102 if(markonly(chan
)) {
1103 chantype
= (ChanType
*)pc
[2];
1104 if(!(chantype
->elem
->__code
& KindNoPointers
)) {
1115 // There are no heap pointers in struct Hchan,
1116 // so we can ignore the leading sizeof(Hchan) bytes.
1117 if(!(chantype
->elem
->__code
& KindNoPointers
)) {
1118 // Channel's buffer follows Hchan immediately in memory.
1119 // Size of buffer (cap(c)) is second int in the chan struct.
1120 chancap
= ((uintgo
*)chan
)[1];
1122 // TODO(atom): split into two chunks so that only the
1123 // in-use part of the circular buffer is scanned.
1124 // (Channel routines zero the unused part, so the current
1125 // code does not lead to leaks, it's just a little inefficient.)
1126 *sbuf
.obj
.pos
++ = (Obj
){(byte
*)chan
+runtime_Hchansize
, chancap
*chantype
->elem
->__size
,
1127 (uintptr
)chantype
->elem
->__gc
| PRECISE
| LOOP
};
1128 if(sbuf
.obj
.pos
== sbuf
.obj
.end
)
1138 runtime_printf("runtime: invalid GC instruction %p at %p\n", pc
[0], pc
);
1139 runtime_throw("scanblock: invalid GC instruction");
1143 if((byte
*)obj
>= arena_start
&& (byte
*)obj
< arena_used
) {
1144 *sbuf
.ptr
.pos
++ = (PtrTarget
){obj
, objti
};
1145 if(sbuf
.ptr
.pos
== sbuf
.ptr
.end
)
1151 // Done scanning [b, b+n). Prepare for the next iteration of
1152 // the loop by setting b, n, ti to the parameters for the next block.
1154 if(sbuf
.nobj
== 0) {
1158 if(sbuf
.nobj
== 0) {
1161 putempty(sbuf
.wbuf
);
1164 // Emptied our buffer: refill.
1165 sbuf
.wbuf
= getfull(sbuf
.wbuf
);
1166 if(sbuf
.wbuf
== nil
)
1168 sbuf
.nobj
= sbuf
.wbuf
->nobj
;
1169 sbuf
.wp
= sbuf
.wbuf
->obj
+ sbuf
.wbuf
->nobj
;
1173 // Fetch b from the work buffer.
1182 static struct root_list
* roots
;
1185 __go_register_gc_roots (struct root_list
* r
)
1187 // FIXME: This needs locking if multiple goroutines can call
1188 // dlopen simultaneously.
1193 // Append obj to the work buffer.
1194 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
1196 enqueue(Obj obj
, Workbuf
**_wbuf
, Obj
**_wp
, uintptr
*_nobj
)
1203 runtime_printf("append obj(%p %D %p)\n", obj
.p
, (int64
)obj
.n
, obj
.ti
);
1205 // Align obj.b to a word boundary.
1206 off
= (uintptr
)obj
.p
& (PtrSize
-1);
1208 obj
.p
+= PtrSize
- off
;
1209 obj
.n
-= PtrSize
- off
;
1213 if(obj
.p
== nil
|| obj
.n
== 0)
1216 // Load work buffer state
1221 // If another proc wants a pointer, give it some.
1222 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
1224 wbuf
= handoff(wbuf
);
1226 wp
= wbuf
->obj
+ nobj
;
1229 // If buffer is full, get a new one.
1230 if(wbuf
== nil
|| nobj
>= nelem(wbuf
->obj
)) {
1233 wbuf
= getempty(wbuf
);
1242 // Save work buffer state
1249 enqueue1(Workbuf
**wbufp
, Obj obj
)
1254 if(wbuf
->nobj
>= nelem(wbuf
->obj
))
1255 *wbufp
= wbuf
= getempty(wbuf
);
1256 wbuf
->obj
[wbuf
->nobj
++] = obj
;
1260 markroot(ParFor
*desc
, uint32 i
)
1265 MSpan
**allspans
, *s
;
1271 wbuf
= getempty(nil
);
1272 // Note: if you add a case here, please also update heapdump.c:dumproots.
1275 // For gccgo this is both data and bss.
1277 struct root_list
*pl
;
1279 for(pl
= roots
; pl
!= nil
; pl
= pl
->next
) {
1280 struct root
*pr
= &pl
->roots
[0];
1282 void *decl
= pr
->decl
;
1285 enqueue1(&wbuf
, (Obj
){decl
, pr
->size
, 0});
1293 // For gccgo we use this for all the other global roots.
1294 enqueue1(&wbuf
, (Obj
){(byte
*)&runtime_m0
, sizeof runtime_m0
, 0});
1295 enqueue1(&wbuf
, (Obj
){(byte
*)&runtime_g0
, sizeof runtime_g0
, 0});
1296 enqueue1(&wbuf
, (Obj
){(byte
*)&runtime_allg
, sizeof runtime_allg
, 0});
1297 enqueue1(&wbuf
, (Obj
){(byte
*)&runtime_allm
, sizeof runtime_allm
, 0});
1298 enqueue1(&wbuf
, (Obj
){(byte
*)&runtime_allp
, sizeof runtime_allp
, 0});
1299 enqueue1(&wbuf
, (Obj
){(byte
*)&work
, sizeof work
, 0});
1300 runtime_proc_scan(&wbuf
, enqueue1
);
1301 runtime_MProf_Mark(&wbuf
, enqueue1
);
1302 runtime_time_scan(&wbuf
, enqueue1
);
1303 runtime_netpoll_scan(&wbuf
, enqueue1
);
1306 case RootFinalizers
:
1307 for(fb
=allfin
; fb
; fb
=fb
->alllink
)
1308 enqueue1(&wbuf
, (Obj
){(byte
*)fb
->fin
, fb
->cnt
*sizeof(fb
->fin
[0]), 0});
1312 // mark span types and MSpan.specials (to walk spans only once)
1315 allspans
= h
->allspans
;
1316 for(spanidx
=0; spanidx
<runtime_mheap
.nspan
; spanidx
++) {
1318 SpecialFinalizer
*spf
;
1320 s
= allspans
[spanidx
];
1321 if(s
->sweepgen
!= sg
) {
1322 runtime_printf("sweep %d %d\n", s
->sweepgen
, sg
);
1323 runtime_throw("gc: unswept span");
1325 if(s
->state
!= MSpanInUse
)
1327 // The garbage collector ignores type pointers stored in MSpan.types:
1328 // - Compiler-generated types are stored outside of heap.
1329 // - The reflect package has runtime-generated types cached in its data structures.
1330 // The garbage collector relies on finding the references via that cache.
1331 if(s
->types
.compression
== MTypes_Words
|| s
->types
.compression
== MTypes_Bytes
)
1332 markonly((byte
*)s
->types
.data
);
1333 for(sp
= s
->specials
; sp
!= nil
; sp
= sp
->next
) {
1334 if(sp
->kind
!= KindSpecialFinalizer
)
1336 // don't mark finalized object, but scan it so we
1337 // retain everything it points to.
1338 spf
= (SpecialFinalizer
*)sp
;
1339 // A finalizer can be set for an inner byte of an object, find object beginning.
1340 p
= (void*)((s
->start
<< PageShift
) + spf
->offset
/s
->elemsize
*s
->elemsize
);
1341 enqueue1(&wbuf
, (Obj
){p
, s
->elemsize
, 0});
1342 enqueue1(&wbuf
, (Obj
){(void*)&spf
->fn
, PtrSize
, 0});
1343 enqueue1(&wbuf
, (Obj
){(void*)&spf
->ft
, PtrSize
, 0});
1344 enqueue1(&wbuf
, (Obj
){(void*)&spf
->ot
, PtrSize
, 0});
1349 case RootFlushCaches
:
1354 // the rest is scanning goroutine stacks
1355 if(i
- RootCount
>= runtime_allglen
)
1356 runtime_throw("markroot: bad index");
1357 gp
= runtime_allg
[i
- RootCount
];
1358 // remember when we've first observed the G blocked
1359 // needed only to output in traceback
1360 if((gp
->status
== Gwaiting
|| gp
->status
== Gsyscall
) && gp
->waitsince
== 0)
1361 gp
->waitsince
= work
.tstart
;
1362 addstackroots(gp
, &wbuf
);
1368 scanblock(wbuf
, false);
1371 static const FuncVal markroot_funcval
= { (void *) markroot
};
1373 // Get an empty work buffer off the work.empty list,
1374 // allocating new buffers as needed.
1376 getempty(Workbuf
*b
)
1379 runtime_lfstackpush(&work
.full
, &b
->node
);
1380 b
= (Workbuf
*)runtime_lfstackpop(&work
.empty
);
1382 // Need to allocate.
1383 runtime_lock(&work
);
1384 if(work
.nchunk
< sizeof *b
) {
1385 work
.nchunk
= 1<<20;
1386 work
.chunk
= runtime_SysAlloc(work
.nchunk
, &mstats
.gc_sys
);
1387 if(work
.chunk
== nil
)
1388 runtime_throw("runtime: cannot allocate memory");
1390 b
= (Workbuf
*)work
.chunk
;
1391 work
.chunk
+= sizeof *b
;
1392 work
.nchunk
-= sizeof *b
;
1393 runtime_unlock(&work
);
1400 putempty(Workbuf
*b
)
1403 runtime_xadd64(&gcstats
.putempty
, 1);
1405 runtime_lfstackpush(&work
.empty
, &b
->node
);
1408 // Get a full work buffer off the work.full list, or return nil.
1416 runtime_xadd64(&gcstats
.getfull
, 1);
1419 runtime_lfstackpush(&work
.empty
, &b
->node
);
1420 b
= (Workbuf
*)runtime_lfstackpop(&work
.full
);
1421 if(b
!= nil
|| work
.nproc
== 1)
1425 runtime_xadd(&work
.nwait
, +1);
1427 if(work
.full
!= 0) {
1428 runtime_xadd(&work
.nwait
, -1);
1429 b
= (Workbuf
*)runtime_lfstackpop(&work
.full
);
1432 runtime_xadd(&work
.nwait
, +1);
1434 if(work
.nwait
== work
.nproc
)
1437 m
->gcstats
.nprocyield
++;
1438 runtime_procyield(20);
1440 m
->gcstats
.nosyield
++;
1443 m
->gcstats
.nsleep
++;
1444 runtime_usleep(100);
1458 // Make new buffer with half of b's pointers.
1463 runtime_memmove(b1
->obj
, b
->obj
+b
->nobj
, n
*sizeof b1
->obj
[0]);
1464 m
->gcstats
.nhandoff
++;
1465 m
->gcstats
.nhandoffcnt
+= n
;
1467 // Put b on full list - let first half of b get stolen.
1468 runtime_lfstackpush(&work
.full
, &b
->node
);
1473 addstackroots(G
*gp
, Workbuf
**wbufp
)
1477 runtime_printf("unexpected G.status %d (goroutine %p %D)\n", gp
->status
, gp
, gp
->goid
);
1478 runtime_throw("mark - bad status");
1482 runtime_throw("mark - world not stopped");
1489 #ifdef USING_SPLIT_STACK
1497 if(gp
== runtime_g()) {
1498 // Scanning our own stack.
1499 sp
= __splitstack_find(nil
, nil
, &spsize
, &next_segment
,
1500 &next_sp
, &initial_sp
);
1501 } else if((mp
= gp
->m
) != nil
&& mp
->helpgc
) {
1502 // gchelper's stack is in active use and has no interesting pointers.
1505 // Scanning another goroutine's stack.
1506 // The goroutine is usually asleep (the world is stopped).
1508 // The exception is that if the goroutine is about to enter or might
1509 // have just exited a system call, it may be executing code such
1510 // as schedlock and may have needed to start a new stack segment.
1511 // Use the stack segment and stack pointer at the time of
1512 // the system call instead, since that won't change underfoot.
1513 if(gp
->gcstack
!= nil
) {
1515 spsize
= gp
->gcstack_size
;
1516 next_segment
= gp
->gcnext_segment
;
1517 next_sp
= gp
->gcnext_sp
;
1518 initial_sp
= gp
->gcinitial_sp
;
1520 sp
= __splitstack_find_context(&gp
->stack_context
[0],
1521 &spsize
, &next_segment
,
1522 &next_sp
, &initial_sp
);
1526 enqueue1(wbufp
, (Obj
){sp
, spsize
, 0});
1527 while((sp
= __splitstack_find(next_segment
, next_sp
,
1528 &spsize
, &next_segment
,
1529 &next_sp
, &initial_sp
)) != nil
)
1530 enqueue1(wbufp
, (Obj
){sp
, spsize
, 0});
1537 if(gp
== runtime_g()) {
1538 // Scanning our own stack.
1539 bottom
= (byte
*)&gp
;
1540 } else if((mp
= gp
->m
) != nil
&& mp
->helpgc
) {
1541 // gchelper's stack is in active use and has no interesting pointers.
1544 // Scanning another goroutine's stack.
1545 // The goroutine is usually asleep (the world is stopped).
1546 bottom
= (byte
*)gp
->gcnext_sp
;
1550 top
= (byte
*)gp
->gcinitial_sp
+ gp
->gcstack_size
;
1552 enqueue1(wbufp
, (Obj
){bottom
, top
- bottom
, 0});
1554 enqueue1(wbufp
, (Obj
){top
, bottom
- top
, 0});
1559 runtime_queuefinalizer(void *p
, FuncVal
*fn
, const FuncType
*ft
, const PtrType
*ot
)
1564 runtime_lock(&finlock
);
1565 if(finq
== nil
|| finq
->cnt
== finq
->cap
) {
1567 finc
= runtime_persistentalloc(FinBlockSize
, 0, &mstats
.gc_sys
);
1568 finc
->cap
= (FinBlockSize
- sizeof(FinBlock
)) / sizeof(Finalizer
) + 1;
1569 finc
->alllink
= allfin
;
1577 f
= &finq
->fin
[finq
->cnt
];
1583 runtime_fingwake
= true;
1584 runtime_unlock(&finlock
);
1588 runtime_iterate_finq(void (*callback
)(FuncVal
*, void*, const FuncType
*, const PtrType
*))
1594 for(fb
= allfin
; fb
; fb
= fb
->alllink
) {
1595 for(i
= 0; i
< fb
->cnt
; i
++) {
1597 callback(f
->fn
, f
->arg
, f
->ft
, f
->ot
);
1603 runtime_MSpan_EnsureSwept(MSpan
*s
)
1609 // Caller must disable preemption.
1610 // Otherwise when this function returns the span can become unswept again
1611 // (if GC is triggered on another goroutine).
1612 if(m
->locks
== 0 && m
->mallocing
== 0 && g
!= m
->g0
)
1613 runtime_throw("MSpan_EnsureSwept: m is not locked");
1615 sg
= runtime_mheap
.sweepgen
;
1616 if(runtime_atomicload(&s
->sweepgen
) == sg
)
1618 if(runtime_cas(&s
->sweepgen
, sg
-2, sg
-1)) {
1619 runtime_MSpan_Sweep(s
);
1622 // unfortunate condition, and we don't have efficient means to wait
1623 while(runtime_atomicload(&s
->sweepgen
) != sg
)
1627 // Sweep frees or collects finalizers for blocks not marked in the mark phase.
1628 // It clears the mark bits in preparation for the next GC round.
1629 // Returns true if the span was returned to heap.
1631 runtime_MSpan_Sweep(MSpan
*s
)
1634 int32 cl
, n
, npages
, nfree
;
1635 uintptr size
, off
, *bitp
, shift
, bits
;
1643 uintptr type_data_inc
;
1645 Special
*special
, **specialp
, *y
;
1646 bool res
, sweepgenset
;
1650 // It's critical that we enter this function with preemption disabled,
1651 // GC must not start while we are in the middle of this function.
1652 if(m
->locks
== 0 && m
->mallocing
== 0 && runtime_g() != m
->g0
)
1653 runtime_throw("MSpan_Sweep: m is not locked");
1654 sweepgen
= runtime_mheap
.sweepgen
;
1655 if(s
->state
!= MSpanInUse
|| s
->sweepgen
!= sweepgen
-1) {
1656 runtime_printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
1657 s
->state
, s
->sweepgen
, sweepgen
);
1658 runtime_throw("MSpan_Sweep: bad span state");
1660 arena_start
= runtime_mheap
.arena_start
;
1666 // Chunk full of small blocks.
1667 npages
= runtime_class_to_allocnpages
[cl
];
1668 n
= (npages
<< PageShift
) / size
;
1674 sweepgenset
= false;
1676 // mark any free objects in this span so we don't collect them
1677 for(x
= s
->freelist
; x
!= nil
; x
= x
->next
) {
1678 // This is markonly(x) but faster because we don't need
1679 // atomic access and we're guaranteed to be pointing at
1680 // the head of a valid object.
1681 off
= (uintptr
*)x
- (uintptr
*)runtime_mheap
.arena_start
;
1682 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
1683 shift
= off
% wordsPerBitmapWord
;
1684 *bitp
|= bitMarked
<<shift
;
1687 // Unlink & free special records for any objects we're about to free.
1688 specialp
= &s
->specials
;
1689 special
= *specialp
;
1690 while(special
!= nil
) {
1691 // A finalizer can be set for an inner byte of an object, find object beginning.
1692 p
= (byte
*)(s
->start
<< PageShift
) + special
->offset
/size
*size
;
1693 off
= (uintptr
*)p
- (uintptr
*)arena_start
;
1694 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
1695 shift
= off
% wordsPerBitmapWord
;
1696 bits
= *bitp
>>shift
;
1697 if((bits
& (bitAllocated
|bitMarked
)) == bitAllocated
) {
1698 // Find the exact byte for which the special was setup
1699 // (as opposed to object beginning).
1700 p
= (byte
*)(s
->start
<< PageShift
) + special
->offset
;
1701 // about to free object: splice out special record
1703 special
= special
->next
;
1704 *specialp
= special
;
1705 if(!runtime_freespecial(y
, p
, size
, false)) {
1706 // stop freeing of object if it has a finalizer
1707 *bitp
|= bitMarked
<< shift
;
1710 // object is still live: keep special record
1711 specialp
= &special
->next
;
1712 special
= *specialp
;
1716 type_data
= (byte
*)s
->types
.data
;
1717 type_data_inc
= sizeof(uintptr
);
1718 compression
= s
->types
.compression
;
1719 switch(compression
) {
1721 type_data
+= 8*sizeof(uintptr
);
1726 // Sweep through n objects of given size starting at p.
1727 // This thread owns the span now, so it can manipulate
1728 // the block bitmap without atomic operations.
1729 p
= (byte
*)(s
->start
<< PageShift
);
1730 for(; n
> 0; n
--, p
+= size
, type_data
+=type_data_inc
) {
1731 off
= (uintptr
*)p
- (uintptr
*)arena_start
;
1732 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
1733 shift
= off
% wordsPerBitmapWord
;
1734 bits
= *bitp
>>shift
;
1736 if((bits
& bitAllocated
) == 0)
1739 if((bits
& bitMarked
) != 0) {
1740 *bitp
&= ~(bitMarked
<<shift
);
1744 if(runtime_debug
.allocfreetrace
)
1745 runtime_tracefree(p
, size
);
1747 // Clear mark and scan bits.
1748 *bitp
&= ~((bitScan
|bitMarked
)<<shift
);
1752 runtime_unmarkspan(p
, 1<<PageShift
);
1754 // important to set sweepgen before returning it to heap
1755 runtime_atomicstore(&s
->sweepgen
, sweepgen
);
1757 // See note about SysFault vs SysFree in malloc.goc.
1758 if(runtime_debug
.efence
)
1759 runtime_SysFault(p
, size
);
1761 runtime_MHeap_Free(&runtime_mheap
, s
, 1);
1762 c
->local_nlargefree
++;
1763 c
->local_largefree
+= size
;
1764 runtime_xadd64(&mstats
.next_gc
, -(uint64
)(size
* (gcpercent
+ 100)/100));
1767 // Free small object.
1768 switch(compression
) {
1770 *(uintptr
*)type_data
= 0;
1773 *(byte
*)type_data
= 0;
1776 if(size
> 2*sizeof(uintptr
))
1777 ((uintptr
*)p
)[1] = (uintptr
)0xdeaddeaddeaddeadll
; // mark as "needs to be zeroed"
1778 else if(size
> sizeof(uintptr
))
1779 ((uintptr
*)p
)[1] = 0;
1781 end
->next
= (MLink
*)p
;
1787 // We need to set s->sweepgen = h->sweepgen only when all blocks are swept,
1788 // because of the potential for a concurrent free/SetFinalizer.
1789 // But we need to set it before we make the span available for allocation
1790 // (return it to heap or mcentral), because allocation code assumes that a
1791 // span is already swept if available for allocation.
1793 if(!sweepgenset
&& nfree
== 0) {
1794 // The span must be in our exclusive ownership until we update sweepgen,
1795 // check for potential races.
1796 if(s
->state
!= MSpanInUse
|| s
->sweepgen
!= sweepgen
-1) {
1797 runtime_printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
1798 s
->state
, s
->sweepgen
, sweepgen
);
1799 runtime_throw("MSpan_Sweep: bad span state after sweep");
1801 runtime_atomicstore(&s
->sweepgen
, sweepgen
);
1804 c
->local_nsmallfree
[cl
] += nfree
;
1805 c
->local_cachealloc
-= nfree
* size
;
1806 runtime_xadd64(&mstats
.next_gc
, -(uint64
)(nfree
* size
* (gcpercent
+ 100)/100));
1807 res
= runtime_MCentral_FreeSpan(&runtime_mheap
.central
[cl
], s
, nfree
, head
.next
, end
);
1808 //MCentral_FreeSpan updates sweepgen
1813 // State of background sweep.
1814 // Protected by gclock.
1825 // background sweeping goroutine
1827 bgsweep(void* dummy
__attribute__ ((unused
)))
1829 runtime_g()->issystem
= 1;
1831 while(runtime_sweepone() != (uintptr
)-1) {
1835 runtime_lock(&gclock
);
1836 if(!runtime_mheap
.sweepdone
) {
1837 // It's possible if GC has happened between sweepone has
1838 // returned -1 and gclock lock.
1839 runtime_unlock(&gclock
);
1842 sweep
.parked
= true;
1843 runtime_g()->isbackground
= true;
1844 runtime_parkunlock(&gclock
, "GC sweep wait");
1845 runtime_g()->isbackground
= false;
1850 // returns number of pages returned to heap, or -1 if there is nothing to sweep
1852 runtime_sweepone(void)
1859 // increment locks to ensure that the goroutine is not preempted
1860 // in the middle of sweep thus leaving the span in an inconsistent state for next GC
1862 sg
= runtime_mheap
.sweepgen
;
1864 idx
= runtime_xadd(&sweep
.spanidx
, 1) - 1;
1865 if(idx
>= sweep
.nspan
) {
1866 runtime_mheap
.sweepdone
= true;
1870 s
= sweep
.spans
[idx
];
1871 if(s
->state
!= MSpanInUse
) {
1875 if(s
->sweepgen
!= sg
-2 || !runtime_cas(&s
->sweepgen
, sg
-2, sg
-1))
1878 runtime_throw("sweep of incache span");
1880 if(!runtime_MSpan_Sweep(s
))
1888 dumpspan(uint32 idx
)
1890 int32 sizeclass
, n
, npages
, i
, column
;
1897 s
= runtime_mheap
.allspans
[idx
];
1898 if(s
->state
!= MSpanInUse
)
1900 arena_start
= runtime_mheap
.arena_start
;
1901 p
= (byte
*)(s
->start
<< PageShift
);
1902 sizeclass
= s
->sizeclass
;
1904 if(sizeclass
== 0) {
1907 npages
= runtime_class_to_allocnpages
[sizeclass
];
1908 n
= (npages
<< PageShift
) / size
;
1911 runtime_printf("%p .. %p:\n", p
, p
+n
*size
);
1913 for(; n
>0; n
--, p
+=size
) {
1914 uintptr off
, *bitp
, shift
, bits
;
1916 off
= (uintptr
*)p
- (uintptr
*)arena_start
;
1917 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
1918 shift
= off
% wordsPerBitmapWord
;
1919 bits
= *bitp
>>shift
;
1921 allocated
= ((bits
& bitAllocated
) != 0);
1923 for(i
=0; (uint32
)i
<size
; i
+=sizeof(void*)) {
1925 runtime_printf("\t");
1928 runtime_printf(allocated
? "(" : "[");
1929 runtime_printf("%p: ", p
+i
);
1931 runtime_printf(" ");
1934 runtime_printf("%p", *(void**)(p
+i
));
1936 if(i
+sizeof(void*) >= size
) {
1937 runtime_printf(allocated
? ") " : "] ");
1942 runtime_printf("\n");
1947 runtime_printf("\n");
1950 // A debugging function to dump the contents of memory
1952 runtime_memorydump(void)
1956 for(spanidx
=0; spanidx
<runtime_mheap
.nspan
; spanidx
++) {
1962 runtime_gchelper(void)
1966 runtime_m()->traceback
= 2;
1969 // parallel mark for over gc roots
1970 runtime_parfordo(work
.markfor
);
1972 // help other threads scan secondary blocks
1973 scanblock(nil
, true);
1975 bufferList
[runtime_m()->helpgc
].busy
= 0;
1976 nproc
= work
.nproc
; // work.nproc can change right after we increment work.ndone
1977 if(runtime_xadd(&work
.ndone
, +1) == nproc
-1)
1978 runtime_notewakeup(&work
.alldone
);
1979 runtime_m()->traceback
= 0;
1988 for(pp
=runtime_allp
; (p
=*pp
) != nil
; pp
++) {
1992 runtime_purgecachedstats(c
);
1997 flushallmcaches(void)
2002 // Flush MCache's to MCentral.
2003 for(pp
=runtime_allp
; (p
=*pp
) != nil
; pp
++) {
2007 runtime_MCache_ReleaseAll(c
);
2012 runtime_updatememstats(GCStats
*stats
)
2017 uint64 stacks_inuse
, smallfree
;
2021 runtime_memclr((byte
*)stats
, sizeof(*stats
));
2023 for(mp
=runtime_allm
; mp
; mp
=mp
->alllink
) {
2024 //stacks_inuse += mp->stackinuse*FixedStack;
2026 src
= (uint64
*)&mp
->gcstats
;
2027 dst
= (uint64
*)stats
;
2028 for(i
=0; i
<sizeof(*stats
)/sizeof(uint64
); i
++)
2030 runtime_memclr((byte
*)&mp
->gcstats
, sizeof(mp
->gcstats
));
2033 mstats
.stacks_inuse
= stacks_inuse
;
2034 mstats
.mcache_inuse
= runtime_mheap
.cachealloc
.inuse
;
2035 mstats
.mspan_inuse
= runtime_mheap
.spanalloc
.inuse
;
2036 mstats
.sys
= mstats
.heap_sys
+ mstats
.stacks_sys
+ mstats
.mspan_sys
+
2037 mstats
.mcache_sys
+ mstats
.buckhash_sys
+ mstats
.gc_sys
+ mstats
.other_sys
;
2039 // Calculate memory allocator stats.
2040 // During program execution we only count number of frees and amount of freed memory.
2041 // Current number of alive object in the heap and amount of alive heap memory
2042 // are calculated by scanning all spans.
2043 // Total number of mallocs is calculated as number of frees plus number of alive objects.
2044 // Similarly, total amount of allocated memory is calculated as amount of freed memory
2045 // plus amount of alive heap memory.
2047 mstats
.total_alloc
= 0;
2050 for(i
= 0; i
< nelem(mstats
.by_size
); i
++) {
2051 mstats
.by_size
[i
].nmalloc
= 0;
2052 mstats
.by_size
[i
].nfree
= 0;
2055 // Flush MCache's to MCentral.
2058 // Aggregate local stats.
2061 // Scan all spans and count number of alive objects.
2062 for(i
= 0; i
< runtime_mheap
.nspan
; i
++) {
2063 s
= runtime_mheap
.allspans
[i
];
2064 if(s
->state
!= MSpanInUse
)
2066 if(s
->sizeclass
== 0) {
2068 mstats
.alloc
+= s
->elemsize
;
2070 mstats
.nmalloc
+= s
->ref
;
2071 mstats
.by_size
[s
->sizeclass
].nmalloc
+= s
->ref
;
2072 mstats
.alloc
+= s
->ref
*s
->elemsize
;
2076 // Aggregate by size class.
2078 mstats
.nfree
= runtime_mheap
.nlargefree
;
2079 for(i
= 0; i
< nelem(mstats
.by_size
); i
++) {
2080 mstats
.nfree
+= runtime_mheap
.nsmallfree
[i
];
2081 mstats
.by_size
[i
].nfree
= runtime_mheap
.nsmallfree
[i
];
2082 mstats
.by_size
[i
].nmalloc
+= runtime_mheap
.nsmallfree
[i
];
2083 smallfree
+= runtime_mheap
.nsmallfree
[i
] * runtime_class_to_size
[i
];
2085 mstats
.nmalloc
+= mstats
.nfree
;
2087 // Calculate derived stats.
2088 mstats
.total_alloc
= mstats
.alloc
+ runtime_mheap
.largefree
+ smallfree
;
2089 mstats
.heap_alloc
= mstats
.alloc
;
2090 mstats
.heap_objects
= mstats
.nmalloc
- mstats
.nfree
;
2093 // Structure of arguments passed to function gc().
2094 // This allows the arguments to be passed via runtime_mcall.
2097 int64 start_time
; // start time of GC in ns (just before stoptheworld)
2101 static void gc(struct gc_args
*args
);
2102 static void mgc(G
*gp
);
2110 s
= runtime_getenv("GOGC");
2114 if(s
.len
== 3 && runtime_strcmp((const char *)p
, "off") == 0)
2116 return runtime_atoi(p
, s
.len
);
2119 // force = 1 - do GC regardless of current heap usage
2120 // force = 2 - go GC and eager sweep
2122 runtime_gc(int32 force
)
2129 // The atomic operations are not atomic if the uint64s
2130 // are not aligned on uint64 boundaries. This has been
2131 // a problem in the past.
2132 if((((uintptr
)&work
.empty
) & 7) != 0)
2133 runtime_throw("runtime: gc work buffer is misaligned");
2134 if((((uintptr
)&work
.full
) & 7) != 0)
2135 runtime_throw("runtime: gc work buffer is misaligned");
2137 // Make sure all registers are saved on stack so that
2138 // scanstack sees them.
2139 __builtin_unwind_init();
2141 // The gc is turned off (via enablegc) until
2142 // the bootstrap has completed.
2143 // Also, malloc gets called in the guts
2144 // of a number of libraries that might be
2145 // holding locks. To avoid priority inversion
2146 // problems, don't bother trying to run gc
2147 // while holding a lock. The next mallocgc
2148 // without a lock will do the gc instead.
2150 if(!mstats
.enablegc
|| runtime_g() == m
->g0
|| m
->locks
> 0 || runtime_panicking
)
2153 if(gcpercent
== GcpercentUnknown
) { // first time through
2154 runtime_lock(&runtime_mheap
);
2155 if(gcpercent
== GcpercentUnknown
)
2156 gcpercent
= readgogc();
2157 runtime_unlock(&runtime_mheap
);
2162 runtime_semacquire(&runtime_worldsema
, false);
2163 if(force
==0 && mstats
.heap_alloc
< mstats
.next_gc
) {
2164 // typically threads which lost the race to grab
2165 // worldsema exit here when gc is done.
2166 runtime_semrelease(&runtime_worldsema
);
2170 // Ok, we're doing it! Stop everybody else
2171 a
.start_time
= runtime_nanotime();
2172 a
.eagersweep
= force
>= 2;
2174 runtime_stoptheworld();
2178 // Run gc on the g0 stack. We do this so that the g stack
2179 // we're currently running on will no longer change. Cuts
2180 // the root set down a bit (g0 stacks are not scanned, and
2181 // we don't need to scan gc's internal state). Also an
2182 // enabler for copyable stacks.
2183 for(i
= 0; i
< (runtime_debug
.gctrace
> 1 ? 2 : 1); i
++) {
2185 a
.start_time
= runtime_nanotime();
2186 // switch to g0, call gc(&a), then switch back
2189 g
->status
= Gwaiting
;
2190 g
->waitreason
= "garbage collection";
2198 runtime_semrelease(&runtime_worldsema
);
2199 runtime_starttheworld();
2202 // now that gc is done, kick off finalizer thread if needed
2203 if(!ConcurrentSweep
) {
2204 // give the queued finalizers, if any, a chance to run
2207 // For gccgo, let other goroutines run.
2217 gp
->status
= Grunning
;
2222 gc(struct gc_args
*args
)
2225 int64 t0
, t1
, t2
, t3
, t4
;
2226 uint64 heap0
, heap1
, obj
, ninstr
;
2233 if(runtime_debug
.allocfreetrace
)
2237 t0
= args
->start_time
;
2238 work
.tstart
= args
->start_time
;
2241 runtime_memclr((byte
*)&gcstats
, sizeof(gcstats
));
2243 m
->locks
++; // disable gc during mallocs in parforalloc
2244 if(work
.markfor
== nil
)
2245 work
.markfor
= runtime_parforalloc(MaxGcproc
);
2249 if(runtime_debug
.gctrace
)
2250 t1
= runtime_nanotime();
2252 // Sweep what is not sweeped by bgsweep.
2253 while(runtime_sweepone() != (uintptr
)-1)
2254 gcstats
.npausesweep
++;
2258 work
.nproc
= runtime_gcprocs();
2259 runtime_parforsetup(work
.markfor
, work
.nproc
, RootCount
+ runtime_allglen
, false, &markroot_funcval
);
2260 if(work
.nproc
> 1) {
2261 runtime_noteclear(&work
.alldone
);
2262 runtime_helpgc(work
.nproc
);
2266 if(runtime_debug
.gctrace
)
2267 t2
= runtime_nanotime();
2270 runtime_parfordo(work
.markfor
);
2271 scanblock(nil
, true);
2274 if(runtime_debug
.gctrace
)
2275 t3
= runtime_nanotime();
2277 bufferList
[m
->helpgc
].busy
= 0;
2279 runtime_notesleep(&work
.alldone
);
2282 // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
2283 // estimate what was live heap size after previous GC (for tracing only)
2284 heap0
= mstats
.next_gc
*100/(gcpercent
+100);
2285 // conservatively set next_gc to high value assuming that everything is live
2286 // concurrent/lazy sweep will reduce this number while discovering new garbage
2287 mstats
.next_gc
= mstats
.heap_alloc
+(mstats
.heap_alloc
-runtime_stacks_sys
)*gcpercent
/100;
2289 t4
= runtime_nanotime();
2290 mstats
.last_gc
= runtime_unixnanotime(); // must be Unix time to make sense to user
2291 mstats
.pause_ns
[mstats
.numgc
%nelem(mstats
.pause_ns
)] = t4
- t0
;
2292 mstats
.pause_end
[mstats
.numgc
%nelem(mstats
.pause_end
)] = mstats
.last_gc
;
2293 mstats
.pause_total_ns
+= t4
- t0
;
2296 runtime_printf("pause %D\n", t4
-t0
);
2298 if(runtime_debug
.gctrace
) {
2299 heap1
= mstats
.heap_alloc
;
2300 runtime_updatememstats(&stats
);
2301 if(heap1
!= mstats
.heap_alloc
) {
2302 runtime_printf("runtime: mstats skew: heap=%D/%D\n", heap1
, mstats
.heap_alloc
);
2303 runtime_throw("mstats skew");
2305 obj
= mstats
.nmalloc
- mstats
.nfree
;
2307 stats
.nprocyield
+= work
.markfor
->nprocyield
;
2308 stats
.nosyield
+= work
.markfor
->nosyield
;
2309 stats
.nsleep
+= work
.markfor
->nsleep
;
2311 runtime_printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects,"
2313 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
2314 mstats
.numgc
, work
.nproc
, (t1
-t0
)/1000, (t2
-t1
)/1000, (t3
-t2
)/1000, (t4
-t3
)/1000,
2315 heap0
>>20, heap1
>>20, obj
,
2316 mstats
.nmalloc
, mstats
.nfree
,
2317 sweep
.nspan
, gcstats
.nbgsweep
, gcstats
.npausesweep
,
2318 stats
.nhandoff
, stats
.nhandoffcnt
,
2319 work
.markfor
->nsteal
, work
.markfor
->nstealcnt
,
2320 stats
.nprocyield
, stats
.nosyield
, stats
.nsleep
);
2321 gcstats
.nbgsweep
= gcstats
.npausesweep
= 0;
2323 runtime_printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n",
2324 gcstats
.nbytes
, gcstats
.obj
.cnt
, gcstats
.obj
.notype
, gcstats
.obj
.typelookup
);
2325 if(gcstats
.ptr
.cnt
!= 0)
2326 runtime_printf("avg ptrbufsize: %D (%D/%D)\n",
2327 gcstats
.ptr
.sum
/gcstats
.ptr
.cnt
, gcstats
.ptr
.sum
, gcstats
.ptr
.cnt
);
2328 if(gcstats
.obj
.cnt
!= 0)
2329 runtime_printf("avg nobj: %D (%D/%D)\n",
2330 gcstats
.obj
.sum
/gcstats
.obj
.cnt
, gcstats
.obj
.sum
, gcstats
.obj
.cnt
);
2331 runtime_printf("rescans: %D, %D bytes\n", gcstats
.rescan
, gcstats
.rescanbytes
);
2333 runtime_printf("instruction counts:\n");
2335 for(i
=0; i
<nelem(gcstats
.instr
); i
++) {
2336 runtime_printf("\t%d:\t%D\n", i
, gcstats
.instr
[i
]);
2337 ninstr
+= gcstats
.instr
[i
];
2339 runtime_printf("\ttotal:\t%D\n", ninstr
);
2341 runtime_printf("putempty: %D, getfull: %D\n", gcstats
.putempty
, gcstats
.getfull
);
2343 runtime_printf("markonly base lookup: bit %D word %D span %D\n", gcstats
.markonly
.foundbit
, gcstats
.markonly
.foundword
, gcstats
.markonly
.foundspan
);
2344 runtime_printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats
.flushptrbuf
.foundbit
, gcstats
.flushptrbuf
.foundword
, gcstats
.flushptrbuf
.foundspan
);
2348 // We cache current runtime_mheap.allspans array in sweep.spans,
2349 // because the former can be resized and freed.
2350 // Otherwise we would need to take heap lock every time
2351 // we want to convert span index to span pointer.
2353 // Free the old cached array if necessary.
2354 if(sweep
.spans
&& sweep
.spans
!= runtime_mheap
.allspans
)
2355 runtime_SysFree(sweep
.spans
, sweep
.nspan
*sizeof(sweep
.spans
[0]), &mstats
.other_sys
);
2356 // Cache the current array.
2357 runtime_mheap
.sweepspans
= runtime_mheap
.allspans
;
2358 runtime_mheap
.sweepgen
+= 2;
2359 runtime_mheap
.sweepdone
= false;
2360 sweep
.spans
= runtime_mheap
.allspans
;
2361 sweep
.nspan
= runtime_mheap
.nspan
;
2364 // Temporary disable concurrent sweep, because we see failures on builders.
2365 if(ConcurrentSweep
&& !args
->eagersweep
) {
2366 runtime_lock(&gclock
);
2368 sweep
.g
= __go_go(bgsweep
, nil
);
2369 else if(sweep
.parked
) {
2370 sweep
.parked
= false;
2371 runtime_ready(sweep
.g
);
2373 runtime_unlock(&gclock
);
2375 // Sweep all spans eagerly.
2376 while(runtime_sweepone() != (uintptr
)-1)
2377 gcstats
.npausesweep
++;
2378 // Do an additional mProf_GC, because all 'free' events are now real as well.
2386 extern uintptr runtime_sizeof_C_MStats
2387 __asm__ (GOSYM_PREFIX
"runtime.Sizeof_C_MStats");
2389 void runtime_ReadMemStats(MStats
*)
2390 __asm__ (GOSYM_PREFIX
"runtime.ReadMemStats");
2393 runtime_ReadMemStats(MStats
*stats
)
2397 // Have to acquire worldsema to stop the world,
2398 // because stoptheworld can only be used by
2399 // one goroutine at a time, and there might be
2400 // a pending garbage collection already calling it.
2401 runtime_semacquire(&runtime_worldsema
, false);
2404 runtime_stoptheworld();
2405 runtime_updatememstats(nil
);
2406 // Size of the trailing by_size array differs between Go and C,
2407 // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
2408 runtime_memmove(stats
, &mstats
, runtime_sizeof_C_MStats
);
2411 runtime_semrelease(&runtime_worldsema
);
2412 runtime_starttheworld();
2416 void runtime_debug_readGCStats(Slice
*)
2417 __asm__("runtime_debug.readGCStats");
2420 runtime_debug_readGCStats(Slice
*pauses
)
2425 // Calling code in runtime/debug should make the slice large enough.
2426 if((size_t)pauses
->cap
< nelem(mstats
.pause_ns
)+3)
2427 runtime_throw("runtime: short slice passed to readGCStats");
2429 // Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
2430 p
= (uint64
*)pauses
->array
;
2431 runtime_lock(&runtime_mheap
);
2433 if(n
> nelem(mstats
.pause_ns
))
2434 n
= nelem(mstats
.pause_ns
);
2436 // The pause buffer is circular. The most recent pause is at
2437 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
2438 // from there to go back farther in time. We deliver the times
2439 // most recent first (in p[0]).
2441 p
[i
] = mstats
.pause_ns
[(mstats
.numgc
-1-i
)%nelem(mstats
.pause_ns
)];
2443 p
[n
] = mstats
.last_gc
;
2444 p
[n
+1] = mstats
.numgc
;
2445 p
[n
+2] = mstats
.pause_total_ns
;
2446 runtime_unlock(&runtime_mheap
);
2447 pauses
->__count
= n
+3;
2451 runtime_setgcpercent(int32 in
) {
2454 runtime_lock(&runtime_mheap
);
2455 if(gcpercent
== GcpercentUnknown
)
2456 gcpercent
= readgogc();
2461 runtime_unlock(&runtime_mheap
);
2471 if(m
->helpgc
< 0 || m
->helpgc
>= MaxGcproc
)
2472 runtime_throw("gchelperstart: bad m->helpgc");
2473 if(runtime_xchg(&bufferList
[m
->helpgc
].busy
, 1))
2474 runtime_throw("gchelperstart: already busy");
2475 if(runtime_g() != m
->g0
)
2476 runtime_throw("gchelper not running on g0 stack");
2480 runfinq(void* dummy
__attribute__ ((unused
)))
2483 FinBlock
*fb
, *next
;
2488 // This function blocks for long periods of time, and because it is written in C
2489 // we have no liveness information. Zero everything so that uninitialized pointers
2490 // do not cause memory leaks.
2495 ef
.__type_descriptor
= nil
;
2498 // force flush to memory
2506 runtime_lock(&finlock
);
2510 runtime_fingwait
= true;
2511 runtime_g()->isbackground
= true;
2512 runtime_parkunlock(&finlock
, "finalizer wait");
2513 runtime_g()->isbackground
= false;
2516 runtime_unlock(&finlock
);
2517 for(; fb
; fb
=next
) {
2519 for(i
=0; i
<(uint32
)fb
->cnt
; i
++) {
2524 fint
= ((const Type
**)f
->ft
->__in
.array
)[0];
2525 if((fint
->__code
& kindMask
) == KindPtr
) {
2526 // direct use of pointer
2528 } else if(((const InterfaceType
*)fint
)->__methods
.__count
== 0) {
2529 // convert to empty interface
2530 ef
.__type_descriptor
= (const Type
*)f
->ot
;
2531 ef
.__object
= f
->arg
;
2534 // convert to interface with methods
2535 iface
.__methods
= __go_convert_interface_2((const Type
*)fint
,
2538 iface
.__object
= f
->arg
;
2539 if(iface
.__methods
== nil
)
2540 runtime_throw("invalid type conversion in runfinq");
2543 reflect_call(f
->ft
, f
->fn
, 0, 0, ¶m
, nil
);
2549 runtime_lock(&finlock
);
2552 runtime_unlock(&finlock
);
2555 // Zero everything that's dead, to avoid memory leaks.
2556 // See comment at top of function.
2561 ef
.__type_descriptor
= nil
;
2563 runtime_gc(1); // trigger another gc to clean up the finalized objects, if possible
2568 runtime_createfing(void)
2572 // Here we use gclock instead of finlock,
2573 // because newproc1 can allocate, which can cause on-demand span sweep,
2574 // which can queue finalizers, which would deadlock.
2575 runtime_lock(&gclock
);
2577 fing
= __go_go(runfinq
, nil
);
2578 runtime_unlock(&gclock
);
2582 runtime_wakefing(void)
2587 runtime_lock(&finlock
);
2588 if(runtime_fingwait
&& runtime_fingwake
) {
2589 runtime_fingwait
= false;
2590 runtime_fingwake
= false;
2593 runtime_unlock(&finlock
);
2598 runtime_marknogc(void *v
)
2600 uintptr
*b
, off
, shift
;
2602 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2603 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2604 shift
= off
% wordsPerBitmapWord
;
2605 *b
= (*b
& ~(bitAllocated
<<shift
)) | bitBlockBoundary
<<shift
;
2609 runtime_markscan(void *v
)
2611 uintptr
*b
, off
, shift
;
2613 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2614 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2615 shift
= off
% wordsPerBitmapWord
;
2616 *b
|= bitScan
<<shift
;
2619 // mark the block at v as freed.
2621 runtime_markfreed(void *v
)
2623 uintptr
*b
, off
, shift
;
2626 runtime_printf("markfreed %p\n", v
);
2628 if((byte
*)v
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2629 runtime_throw("markfreed: bad pointer");
2631 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2632 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2633 shift
= off
% wordsPerBitmapWord
;
2634 *b
= (*b
& ~(bitMask
<<shift
)) | (bitAllocated
<<shift
);
2637 // check that the block at v of size n is marked freed.
2639 runtime_checkfreed(void *v
, uintptr n
)
2641 uintptr
*b
, bits
, off
, shift
;
2643 if(!runtime_checking
)
2646 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2647 return; // not allocated, so okay
2649 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2650 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2651 shift
= off
% wordsPerBitmapWord
;
2654 if((bits
& bitAllocated
) != 0) {
2655 runtime_printf("checkfreed %p+%p: off=%p have=%p\n",
2656 v
, n
, off
, bits
& bitMask
);
2657 runtime_throw("checkfreed: not freed");
2661 // mark the span of memory at v as having n blocks of the given size.
2662 // if leftover is true, there is left over space at the end of the span.
2664 runtime_markspan(void *v
, uintptr size
, uintptr n
, bool leftover
)
2666 uintptr
*b
, *b0
, off
, shift
, i
, x
;
2669 if((byte
*)v
+size
*n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2670 runtime_throw("markspan: bad pointer");
2672 if(runtime_checking
) {
2673 // bits should be all zero at the start
2674 off
= (byte
*)v
+ size
- runtime_mheap
.arena_start
;
2675 b
= (uintptr
*)(runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
);
2676 for(i
= 0; i
< size
/PtrSize
/wordsPerBitmapWord
; i
++) {
2678 runtime_throw("markspan: span bits not zero");
2683 if(leftover
) // mark a boundary just past end of last block too
2688 for(; n
-- > 0; p
+= size
) {
2689 // Okay to use non-atomic ops here, because we control
2690 // the entire span, and each bitmap word has bits for only
2691 // one span, so no other goroutines are changing these
2693 off
= (uintptr
*)p
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2694 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2695 shift
= off
% wordsPerBitmapWord
;
2702 x
|= bitAllocated
<<shift
;
2707 // unmark the span of memory at v of length n bytes.
2709 runtime_unmarkspan(void *v
, uintptr n
)
2711 uintptr
*p
, *b
, off
;
2713 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2714 runtime_throw("markspan: bad pointer");
2717 off
= p
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2718 if(off
% wordsPerBitmapWord
!= 0)
2719 runtime_throw("markspan: unaligned pointer");
2720 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2722 if(n
%wordsPerBitmapWord
!= 0)
2723 runtime_throw("unmarkspan: unaligned length");
2724 // Okay to use non-atomic ops here, because we control
2725 // the entire span, and each bitmap word has bits for only
2726 // one span, so no other goroutines are changing these
2728 n
/= wordsPerBitmapWord
;
2734 runtime_MHeap_MapBits(MHeap
*h
)
2738 // Caller has added extra mappings to the arena.
2739 // Add extra mappings of bitmap words as needed.
2740 // We allocate extra bitmap pieces in chunks of bitmapChunk.
2746 n
= (h
->arena_used
- h
->arena_start
) / wordsPerBitmapWord
;
2747 n
= ROUND(n
, bitmapChunk
);
2748 n
= ROUND(n
, PageSize
);
2749 page_size
= getpagesize();
2750 n
= ROUND(n
, page_size
);
2751 if(h
->bitmap_mapped
>= n
)
2754 runtime_SysMap(h
->arena_start
- n
, n
- h
->bitmap_mapped
, h
->arena_reserved
, &mstats
.gc_sys
);
2755 h
->bitmap_mapped
= n
;
2758 // typedmemmove copies a value of type t to dst from src.
2760 extern void typedmemmove(const Type
* td
, void *dst
, const void *src
)
2761 __asm__ (GOSYM_PREFIX
"reflect.typedmemmove");
2764 typedmemmove(const Type
* td
, void *dst
, const void *src
)
2766 runtime_memmove(dst
, src
, td
->__size
);
2769 // typedslicecopy copies a slice of elemType values from src to dst,
2770 // returning the number of elements copied.
2772 extern intgo
typedslicecopy(const Type
* elem
, Slice dst
, Slice src
)
2773 __asm__ (GOSYM_PREFIX
"reflect.typedslicecopy");
2776 typedslicecopy(const Type
* elem
, Slice dst
, Slice src
)
2783 if (n
> src
.__count
)
2787 dstp
= dst
.__values
;
2788 srcp
= src
.__values
;
2789 memmove(dstp
, srcp
, (uintptr_t)n
* elem
->__size
);