1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
16 // Map gccgo field names to gc field names.
17 // Slice aka __go_open_array.
18 #define array __values
19 #define cap __capacity
20 // Iface aka __go_interface
22 // Eface aka __go_empty_interface.
23 #define type __type_descriptor
25 typedef struct __go_map Hmap
;
26 // Type aka __go_type_descriptor
28 #define string __reflection
29 #define KindPtr GO_PTR
30 #define KindNoPointers GO_NO_POINTERS
31 // PtrType aka __go_ptr_type
32 #define elem __element_type
34 #ifdef USING_SPLIT_STACK
36 extern void * __splitstack_find (void *, void *, size_t *, void **, void **,
39 extern void * __splitstack_find_context (void *context
[10], size_t *, void **,
46 DebugMark
= 0, // run second pass to check mark
48 ScanStackByFrames
= 0,
51 // Four bits per word (see #defines below).
52 wordsPerBitmapWord
= sizeof(void*)*8/4,
53 bitShift
= sizeof(void*)*8/4,
56 IntermediateBufferCapacity
= 64,
58 // Bits in type information
61 PC_BITS
= PRECISE
| LOOP
,
71 // Bits in per-word bitmap.
72 // #defines because enum might not be able to hold the values.
74 // Each word in the bitmap describes wordsPerBitmapWord words
75 // of heap memory. There are 4 bitmap bits dedicated to each heap word,
76 // so on a 64-bit system there is one bitmap word per 16 heap words.
77 // The bits in the word are packed together by type first, then by
78 // heap location, so each 64-bit bitmap word consists of, from top to bottom,
79 // the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
80 // then the 16 bitNoScan/bitBlockBoundary bits, then the 16 bitAllocated bits.
81 // This layout makes it easier to iterate over the bits of a given type.
83 // The bitmap starts at mheap.arena_start and extends *backward* from
84 // there. On a 64-bit system the off'th word in the arena is tracked by
85 // the off/16+1'th word before mheap.arena_start. (On a 32-bit system,
86 // the only difference is that the divisor is 8.)
88 // To pull out the bits corresponding to a given pointer p, we use:
90 // off = p - (uintptr*)mheap.arena_start; // word offset
91 // b = (uintptr*)mheap.arena_start - off/wordsPerBitmapWord - 1;
92 // shift = off % wordsPerBitmapWord
93 // bits = *b >> shift;
94 // /* then test bits & bitAllocated, bits & bitMarked, etc. */
96 #define bitAllocated ((uintptr)1<<(bitShift*0))
97 #define bitNoScan ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
98 #define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
99 #define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
100 #define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */
102 #define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial)
104 // Holding worldsema grants an M the right to try to stop the world.
107 // runtime_semacquire(&runtime_worldsema);
109 // runtime_stoptheworld();
114 // runtime_semrelease(&runtime_worldsema);
115 // runtime_starttheworld();
117 uint32 runtime_worldsema
= 1;
119 // The size of Workbuf is N*PageSize.
120 typedef struct Workbuf Workbuf
;
123 #define SIZE (2*PageSize-sizeof(LFNode)-sizeof(uintptr))
124 LFNode node
; // must be first
126 Obj obj
[SIZE
/sizeof(Obj
) - 1];
127 uint8 _padding
[SIZE
%sizeof(Obj
) + sizeof(Obj
)];
131 typedef struct Finalizer Finalizer
;
136 const struct __go_func_type
*ft
;
137 const struct __go_ptr_type
*ot
;
140 typedef struct FinBlock FinBlock
;
151 static FinBlock
*finq
; // list of finalizers that are to be executed
152 static FinBlock
*finc
; // cache of free blocks
153 static FinBlock
*allfin
; // list of all blocks
155 static int32 fingwait
;
157 static void runfinq(void*);
158 static Workbuf
* getempty(Workbuf
*);
159 static Workbuf
* getfull(Workbuf
*);
160 static void putempty(Workbuf
*);
161 static Workbuf
* handoff(Workbuf
*);
162 static void gchelperstart(void);
165 uint64 full
; // lock-free list of full blocks
166 uint64 empty
; // lock-free list of empty blocks
167 byte pad0
[CacheLineSize
]; // prevents false-sharing between full/empty and nproc/nwait
169 volatile uint32 nwait
;
170 volatile uint32 ndone
;
171 volatile uint32 debugmarkdone
;
183 } work
__attribute__((aligned(8)));
186 GC_DEFAULT_PTR
= GC_NUM_INSTR
,
206 uint64 instr
[GC_NUM_INSTR2
];
221 // markonly marks an object. It returns true if the object
222 // has been marked by this function, false otherwise.
223 // This function doesn't append the object to any buffer.
228 uintptr
*bitp
, bits
, shift
, x
, xbits
, off
, j
;
232 // Words outside the arena cannot be pointers.
233 if((byte
*)obj
< runtime_mheap
.arena_start
|| (byte
*)obj
>= runtime_mheap
.arena_used
)
236 // obj may be a pointer to a live object.
237 // Try to find the beginning of the object.
239 // Round down to word boundary.
240 obj
= (void*)((uintptr
)obj
& ~((uintptr
)PtrSize
-1));
242 // Find bits for this word.
243 off
= (uintptr
*)obj
- (uintptr
*)runtime_mheap
.arena_start
;
244 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
245 shift
= off
% wordsPerBitmapWord
;
247 bits
= xbits
>> shift
;
249 // Pointing at the beginning of a block?
250 if((bits
& (bitAllocated
|bitBlockBoundary
)) != 0) {
252 runtime_xadd64(&gcstats
.markonly
.foundbit
, 1);
256 // Pointing just past the beginning?
257 // Scan backward a little to find a block boundary.
258 for(j
=shift
; j
-->0; ) {
259 if(((xbits
>>j
) & (bitAllocated
|bitBlockBoundary
)) != 0) {
263 runtime_xadd64(&gcstats
.markonly
.foundword
, 1);
268 // Otherwise consult span table to find beginning.
269 // (Manually inlined copy of MHeap_LookupMaybe.)
270 k
= (uintptr
)obj
>>PageShift
;
272 x
-= (uintptr
)runtime_mheap
.arena_start
>>PageShift
;
273 s
= runtime_mheap
.spans
[x
];
274 if(s
== nil
|| k
< s
->start
|| (byte
*)obj
>= s
->limit
|| s
->state
!= MSpanInUse
)
276 p
= (byte
*)((uintptr
)s
->start
<<PageShift
);
277 if(s
->sizeclass
== 0) {
280 uintptr size
= s
->elemsize
;
281 int32 i
= ((byte
*)obj
- p
)/size
;
285 // Now that we know the object header, reload bits.
286 off
= (uintptr
*)obj
- (uintptr
*)runtime_mheap
.arena_start
;
287 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
288 shift
= off
% wordsPerBitmapWord
;
290 bits
= xbits
>> shift
;
292 runtime_xadd64(&gcstats
.markonly
.foundspan
, 1);
295 // Now we have bits, bitp, and shift correct for
296 // obj pointing at the base of the object.
297 // Only care about allocated and not marked.
298 if((bits
& (bitAllocated
|bitMarked
)) != bitAllocated
)
301 *bitp
|= bitMarked
<<shift
;
305 if(x
& (bitMarked
<<shift
))
307 if(runtime_casp((void**)bitp
, (void*)x
, (void*)(x
|(bitMarked
<<shift
))))
312 // The object is now marked
316 // PtrTarget is a structure used by intermediate buffers.
317 // The intermediate buffers hold GC data before it
318 // is moved/flushed to the work buffer (Workbuf).
319 // The size of an intermediate buffer is very small,
320 // such as 32 or 64 elements.
321 typedef struct PtrTarget PtrTarget
;
328 typedef struct BufferList BufferList
;
331 PtrTarget ptrtarget
[IntermediateBufferCapacity
];
332 Obj obj
[IntermediateBufferCapacity
];
334 byte pad
[CacheLineSize
];
336 static BufferList bufferList
[MaxGcproc
];
338 static Type
*itabtype
;
340 static void enqueue(Obj obj
, Workbuf
**_wbuf
, Obj
**_wp
, uintptr
*_nobj
);
342 // flushptrbuf moves data from the PtrTarget buffer to the work buffer.
343 // The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
344 // while the work buffer contains blocks which have been marked
345 // and are prepared to be scanned by the garbage collector.
347 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
349 // A simplified drawing explaining how the todo-list moves from a structure to another:
353 // Obj ------> PtrTarget (pointer targets)
358 // (find block start, mark and enqueue)
360 flushptrbuf(PtrTarget
*ptrbuf
, PtrTarget
**ptrbufpos
, Obj
**_wp
, Workbuf
**_wbuf
, uintptr
*_nobj
)
362 byte
*p
, *arena_start
, *obj
;
363 uintptr size
, *bitp
, bits
, shift
, j
, x
, xbits
, off
, nobj
, ti
, n
;
368 PtrTarget
*ptrbuf_end
;
370 arena_start
= runtime_mheap
.arena_start
;
376 ptrbuf_end
= *ptrbufpos
;
377 n
= ptrbuf_end
- ptrbuf
;
381 runtime_xadd64(&gcstats
.ptr
.sum
, n
);
382 runtime_xadd64(&gcstats
.ptr
.cnt
, 1);
385 // If buffer is nearly full, get a new one.
386 if(wbuf
== nil
|| nobj
+n
>= nelem(wbuf
->obj
)) {
389 wbuf
= getempty(wbuf
);
393 if(n
>= nelem(wbuf
->obj
))
394 runtime_throw("ptrbuf has to be smaller than WorkBuf");
397 // TODO(atom): This block is a branch of an if-then-else statement.
398 // The single-threaded branch may be added in a next CL.
400 // Multi-threaded version.
402 while(ptrbuf
< ptrbuf_end
) {
407 // obj belongs to interval [mheap.arena_start, mheap.arena_used).
409 if(obj
< runtime_mheap
.arena_start
|| obj
>= runtime_mheap
.arena_used
)
410 runtime_throw("object is outside of mheap");
413 // obj may be a pointer to a live object.
414 // Try to find the beginning of the object.
416 // Round down to word boundary.
417 if(((uintptr
)obj
& ((uintptr
)PtrSize
-1)) != 0) {
418 obj
= (void*)((uintptr
)obj
& ~((uintptr
)PtrSize
-1));
422 // Find bits for this word.
423 off
= (uintptr
*)obj
- (uintptr
*)arena_start
;
424 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
425 shift
= off
% wordsPerBitmapWord
;
427 bits
= xbits
>> shift
;
429 // Pointing at the beginning of a block?
430 if((bits
& (bitAllocated
|bitBlockBoundary
)) != 0) {
432 runtime_xadd64(&gcstats
.flushptrbuf
.foundbit
, 1);
438 // Pointing just past the beginning?
439 // Scan backward a little to find a block boundary.
440 for(j
=shift
; j
-->0; ) {
441 if(((xbits
>>j
) & (bitAllocated
|bitBlockBoundary
)) != 0) {
442 obj
= (byte
*)obj
- (shift
-j
)*PtrSize
;
446 runtime_xadd64(&gcstats
.flushptrbuf
.foundword
, 1);
451 // Otherwise consult span table to find beginning.
452 // (Manually inlined copy of MHeap_LookupMaybe.)
453 k
= (uintptr
)obj
>>PageShift
;
455 x
-= (uintptr
)arena_start
>>PageShift
;
456 s
= runtime_mheap
.spans
[x
];
457 if(s
== nil
|| k
< s
->start
|| obj
>= s
->limit
|| s
->state
!= MSpanInUse
)
459 p
= (byte
*)((uintptr
)s
->start
<<PageShift
);
460 if(s
->sizeclass
== 0) {
464 int32 i
= ((byte
*)obj
- p
)/size
;
468 // Now that we know the object header, reload bits.
469 off
= (uintptr
*)obj
- (uintptr
*)arena_start
;
470 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
471 shift
= off
% wordsPerBitmapWord
;
473 bits
= xbits
>> shift
;
475 runtime_xadd64(&gcstats
.flushptrbuf
.foundspan
, 1);
478 // Now we have bits, bitp, and shift correct for
479 // obj pointing at the base of the object.
480 // Only care about allocated and not marked.
481 if((bits
& (bitAllocated
|bitMarked
)) != bitAllocated
)
484 *bitp
|= bitMarked
<<shift
;
488 if(x
& (bitMarked
<<shift
))
490 if(runtime_casp((void**)bitp
, (void*)x
, (void*)(x
|(bitMarked
<<shift
))))
495 // If object has no pointers, don't need to scan further.
496 if((bits
& bitNoScan
) != 0)
499 // Ask span about size class.
500 // (Manually inlined copy of MHeap_Lookup.)
501 x
= (uintptr
)obj
>> PageShift
;
502 x
-= (uintptr
)arena_start
>>PageShift
;
503 s
= runtime_mheap
.spans
[x
];
507 *wp
= (Obj
){obj
, s
->elemsize
, ti
};
513 // If another proc wants a pointer, give it some.
514 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
516 wbuf
= handoff(wbuf
);
518 wp
= wbuf
->obj
+ nobj
;
528 flushobjbuf(Obj
*objbuf
, Obj
**objbufpos
, Obj
**_wp
, Workbuf
**_wbuf
, uintptr
*_nobj
)
539 objbuf_end
= *objbufpos
;
542 while(objbuf
< objbuf_end
) {
545 // Align obj.b to a word boundary.
546 off
= (uintptr
)obj
.p
& (PtrSize
-1);
548 obj
.p
+= PtrSize
- off
;
549 obj
.n
-= PtrSize
- off
;
553 if(obj
.p
== nil
|| obj
.n
== 0)
556 // If buffer is full, get a new one.
557 if(wbuf
== nil
|| nobj
>= nelem(wbuf
->obj
)) {
560 wbuf
= getempty(wbuf
);
570 // If another proc wants a pointer, give it some.
571 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
573 wbuf
= handoff(wbuf
);
575 wp
= wbuf
->obj
+ nobj
;
583 // Program that scans the whole block and treats every block element as a potential pointer
584 static uintptr defaultProg
[2] = {PtrSize
, GC_DEFAULT_PTR
};
588 static uintptr chanProg
[2] = {0, GC_CHAN
};
591 // Local variables of a program fragment or loop
592 typedef struct Frame Frame
;
594 uintptr count
, elemsize
, b
;
595 uintptr
*loop_or_ret
;
598 // Sanity check for the derived type info objti.
600 checkptr(void *obj
, uintptr objti
)
602 uintptr type
, tisize
, i
, x
;
608 runtime_throw("checkptr is debug only");
610 if((byte
*)obj
< runtime_mheap
.arena_start
|| (byte
*)obj
>= runtime_mheap
.arena_used
)
612 type
= runtime_gettype(obj
);
613 t
= (Type
*)(type
& ~(uintptr
)(PtrSize
-1));
616 x
= (uintptr
)obj
>> PageShift
;
617 x
-= (uintptr
)(runtime_mheap
.arena_start
)>>PageShift
;
618 s
= runtime_mheap
.spans
[x
];
619 objstart
= (byte
*)((uintptr
)s
->start
<<PageShift
);
620 if(s
->sizeclass
!= 0) {
621 i
= ((byte
*)obj
- objstart
)/s
->elemsize
;
622 objstart
+= i
*s
->elemsize
;
624 tisize
= *(uintptr
*)objti
;
625 // Sanity check for object size: it should fit into the memory block.
626 if((byte
*)obj
+ tisize
> objstart
+ s
->elemsize
) {
627 runtime_printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
628 *t
->string
, obj
, tisize
, objstart
, s
->elemsize
);
629 runtime_throw("invalid gc type info");
633 // If obj points to the beginning of the memory block,
634 // check type info as well.
635 if(t
->string
== nil
||
636 // Gob allocates unsafe pointers for indirection.
637 (runtime_strcmp((const char *)t
->string
->str
, (const char*)"unsafe.Pointer") &&
638 // Runtime and gc think differently about closures.
639 runtime_strstr((const char *)t
->string
->str
, (const char*)"struct { F uintptr") != (const char *)t
->string
->str
)) {
641 pc1
= (uintptr
*)objti
;
642 pc2
= (uintptr
*)t
->gc
;
643 // A simple best-effort check until first GC_END.
644 for(j
= 1; pc1
[j
] != GC_END
&& pc2
[j
] != GC_END
; j
++) {
645 if(pc1
[j
] != pc2
[j
]) {
646 runtime_printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n",
647 t
->string
? (const int8
*)t
->string
->str
: (const int8
*)"?", j
, pc1
[j
], pc2
[j
]);
648 runtime_throw("invalid gc type info");
655 // scanblock scans a block of n bytes starting at pointer b for references
656 // to other objects, scanning any it finds recursively until there are no
657 // unscanned objects left. Instead of using an explicit recursion, it keeps
658 // a work list in the Workbuf* structures and loops in the main function
659 // body. Keeping an explicit work list is easier on the stack allocator and
662 // wbuf: current work buffer
663 // wp: storage for next queued pointer (write pointer)
664 // nobj: number of queued objects
666 scanblock(Workbuf
*wbuf
, Obj
*wp
, uintptr nobj
, bool keepworking
)
668 byte
*b
, *arena_start
, *arena_used
;
669 uintptr n
, i
, end_b
, elemsize
, size
, ti
, objti
, count
/* , type */;
670 uintptr
*pc
, precise_type
, nominal_size
;
672 uintptr
*chan_ret
, chancap
;
677 Frame
*stack_ptr
, stack_top
, stack
[GC_STACK_CAPACITY
+4];
678 BufferList
*scanbuffers
;
679 PtrTarget
*ptrbuf
, *ptrbuf_end
, *ptrbufpos
;
680 Obj
*objbuf
, *objbuf_end
, *objbufpos
;
688 if(sizeof(Workbuf
) % PageSize
!= 0)
689 runtime_throw("scanblock: size of Workbuf is suboptimal");
691 // Memory arena parameters.
692 arena_start
= runtime_mheap
.arena_start
;
693 arena_used
= runtime_mheap
.arena_used
;
695 stack_ptr
= stack
+nelem(stack
)-1;
697 precise_type
= false;
702 scanbuffers
= &bufferList
[runtime_m()->helpgc
];
703 ptrbuf
= &scanbuffers
->ptrtarget
[0];
704 ptrbuf_end
= &scanbuffers
->ptrtarget
[0] + nelem(scanbuffers
->ptrtarget
);
705 objbuf
= &scanbuffers
->obj
[0];
706 objbuf_end
= &scanbuffers
->obj
[0] + nelem(scanbuffers
->obj
);
712 // (Silence the compiler)
722 // Each iteration scans the block b of length n, queueing pointers in
725 runtime_printf("scanblock %p %D\n", b
, (int64
)n
);
729 runtime_xadd64(&gcstats
.nbytes
, n
);
730 runtime_xadd64(&gcstats
.obj
.sum
, nobj
);
731 runtime_xadd64(&gcstats
.obj
.cnt
, 1);
734 if(ti
!= 0 && false) {
735 pc
= (uintptr
*)(ti
& ~(uintptr
)PC_BITS
);
736 precise_type
= (ti
& PRECISE
);
737 stack_top
.elemsize
= pc
[0];
739 nominal_size
= pc
[0];
741 stack_top
.count
= 0; // 0 means an infinite number of iterations
742 stack_top
.loop_or_ret
= pc
+1;
747 // Simple sanity check for provided type info ti:
748 // The declared size of the object must be not larger than the actual size
749 // (it can be smaller due to inferior pointers).
750 // It's difficult to make a comprehensive check due to inferior pointers,
751 // reflection, gob, etc.
753 runtime_printf("invalid gc type info: type info size %p, block size %p\n", pc
[0], n
);
754 runtime_throw("invalid gc type info");
757 } else if(UseSpanType
&& false) {
759 runtime_xadd64(&gcstats
.obj
.notype
, 1);
762 type
= runtime_gettype(b
);
765 runtime_xadd64(&gcstats
.obj
.typelookup
, 1);
767 t
= (Type
*)(type
& ~(uintptr
)(PtrSize
-1));
768 switch(type
& (PtrSize
-1)) {
769 case TypeInfo_SingleObject
:
770 pc
= (uintptr
*)t
->gc
;
771 precise_type
= true; // type information about 'b' is precise
773 stack_top
.elemsize
= pc
[0];
776 pc
= (uintptr
*)t
->gc
;
779 precise_type
= true; // type information about 'b' is precise
780 stack_top
.count
= 0; // 0 means an infinite number of iterations
781 stack_top
.elemsize
= pc
[0];
782 stack_top
.loop_or_ret
= pc
+1;
786 chantype
= (ChanType
*)t
;
791 runtime_throw("scanblock: invalid type");
806 stack_top
.b
= (uintptr
)b
;
808 end_b
= (uintptr
)b
+ n
- PtrSize
;
812 runtime_xadd64(&gcstats
.instr
[pc
[0]], 1);
818 obj
= *(void**)(stack_top
.b
+ pc
[1]);
822 checkptr(obj
, objti
);
826 sliceptr
= (Slice
*)(stack_top
.b
+ pc
[1]);
827 if(sliceptr
->cap
!= 0) {
828 obj
= sliceptr
->array
;
829 // Can't use slice element type for scanning,
830 // because if it points to an array embedded
831 // in the beginning of a struct,
832 // we will scan the whole struct as the slice.
833 // So just obtain type info from heap.
839 obj
= *(void**)(stack_top
.b
+ pc
[1]);
844 obj
= *(void**)(stack_top
.b
+ pc
[1]);
850 eface
= (Eface
*)(stack_top
.b
+ pc
[1]);
852 if(eface
->type
== nil
)
857 if((const byte
*)t
>= arena_start
&& (const byte
*)t
< arena_used
) {
858 union { const Type
*tc
; Type
*tr
; } u
;
860 *ptrbufpos
++ = (struct PtrTarget
){(void*)u
.tr
, 0};
861 if(ptrbufpos
== ptrbuf_end
)
862 flushptrbuf(ptrbuf
, &ptrbufpos
, &wp
, &wbuf
, &nobj
);
866 if((byte
*)eface
->__object
>= arena_start
&& (byte
*)eface
->__object
< arena_used
) {
867 if(t
->__size
<= sizeof(void*)) {
868 if((t
->kind
& KindNoPointers
))
871 obj
= eface
->__object
;
872 if((t
->kind
& ~KindNoPointers
) == KindPtr
)
873 // objti = (uintptr)((PtrType*)t)->elem->gc;
876 obj
= eface
->__object
;
877 // objti = (uintptr)t->gc;
884 iface
= (Iface
*)(stack_top
.b
+ pc
[1]);
886 if(iface
->tab
== nil
)
890 if((byte
*)iface
->tab
>= arena_start
&& (byte
*)iface
->tab
< arena_used
) {
891 // *ptrbufpos++ = (struct PtrTarget){iface->tab, (uintptr)itabtype->gc};
892 *ptrbufpos
++ = (struct PtrTarget
){iface
->tab
, 0};
893 if(ptrbufpos
== ptrbuf_end
)
894 flushptrbuf(ptrbuf
, &ptrbufpos
, &wp
, &wbuf
, &nobj
);
898 if((byte
*)iface
->__object
>= arena_start
&& (byte
*)iface
->__object
< arena_used
) {
899 // t = iface->tab->type;
901 if(t
->__size
<= sizeof(void*)) {
902 if((t
->kind
& KindNoPointers
))
905 obj
= iface
->__object
;
906 if((t
->kind
& ~KindNoPointers
) == KindPtr
)
907 // objti = (uintptr)((const PtrType*)t)->elem->gc;
910 obj
= iface
->__object
;
911 // objti = (uintptr)t->gc;
918 while(stack_top
.b
<= end_b
) {
919 obj
= *(byte
**)stack_top
.b
;
920 stack_top
.b
+= PtrSize
;
921 if((byte
*)obj
>= arena_start
&& (byte
*)obj
< arena_used
) {
922 *ptrbufpos
++ = (struct PtrTarget
){obj
, 0};
923 if(ptrbufpos
== ptrbuf_end
)
924 flushptrbuf(ptrbuf
, &ptrbufpos
, &wp
, &wbuf
, &nobj
);
930 if(--stack_top
.count
!= 0) {
931 // Next iteration of a loop if possible.
932 stack_top
.b
+= stack_top
.elemsize
;
933 if(stack_top
.b
+ stack_top
.elemsize
<= end_b
+PtrSize
) {
934 pc
= stack_top
.loop_or_ret
;
939 // Stack pop if possible.
940 if(stack_ptr
+1 < stack
+nelem(stack
)) {
941 pc
= stack_top
.loop_or_ret
;
942 stack_top
= *(++stack_ptr
);
945 i
= (uintptr
)b
+ nominal_size
;
948 // Quickly scan [b+i,b+n) for possible pointers.
949 for(; i
<=end_b
; i
+=PtrSize
) {
950 if(*(byte
**)i
!= nil
) {
951 // Found a value that may be a pointer.
952 // Do a rescan of the entire block.
953 enqueue((Obj
){b
, n
, 0}, &wbuf
, &wp
, &nobj
);
955 runtime_xadd64(&gcstats
.rescan
, 1);
956 runtime_xadd64(&gcstats
.rescanbytes
, n
);
965 i
= stack_top
.b
+ pc
[1];
971 *stack_ptr
-- = stack_top
;
972 stack_top
= (Frame
){count
, elemsize
, i
, pc
};
976 if(--stack_top
.count
!= 0) {
977 stack_top
.b
+= stack_top
.elemsize
;
978 pc
= stack_top
.loop_or_ret
;
981 stack_top
= *(++stack_ptr
);
988 *stack_ptr
-- = stack_top
;
989 stack_top
= (Frame
){1, 0, stack_top
.b
+ pc
[1], pc
+3 /*return address*/};
990 pc
= (uintptr
*)((byte
*)pc
+ *(int32
*)(pc
+2)); // target of the CALL instruction
994 obj
= (void*)(stack_top
.b
+ pc
[1]);
999 *objbufpos
++ = (Obj
){obj
, size
, objti
};
1000 if(objbufpos
== objbuf_end
)
1001 flushobjbuf(objbuf
, &objbufpos
, &wp
, &wbuf
, &nobj
);
1006 chan
= *(Hchan
**)(stack_top
.b
+ pc
[1]);
1011 if(markonly(chan
)) {
1012 chantype
= (ChanType
*)pc
[2];
1013 if(!(chantype
->elem
->kind
& KindNoPointers
)) {
1024 // There are no heap pointers in struct Hchan,
1025 // so we can ignore the leading sizeof(Hchan) bytes.
1026 if(!(chantype
->elem
->kind
& KindNoPointers
)) {
1027 // Channel's buffer follows Hchan immediately in memory.
1028 // Size of buffer (cap(c)) is second int in the chan struct.
1029 chancap
= ((uintgo
*)chan
)[1];
1031 // TODO(atom): split into two chunks so that only the
1032 // in-use part of the circular buffer is scanned.
1033 // (Channel routines zero the unused part, so the current
1034 // code does not lead to leaks, it's just a little inefficient.)
1035 *objbufpos
++ = (Obj
){(byte
*)chan
+runtime_Hchansize
, chancap
*chantype
->elem
->size
,
1036 (uintptr
)chantype
->elem
->gc
| PRECISE
| LOOP
};
1037 if(objbufpos
== objbuf_end
)
1038 flushobjbuf(objbuf
, &objbufpos
, &wp
, &wbuf
, &nobj
);
1048 runtime_throw("scanblock: invalid GC instruction");
1052 if((byte
*)obj
>= arena_start
&& (byte
*)obj
< arena_used
) {
1053 *ptrbufpos
++ = (struct PtrTarget
){obj
, objti
};
1054 if(ptrbufpos
== ptrbuf_end
)
1055 flushptrbuf(ptrbuf
, &ptrbufpos
, &wp
, &wbuf
, &nobj
);
1060 // Done scanning [b, b+n). Prepare for the next iteration of
1061 // the loop by setting b, n, ti to the parameters for the next block.
1064 flushptrbuf(ptrbuf
, &ptrbufpos
, &wp
, &wbuf
, &nobj
);
1065 flushobjbuf(objbuf
, &objbufpos
, &wp
, &wbuf
, &nobj
);
1073 // Emptied our buffer: refill.
1074 wbuf
= getfull(wbuf
);
1078 wp
= wbuf
->obj
+ wbuf
->nobj
;
1082 // Fetch b from the work buffer.
1093 // debug_scanblock is the debug copy of scanblock.
1094 // it is simpler, slower, single-threaded, recursive,
1095 // and uses bitSpecial as the mark bit.
1097 debug_scanblock(byte
*b
, uintptr n
)
1101 uintptr size
, *bitp
, bits
, shift
, i
, xbits
, off
;
1105 runtime_throw("debug_scanblock without DebugMark");
1108 runtime_printf("debug_scanblock %p %D\n", b
, (int64
)n
);
1109 runtime_throw("debug_scanblock");
1112 // Align b to a word boundary.
1113 off
= (uintptr
)b
& (PtrSize
-1);
1121 for(i
=0; i
<(uintptr
)n
; i
++) {
1124 // Words outside the arena cannot be pointers.
1125 if((byte
*)obj
< runtime_mheap
.arena_start
|| (byte
*)obj
>= runtime_mheap
.arena_used
)
1128 // Round down to word boundary.
1129 obj
= (void*)((uintptr
)obj
& ~((uintptr
)PtrSize
-1));
1131 // Consult span table to find beginning.
1132 s
= runtime_MHeap_LookupMaybe(&runtime_mheap
, obj
);
1136 p
= (byte
*)((uintptr
)s
->start
<<PageShift
);
1138 if(s
->sizeclass
== 0) {
1141 int32 i
= ((byte
*)obj
- p
)/size
;
1145 // Now that we know the object header, reload bits.
1146 off
= (uintptr
*)obj
- (uintptr
*)runtime_mheap
.arena_start
;
1147 bitp
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
1148 shift
= off
% wordsPerBitmapWord
;
1150 bits
= xbits
>> shift
;
1152 // Now we have bits, bitp, and shift correct for
1153 // obj pointing at the base of the object.
1154 // If not allocated or already marked, done.
1155 if((bits
& bitAllocated
) == 0 || (bits
& bitSpecial
) != 0) // NOTE: bitSpecial not bitMarked
1157 *bitp
|= bitSpecial
<<shift
;
1158 if(!(bits
& bitMarked
))
1159 runtime_printf("found unmarked block %p in %p\n", obj
, vp
+i
);
1161 // If object has no pointers, don't need to scan further.
1162 if((bits
& bitNoScan
) != 0)
1165 debug_scanblock(obj
, size
);
1169 // Append obj to the work buffer.
1170 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
1172 enqueue(Obj obj
, Workbuf
**_wbuf
, Obj
**_wp
, uintptr
*_nobj
)
1179 runtime_printf("append obj(%p %D %p)\n", obj
.p
, (int64
)obj
.n
, obj
.ti
);
1181 // Align obj.b to a word boundary.
1182 off
= (uintptr
)obj
.p
& (PtrSize
-1);
1184 obj
.p
+= PtrSize
- off
;
1185 obj
.n
-= PtrSize
- off
;
1189 if(obj
.p
== nil
|| obj
.n
== 0)
1192 // Load work buffer state
1197 // If another proc wants a pointer, give it some.
1198 if(work
.nwait
> 0 && nobj
> handoffThreshold
&& work
.full
== 0) {
1200 wbuf
= handoff(wbuf
);
1202 wp
= wbuf
->obj
+ nobj
;
1205 // If buffer is full, get a new one.
1206 if(wbuf
== nil
|| nobj
>= nelem(wbuf
->obj
)) {
1209 wbuf
= getempty(wbuf
);
1218 // Save work buffer state
1225 markroot(ParFor
*desc
, uint32 i
)
1235 enqueue(work
.roots
[i
], &wbuf
, &wp
, &nobj
);
1236 scanblock(wbuf
, wp
, nobj
, false);
1239 // Get an empty work buffer off the work.empty list,
1240 // allocating new buffers as needed.
1242 getempty(Workbuf
*b
)
1245 runtime_lfstackpush(&work
.full
, &b
->node
);
1246 b
= (Workbuf
*)runtime_lfstackpop(&work
.empty
);
1248 // Need to allocate.
1249 runtime_lock(&work
);
1250 if(work
.nchunk
< sizeof *b
) {
1251 work
.nchunk
= 1<<20;
1252 work
.chunk
= runtime_SysAlloc(work
.nchunk
, &mstats
.gc_sys
);
1253 if(work
.chunk
== nil
)
1254 runtime_throw("runtime: cannot allocate memory");
1256 b
= (Workbuf
*)work
.chunk
;
1257 work
.chunk
+= sizeof *b
;
1258 work
.nchunk
-= sizeof *b
;
1259 runtime_unlock(&work
);
1266 putempty(Workbuf
*b
)
1269 runtime_xadd64(&gcstats
.putempty
, 1);
1271 runtime_lfstackpush(&work
.empty
, &b
->node
);
1274 // Get a full work buffer off the work.full list, or return nil.
1282 runtime_xadd64(&gcstats
.getfull
, 1);
1285 runtime_lfstackpush(&work
.empty
, &b
->node
);
1286 b
= (Workbuf
*)runtime_lfstackpop(&work
.full
);
1287 if(b
!= nil
|| work
.nproc
== 1)
1291 runtime_xadd(&work
.nwait
, +1);
1293 if(work
.full
!= 0) {
1294 runtime_xadd(&work
.nwait
, -1);
1295 b
= (Workbuf
*)runtime_lfstackpop(&work
.full
);
1298 runtime_xadd(&work
.nwait
, +1);
1300 if(work
.nwait
== work
.nproc
)
1303 m
->gcstats
.nprocyield
++;
1304 runtime_procyield(20);
1306 m
->gcstats
.nosyield
++;
1309 m
->gcstats
.nsleep
++;
1310 runtime_usleep(100);
1324 // Make new buffer with half of b's pointers.
1329 runtime_memmove(b1
->obj
, b
->obj
+b
->nobj
, n
*sizeof b1
->obj
[0]);
1330 m
->gcstats
.nhandoff
++;
1331 m
->gcstats
.nhandoffcnt
+= n
;
1333 // Put b on full list - let first half of b get stolen.
1334 runtime_lfstackpush(&work
.full
, &b
->node
);
1344 if(work
.nroot
>= work
.rootcap
) {
1345 cap
= PageSize
/sizeof(Obj
);
1346 if(cap
< 2*work
.rootcap
)
1347 cap
= 2*work
.rootcap
;
1348 new = (Obj
*)runtime_SysAlloc(cap
*sizeof(Obj
), &mstats
.gc_sys
);
1350 runtime_throw("runtime: cannot allocate memory");
1351 if(work
.roots
!= nil
) {
1352 runtime_memmove(new, work
.roots
, work
.rootcap
*sizeof(Obj
));
1353 runtime_SysFree(work
.roots
, work
.rootcap
*sizeof(Obj
), &mstats
.gc_sys
);
1358 work
.roots
[work
.nroot
] = obj
;
1363 addstackroots(G
*gp
)
1365 #ifdef USING_SPLIT_STACK
1373 if(gp
== runtime_g()) {
1374 // Scanning our own stack.
1375 sp
= __splitstack_find(nil
, nil
, &spsize
, &next_segment
,
1376 &next_sp
, &initial_sp
);
1377 } else if((mp
= gp
->m
) != nil
&& mp
->helpgc
) {
1378 // gchelper's stack is in active use and has no interesting pointers.
1381 // Scanning another goroutine's stack.
1382 // The goroutine is usually asleep (the world is stopped).
1384 // The exception is that if the goroutine is about to enter or might
1385 // have just exited a system call, it may be executing code such
1386 // as schedlock and may have needed to start a new stack segment.
1387 // Use the stack segment and stack pointer at the time of
1388 // the system call instead, since that won't change underfoot.
1389 if(gp
->gcstack
!= nil
) {
1391 spsize
= gp
->gcstack_size
;
1392 next_segment
= gp
->gcnext_segment
;
1393 next_sp
= gp
->gcnext_sp
;
1394 initial_sp
= gp
->gcinitial_sp
;
1396 sp
= __splitstack_find_context(&gp
->stack_context
[0],
1397 &spsize
, &next_segment
,
1398 &next_sp
, &initial_sp
);
1402 addroot((Obj
){sp
, spsize
, 0});
1403 while((sp
= __splitstack_find(next_segment
, next_sp
,
1404 &spsize
, &next_segment
,
1405 &next_sp
, &initial_sp
)) != nil
)
1406 addroot((Obj
){sp
, spsize
, 0});
1413 if(gp
== runtime_g()) {
1414 // Scanning our own stack.
1415 bottom
= (byte
*)&gp
;
1416 } else if((mp
= gp
->m
) != nil
&& mp
->helpgc
) {
1417 // gchelper's stack is in active use and has no interesting pointers.
1420 // Scanning another goroutine's stack.
1421 // The goroutine is usually asleep (the world is stopped).
1422 bottom
= (byte
*)gp
->gcnext_sp
;
1426 top
= (byte
*)gp
->gcinitial_sp
+ gp
->gcstack_size
;
1428 addroot((Obj
){bottom
, top
- bottom
, 0});
1430 addroot((Obj
){top
, bottom
- top
, 0});
1435 addfinroots(void *v
)
1441 if(!runtime_mlookup(v
, (byte
**)&base
, &size
, nil
) || !runtime_blockspecial(base
))
1442 runtime_throw("mark - finalizer inconsistency");
1444 // do not mark the finalizer block itself. just mark the things it points at.
1445 addroot((Obj
){base
, size
, 0});
1448 static struct root_list
* roots
;
1451 __go_register_gc_roots (struct root_list
* r
)
1453 // FIXME: This needs locking if multiple goroutines can call
1454 // dlopen simultaneously.
1462 struct root_list
*pl
;
1465 MSpan
*s
, **allspans
;
1471 for(pl
= roots
; pl
!= nil
; pl
= pl
->next
) {
1472 struct root
* pr
= &pl
->roots
[0];
1474 void *decl
= pr
->decl
;
1477 addroot((Obj
){decl
, pr
->size
, 0});
1482 addroot((Obj
){(byte
*)&runtime_m0
, sizeof runtime_m0
, 0});
1483 addroot((Obj
){(byte
*)&runtime_g0
, sizeof runtime_g0
, 0});
1484 addroot((Obj
){(byte
*)&runtime_allg
, sizeof runtime_allg
, 0});
1485 addroot((Obj
){(byte
*)&runtime_allm
, sizeof runtime_allm
, 0});
1486 addroot((Obj
){(byte
*)&runtime_allp
, sizeof runtime_allp
, 0});
1487 runtime_proc_scan(addroot
);
1488 runtime_MProf_Mark(addroot
);
1489 runtime_time_scan(addroot
);
1490 runtime_netpoll_scan(addroot
);
1493 allspans
= runtime_mheap
.allspans
;
1494 for(spanidx
=0; spanidx
<runtime_mheap
.nspan
; spanidx
++) {
1495 s
= allspans
[spanidx
];
1496 if(s
->state
== MSpanInUse
) {
1497 // The garbage collector ignores type pointers stored in MSpan.types:
1498 // - Compiler-generated types are stored outside of heap.
1499 // - The reflect package has runtime-generated types cached in its data structures.
1500 // The garbage collector relies on finding the references via that cache.
1501 switch(s
->types
.compression
) {
1507 markonly((byte
*)s
->types
.data
);
1514 for(gp
=runtime_allg
; gp
!=nil
; gp
=gp
->alllink
) {
1517 runtime_printf("unexpected G.status %d\n", gp
->status
);
1518 runtime_throw("mark - bad status");
1522 runtime_throw("mark - world not stopped");
1531 runtime_walkfintab(addfinroots
, addroot
);
1533 for(fb
=allfin
; fb
; fb
=fb
->alllink
)
1534 addroot((Obj
){(byte
*)fb
->fin
, fb
->cnt
*sizeof(fb
->fin
[0]), 0});
1536 addroot((Obj
){(byte
*)&work
, sizeof work
, 0});
1540 handlespecial(byte
*p
, uintptr size
)
1543 const struct __go_func_type
*ft
;
1544 const struct __go_ptr_type
*ot
;
1548 if(!runtime_getfinalizer(p
, true, &fn
, &ft
, &ot
)) {
1549 runtime_setblockspecial(p
, false);
1550 runtime_MProf_Free(p
, size
);
1554 runtime_lock(&finlock
);
1555 if(finq
== nil
|| finq
->cnt
== finq
->cap
) {
1557 finc
= runtime_persistentalloc(PageSize
, 0, &mstats
.gc_sys
);
1558 finc
->cap
= (PageSize
- sizeof(FinBlock
)) / sizeof(Finalizer
) + 1;
1559 finc
->alllink
= allfin
;
1567 f
= &finq
->fin
[finq
->cnt
];
1573 runtime_unlock(&finlock
);
1577 // Sweep frees or collects finalizers for blocks not marked in the mark phase.
1578 // It clears the mark bits in preparation for the next GC round.
1580 sweepspan(ParFor
*desc
, uint32 idx
)
1583 int32 cl
, n
, npages
;
1592 uintptr type_data_inc
;
1598 s
= runtime_mheap
.allspans
[idx
];
1599 if(s
->state
!= MSpanInUse
)
1601 arena_start
= runtime_mheap
.arena_start
;
1602 p
= (byte
*)(s
->start
<< PageShift
);
1608 // Chunk full of small blocks.
1609 npages
= runtime_class_to_allocnpages
[cl
];
1610 n
= (npages
<< PageShift
) / size
;
1616 type_data
= (byte
*)s
->types
.data
;
1617 type_data_inc
= sizeof(uintptr
);
1618 compression
= s
->types
.compression
;
1619 switch(compression
) {
1621 type_data
+= 8*sizeof(uintptr
);
1626 // Sweep through n objects of given size starting at p.
1627 // This thread owns the span now, so it can manipulate
1628 // the block bitmap without atomic operations.
1629 for(; n
> 0; n
--, p
+= size
, type_data
+=type_data_inc
) {
1630 uintptr off
, *bitp
, shift
, bits
;
1632 off
= (uintptr
*)p
- (uintptr
*)arena_start
;
1633 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
1634 shift
= off
% wordsPerBitmapWord
;
1635 bits
= *bitp
>>shift
;
1637 if((bits
& bitAllocated
) == 0)
1640 if((bits
& bitMarked
) != 0) {
1642 if(!(bits
& bitSpecial
))
1643 runtime_printf("found spurious mark on %p\n", p
);
1644 *bitp
&= ~(bitSpecial
<<shift
);
1646 *bitp
&= ~(bitMarked
<<shift
);
1650 // Special means it has a finalizer or is being profiled.
1651 // In DebugMark mode, the bit has been coopted so
1652 // we have to assume all blocks are special.
1653 if(DebugMark
|| (bits
& bitSpecial
) != 0) {
1654 if(handlespecial(p
, size
))
1658 // Mark freed; restore block boundary bit.
1659 *bitp
= (*bitp
& ~(bitMask
<<shift
)) | (bitBlockBoundary
<<shift
);
1663 runtime_unmarkspan(p
, 1<<PageShift
);
1664 *(uintptr
*)p
= (uintptr
)0xdeaddeaddeaddeadll
; // needs zeroing
1665 runtime_MHeap_Free(&runtime_mheap
, s
, 1);
1666 c
->local_nlargefree
++;
1667 c
->local_largefree
+= size
;
1669 // Free small object.
1670 switch(compression
) {
1672 *(uintptr
*)type_data
= 0;
1675 *(byte
*)type_data
= 0;
1678 if(size
> sizeof(uintptr
))
1679 ((uintptr
*)p
)[1] = (uintptr
)0xdeaddeaddeaddeadll
; // mark as "needs to be zeroed"
1681 end
->next
= (MLink
*)p
;
1688 c
->local_nsmallfree
[cl
] += nfree
;
1689 c
->local_cachealloc
-= nfree
* size
;
1690 runtime_MCentral_FreeSpan(&runtime_mheap
.central
[cl
], s
, nfree
, head
.next
, end
);
1695 dumpspan(uint32 idx
)
1697 int32 sizeclass
, n
, npages
, i
, column
;
1702 bool allocated
, special
;
1704 s
= runtime_mheap
.allspans
[idx
];
1705 if(s
->state
!= MSpanInUse
)
1707 arena_start
= runtime_mheap
.arena_start
;
1708 p
= (byte
*)(s
->start
<< PageShift
);
1709 sizeclass
= s
->sizeclass
;
1711 if(sizeclass
== 0) {
1714 npages
= runtime_class_to_allocnpages
[sizeclass
];
1715 n
= (npages
<< PageShift
) / size
;
1718 runtime_printf("%p .. %p:\n", p
, p
+n
*size
);
1720 for(; n
>0; n
--, p
+=size
) {
1721 uintptr off
, *bitp
, shift
, bits
;
1723 off
= (uintptr
*)p
- (uintptr
*)arena_start
;
1724 bitp
= (uintptr
*)arena_start
- off
/wordsPerBitmapWord
- 1;
1725 shift
= off
% wordsPerBitmapWord
;
1726 bits
= *bitp
>>shift
;
1728 allocated
= ((bits
& bitAllocated
) != 0);
1729 special
= ((bits
& bitSpecial
) != 0);
1731 for(i
=0; (uint32
)i
<size
; i
+=sizeof(void*)) {
1733 runtime_printf("\t");
1736 runtime_printf(allocated
? "(" : "[");
1737 runtime_printf(special
? "@" : "");
1738 runtime_printf("%p: ", p
+i
);
1740 runtime_printf(" ");
1743 runtime_printf("%p", *(void**)(p
+i
));
1745 if(i
+sizeof(void*) >= size
) {
1746 runtime_printf(allocated
? ") " : "] ");
1751 runtime_printf("\n");
1756 runtime_printf("\n");
1759 // A debugging function to dump the contents of memory
1761 runtime_memorydump(void)
1765 for(spanidx
=0; spanidx
<runtime_mheap
.nspan
; spanidx
++) {
1771 runtime_gchelper(void)
1775 // parallel mark for over gc roots
1776 runtime_parfordo(work
.markfor
);
1778 // help other threads scan secondary blocks
1779 scanblock(nil
, nil
, 0, true);
1782 // wait while the main thread executes mark(debug_scanblock)
1783 while(runtime_atomicload(&work
.debugmarkdone
) == 0)
1787 runtime_parfordo(work
.sweepfor
);
1788 bufferList
[runtime_m()->helpgc
].busy
= 0;
1789 if(runtime_xadd(&work
.ndone
, +1) == work
.nproc
-1)
1790 runtime_notewakeup(&work
.alldone
);
1793 #define GcpercentUnknown (-2)
1795 // Initialized from $GOGC. GOGC=off means no gc.
1797 // Next gc is after we've allocated an extra amount of
1798 // memory proportional to the amount already in use.
1799 // If gcpercent=100 and we're using 4M, we'll gc again
1800 // when we get to 8M. This keeps the gc cost in linear
1801 // proportion to the allocation cost. Adjusting gcpercent
1802 // just changes the linear constant (and also the amount of
1803 // extra memory used).
1804 static int32 gcpercent
= GcpercentUnknown
;
1812 for(pp
=runtime_allp
; (p
=*pp
) != nil
; pp
++) {
1816 runtime_purgecachedstats(c
);
1821 updatememstats(GCStats
*stats
)
1828 uint64 stacks_inuse
, smallfree
;
1832 runtime_memclr((byte
*)stats
, sizeof(*stats
));
1834 for(mp
=runtime_allm
; mp
; mp
=mp
->alllink
) {
1835 //stacks_inuse += mp->stackinuse*FixedStack;
1837 src
= (uint64
*)&mp
->gcstats
;
1838 dst
= (uint64
*)stats
;
1839 for(i
=0; i
<sizeof(*stats
)/sizeof(uint64
); i
++)
1841 runtime_memclr((byte
*)&mp
->gcstats
, sizeof(mp
->gcstats
));
1844 mstats
.stacks_inuse
= stacks_inuse
;
1845 mstats
.mcache_inuse
= runtime_mheap
.cachealloc
.inuse
;
1846 mstats
.mspan_inuse
= runtime_mheap
.spanalloc
.inuse
;
1847 mstats
.sys
= mstats
.heap_sys
+ mstats
.stacks_sys
+ mstats
.mspan_sys
+
1848 mstats
.mcache_sys
+ mstats
.buckhash_sys
+ mstats
.gc_sys
+ mstats
.other_sys
;
1850 // Calculate memory allocator stats.
1851 // During program execution we only count number of frees and amount of freed memory.
1852 // Current number of alive object in the heap and amount of alive heap memory
1853 // are calculated by scanning all spans.
1854 // Total number of mallocs is calculated as number of frees plus number of alive objects.
1855 // Similarly, total amount of allocated memory is calculated as amount of freed memory
1856 // plus amount of alive heap memory.
1858 mstats
.total_alloc
= 0;
1861 for(i
= 0; i
< nelem(mstats
.by_size
); i
++) {
1862 mstats
.by_size
[i
].nmalloc
= 0;
1863 mstats
.by_size
[i
].nfree
= 0;
1866 // Flush MCache's to MCentral.
1867 for(pp
=runtime_allp
; (p
=*pp
) != nil
; pp
++) {
1871 runtime_MCache_ReleaseAll(c
);
1874 // Aggregate local stats.
1877 // Scan all spans and count number of alive objects.
1878 for(i
= 0; i
< runtime_mheap
.nspan
; i
++) {
1879 s
= runtime_mheap
.allspans
[i
];
1880 if(s
->state
!= MSpanInUse
)
1882 if(s
->sizeclass
== 0) {
1884 mstats
.alloc
+= s
->elemsize
;
1886 mstats
.nmalloc
+= s
->ref
;
1887 mstats
.by_size
[s
->sizeclass
].nmalloc
+= s
->ref
;
1888 mstats
.alloc
+= s
->ref
*s
->elemsize
;
1892 // Aggregate by size class.
1894 mstats
.nfree
= runtime_mheap
.nlargefree
;
1895 for(i
= 0; i
< nelem(mstats
.by_size
); i
++) {
1896 mstats
.nfree
+= runtime_mheap
.nsmallfree
[i
];
1897 mstats
.by_size
[i
].nfree
= runtime_mheap
.nsmallfree
[i
];
1898 mstats
.by_size
[i
].nmalloc
+= runtime_mheap
.nsmallfree
[i
];
1899 smallfree
+= runtime_mheap
.nsmallfree
[i
] * runtime_class_to_size
[i
];
1901 mstats
.nmalloc
+= mstats
.nfree
;
1903 // Calculate derived stats.
1904 mstats
.total_alloc
= mstats
.alloc
+ runtime_mheap
.largefree
+ smallfree
;
1905 mstats
.heap_alloc
= mstats
.alloc
;
1906 mstats
.heap_objects
= mstats
.nmalloc
- mstats
.nfree
;
1909 // Structure of arguments passed to function gc().
1910 // This allows the arguments to be passed via runtime_mcall.
1913 int64 start_time
; // start time of GC in ns (just before stoptheworld)
1916 static void gc(struct gc_args
*args
);
1917 static void mgc(G
*gp
);
1924 p
= runtime_getenv("GOGC");
1925 if(p
== nil
|| p
[0] == '\0')
1927 if(runtime_strcmp((const char *)p
, "off") == 0)
1929 return runtime_atoi(p
);
1933 runtime_gc(int32 force
)
1940 // The atomic operations are not atomic if the uint64s
1941 // are not aligned on uint64 boundaries. This has been
1942 // a problem in the past.
1943 if((((uintptr
)&work
.empty
) & 7) != 0)
1944 runtime_throw("runtime: gc work buffer is misaligned");
1945 if((((uintptr
)&work
.full
) & 7) != 0)
1946 runtime_throw("runtime: gc work buffer is misaligned");
1948 // Make sure all registers are saved on stack so that
1949 // scanstack sees them.
1950 __builtin_unwind_init();
1952 // The gc is turned off (via enablegc) until
1953 // the bootstrap has completed.
1954 // Also, malloc gets called in the guts
1955 // of a number of libraries that might be
1956 // holding locks. To avoid priority inversion
1957 // problems, don't bother trying to run gc
1958 // while holding a lock. The next mallocgc
1959 // without a lock will do the gc instead.
1961 if(!mstats
.enablegc
|| runtime_g() == m
->g0
|| m
->locks
> 0 || runtime_panicking
)
1964 if(gcpercent
== GcpercentUnknown
) { // first time through
1965 runtime_lock(&runtime_mheap
);
1966 if(gcpercent
== GcpercentUnknown
)
1967 gcpercent
= readgogc();
1968 runtime_unlock(&runtime_mheap
);
1973 runtime_semacquire(&runtime_worldsema
, false);
1974 if(!force
&& mstats
.heap_alloc
< mstats
.next_gc
) {
1975 // typically threads which lost the race to grab
1976 // worldsema exit here when gc is done.
1977 runtime_semrelease(&runtime_worldsema
);
1981 // Ok, we're doing it! Stop everybody else
1982 a
.start_time
= runtime_nanotime();
1984 runtime_stoptheworld();
1986 // Run gc on the g0 stack. We do this so that the g stack
1987 // we're currently running on will no longer change. Cuts
1988 // the root set down a bit (g0 stacks are not scanned, and
1989 // we don't need to scan gc's internal state). Also an
1990 // enabler for copyable stacks.
1991 for(i
= 0; i
< (runtime_debug
.gctrace
> 1 ? 2 : 1); i
++) {
1992 // switch to g0, call gc(&a), then switch back
1995 g
->status
= Gwaiting
;
1996 g
->waitreason
= "garbage collection";
1998 // record a new start time in case we're going around again
1999 a
.start_time
= runtime_nanotime();
2005 runtime_semrelease(&runtime_worldsema
);
2006 runtime_starttheworld();
2009 // now that gc is done, kick off finalizer thread if needed
2011 runtime_lock(&finlock
);
2012 // kick off or wake up goroutine to run queued finalizers
2014 fing
= __go_go(runfinq
, nil
);
2017 runtime_ready(fing
);
2019 runtime_unlock(&finlock
);
2021 // give the queued finalizers, if any, a chance to run
2030 gp
->status
= Grunning
;
2035 gc(struct gc_args
*args
)
2038 int64 t0
, t1
, t2
, t3
, t4
;
2039 uint64 heap0
, heap1
, obj0
, obj1
, ninstr
;
2047 t0
= args
->start_time
;
2050 runtime_memclr((byte
*)&gcstats
, sizeof(gcstats
));
2052 for(mp
=runtime_allm
; mp
; mp
=mp
->alllink
)
2053 runtime_settype_flush(mp
);
2057 if(runtime_debug
.gctrace
) {
2058 updatememstats(nil
);
2059 heap0
= mstats
.heap_alloc
;
2060 obj0
= mstats
.nmalloc
- mstats
.nfree
;
2063 m
->locks
++; // disable gc during mallocs in parforalloc
2064 if(work
.markfor
== nil
)
2065 work
.markfor
= runtime_parforalloc(MaxGcproc
);
2066 if(work
.sweepfor
== nil
)
2067 work
.sweepfor
= runtime_parforalloc(MaxGcproc
);
2070 if(itabtype
== nil
) {
2071 // get C pointer to the Go type "itab"
2072 // runtime_gc_itab_ptr(&eface);
2073 // itabtype = ((PtrType*)eface.type)->elem;
2078 work
.debugmarkdone
= 0;
2079 work
.nproc
= runtime_gcprocs();
2081 runtime_parforsetup(work
.markfor
, work
.nproc
, work
.nroot
, nil
, false, markroot
);
2082 runtime_parforsetup(work
.sweepfor
, work
.nproc
, runtime_mheap
.nspan
, nil
, true, sweepspan
);
2083 if(work
.nproc
> 1) {
2084 runtime_noteclear(&work
.alldone
);
2085 runtime_helpgc(work
.nproc
);
2088 t1
= runtime_nanotime();
2091 runtime_parfordo(work
.markfor
);
2092 scanblock(nil
, nil
, 0, true);
2095 for(i
=0; i
<work
.nroot
; i
++)
2096 debug_scanblock(work
.roots
[i
].p
, work
.roots
[i
].n
);
2097 runtime_atomicstore(&work
.debugmarkdone
, 1);
2099 t2
= runtime_nanotime();
2101 runtime_parfordo(work
.sweepfor
);
2102 bufferList
[m
->helpgc
].busy
= 0;
2103 t3
= runtime_nanotime();
2106 runtime_notesleep(&work
.alldone
);
2109 mstats
.next_gc
= mstats
.heap_alloc
+mstats
.heap_alloc
*gcpercent
/100;
2111 t4
= runtime_nanotime();
2112 mstats
.last_gc
= t4
;
2113 mstats
.pause_ns
[mstats
.numgc
%nelem(mstats
.pause_ns
)] = t4
- t0
;
2114 mstats
.pause_total_ns
+= t4
- t0
;
2117 runtime_printf("pause %D\n", t4
-t0
);
2119 if(runtime_debug
.gctrace
) {
2120 updatememstats(&stats
);
2121 heap1
= mstats
.heap_alloc
;
2122 obj1
= mstats
.nmalloc
- mstats
.nfree
;
2124 stats
.nprocyield
+= work
.sweepfor
->nprocyield
;
2125 stats
.nosyield
+= work
.sweepfor
->nosyield
;
2126 stats
.nsleep
+= work
.sweepfor
->nsleep
;
2128 runtime_printf("gc%d(%d): %D+%D+%D ms, %D -> %D MB %D -> %D (%D-%D) objects,"
2129 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
2130 mstats
.numgc
, work
.nproc
, (t2
-t1
)/1000000, (t3
-t2
)/1000000, (t1
-t0
+t4
-t3
)/1000000,
2131 heap0
>>20, heap1
>>20, obj0
, obj1
,
2132 mstats
.nmalloc
, mstats
.nfree
,
2133 stats
.nhandoff
, stats
.nhandoffcnt
,
2134 work
.sweepfor
->nsteal
, work
.sweepfor
->nstealcnt
,
2135 stats
.nprocyield
, stats
.nosyield
, stats
.nsleep
);
2137 runtime_printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n",
2138 gcstats
.nbytes
, gcstats
.obj
.cnt
, gcstats
.obj
.notype
, gcstats
.obj
.typelookup
);
2139 if(gcstats
.ptr
.cnt
!= 0)
2140 runtime_printf("avg ptrbufsize: %D (%D/%D)\n",
2141 gcstats
.ptr
.sum
/gcstats
.ptr
.cnt
, gcstats
.ptr
.sum
, gcstats
.ptr
.cnt
);
2142 if(gcstats
.obj
.cnt
!= 0)
2143 runtime_printf("avg nobj: %D (%D/%D)\n",
2144 gcstats
.obj
.sum
/gcstats
.obj
.cnt
, gcstats
.obj
.sum
, gcstats
.obj
.cnt
);
2145 runtime_printf("rescans: %D, %D bytes\n", gcstats
.rescan
, gcstats
.rescanbytes
);
2147 runtime_printf("instruction counts:\n");
2149 for(i
=0; i
<nelem(gcstats
.instr
); i
++) {
2150 runtime_printf("\t%d:\t%D\n", i
, gcstats
.instr
[i
]);
2151 ninstr
+= gcstats
.instr
[i
];
2153 runtime_printf("\ttotal:\t%D\n", ninstr
);
2155 runtime_printf("putempty: %D, getfull: %D\n", gcstats
.putempty
, gcstats
.getfull
);
2157 runtime_printf("markonly base lookup: bit %D word %D span %D\n", gcstats
.markonly
.foundbit
, gcstats
.markonly
.foundword
, gcstats
.markonly
.foundspan
);
2158 runtime_printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats
.flushptrbuf
.foundbit
, gcstats
.flushptrbuf
.foundword
, gcstats
.flushptrbuf
.foundspan
);
2165 void runtime_ReadMemStats(MStats
*)
2166 __asm__ (GOSYM_PREFIX
"runtime.ReadMemStats");
2169 runtime_ReadMemStats(MStats
*stats
)
2173 // Have to acquire worldsema to stop the world,
2174 // because stoptheworld can only be used by
2175 // one goroutine at a time, and there might be
2176 // a pending garbage collection already calling it.
2177 runtime_semacquire(&runtime_worldsema
, false);
2180 runtime_stoptheworld();
2181 updatememstats(nil
);
2185 runtime_semrelease(&runtime_worldsema
);
2186 runtime_starttheworld();
2190 void runtime_debug_readGCStats(Slice
*)
2191 __asm__("runtime_debug.readGCStats");
2194 runtime_debug_readGCStats(Slice
*pauses
)
2199 // Calling code in runtime/debug should make the slice large enough.
2200 if((size_t)pauses
->cap
< nelem(mstats
.pause_ns
)+3)
2201 runtime_throw("runtime: short slice passed to readGCStats");
2203 // Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
2204 p
= (uint64
*)pauses
->array
;
2205 runtime_lock(&runtime_mheap
);
2207 if(n
> nelem(mstats
.pause_ns
))
2208 n
= nelem(mstats
.pause_ns
);
2210 // The pause buffer is circular. The most recent pause is at
2211 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
2212 // from there to go back farther in time. We deliver the times
2213 // most recent first (in p[0]).
2215 p
[i
] = mstats
.pause_ns
[(mstats
.numgc
-1-i
)%nelem(mstats
.pause_ns
)];
2217 p
[n
] = mstats
.last_gc
;
2218 p
[n
+1] = mstats
.numgc
;
2219 p
[n
+2] = mstats
.pause_total_ns
;
2220 runtime_unlock(&runtime_mheap
);
2221 pauses
->__count
= n
+3;
2224 intgo
runtime_debug_setGCPercent(intgo
)
2225 __asm__("runtime_debug.setGCPercent");
2228 runtime_debug_setGCPercent(intgo in
)
2232 runtime_lock(&runtime_mheap
);
2233 if(gcpercent
== GcpercentUnknown
)
2234 gcpercent
= readgogc();
2239 runtime_unlock(&runtime_mheap
);
2249 if(m
->helpgc
< 0 || m
->helpgc
>= MaxGcproc
)
2250 runtime_throw("gchelperstart: bad m->helpgc");
2251 if(runtime_xchg(&bufferList
[m
->helpgc
].busy
, 1))
2252 runtime_throw("gchelperstart: already busy");
2253 if(runtime_g() != m
->g0
)
2254 runtime_throw("gchelper not running on g0 stack");
2258 runfinq(void* dummy
__attribute__ ((unused
)))
2261 FinBlock
*fb
, *next
;
2267 runtime_lock(&finlock
);
2272 runtime_park(runtime_unlock
, &finlock
, "finalizer wait");
2275 runtime_unlock(&finlock
);
2277 runtime_racefingo();
2278 for(; fb
; fb
=next
) {
2280 for(i
=0; i
<(uint32
)fb
->cnt
; i
++) {
2285 fint
= ((const Type
**)f
->ft
->__in
.array
)[0];
2286 if(fint
->kind
== KindPtr
) {
2287 // direct use of pointer
2289 } else if(((const InterfaceType
*)fint
)->__methods
.__count
== 0) {
2290 // convert to empty interface
2291 ef
.type
= (const Type
*)f
->ot
;
2292 ef
.__object
= f
->arg
;
2295 // convert to interface with methods
2296 iface
.__methods
= __go_convert_interface_2((const Type
*)fint
,
2299 iface
.__object
= f
->arg
;
2300 if(iface
.__methods
== nil
)
2301 runtime_throw("invalid type conversion in runfinq");
2304 reflect_call(f
->ft
, f
->fn
, 0, 0, ¶m
, nil
);
2313 runtime_gc(1); // trigger another gc to clean up the finalized objects, if possible
2317 // mark the block at v of size n as allocated.
2318 // If noscan is true, mark it as not needing scanning.
2320 runtime_markallocated(void *v
, uintptr n
, bool noscan
)
2322 uintptr
*b
, obits
, bits
, off
, shift
;
2325 runtime_printf("markallocated %p+%p\n", v
, n
);
2327 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2328 runtime_throw("markallocated: bad pointer");
2330 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2331 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2332 shift
= off
% wordsPerBitmapWord
;
2336 bits
= (obits
& ~(bitMask
<<shift
)) | (bitAllocated
<<shift
);
2338 bits
|= bitNoScan
<<shift
;
2339 if(runtime_gomaxprocs
== 1) {
2343 // more than one goroutine is potentially running: use atomic op
2344 if(runtime_casp((void**)b
, (void*)obits
, (void*)bits
))
2350 // mark the block at v of size n as freed.
2352 runtime_markfreed(void *v
, uintptr n
)
2354 uintptr
*b
, obits
, bits
, off
, shift
;
2357 runtime_printf("markfreed %p+%p\n", v
, n
);
2359 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2360 runtime_throw("markfreed: bad pointer");
2362 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2363 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2364 shift
= off
% wordsPerBitmapWord
;
2368 bits
= (obits
& ~(bitMask
<<shift
)) | (bitBlockBoundary
<<shift
);
2369 if(runtime_gomaxprocs
== 1) {
2373 // more than one goroutine is potentially running: use atomic op
2374 if(runtime_casp((void**)b
, (void*)obits
, (void*)bits
))
2380 // check that the block at v of size n is marked freed.
2382 runtime_checkfreed(void *v
, uintptr n
)
2384 uintptr
*b
, bits
, off
, shift
;
2386 if(!runtime_checking
)
2389 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2390 return; // not allocated, so okay
2392 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2393 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2394 shift
= off
% wordsPerBitmapWord
;
2397 if((bits
& bitAllocated
) != 0) {
2398 runtime_printf("checkfreed %p+%p: off=%p have=%p\n",
2399 v
, n
, off
, bits
& bitMask
);
2400 runtime_throw("checkfreed: not freed");
2404 // mark the span of memory at v as having n blocks of the given size.
2405 // if leftover is true, there is left over space at the end of the span.
2407 runtime_markspan(void *v
, uintptr size
, uintptr n
, bool leftover
)
2409 uintptr
*b
, off
, shift
;
2412 if((byte
*)v
+size
*n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2413 runtime_throw("markspan: bad pointer");
2416 if(leftover
) // mark a boundary just past end of last block too
2418 for(; n
-- > 0; p
+= size
) {
2419 // Okay to use non-atomic ops here, because we control
2420 // the entire span, and each bitmap word has bits for only
2421 // one span, so no other goroutines are changing these
2423 off
= (uintptr
*)p
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2424 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2425 shift
= off
% wordsPerBitmapWord
;
2426 *b
= (*b
& ~(bitMask
<<shift
)) | (bitBlockBoundary
<<shift
);
2430 // unmark the span of memory at v of length n bytes.
2432 runtime_unmarkspan(void *v
, uintptr n
)
2434 uintptr
*p
, *b
, off
;
2436 if((byte
*)v
+n
> (byte
*)runtime_mheap
.arena_used
|| (byte
*)v
< runtime_mheap
.arena_start
)
2437 runtime_throw("markspan: bad pointer");
2440 off
= p
- (uintptr
*)runtime_mheap
.arena_start
; // word offset
2441 if(off
% wordsPerBitmapWord
!= 0)
2442 runtime_throw("markspan: unaligned pointer");
2443 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2445 if(n
%wordsPerBitmapWord
!= 0)
2446 runtime_throw("unmarkspan: unaligned length");
2447 // Okay to use non-atomic ops here, because we control
2448 // the entire span, and each bitmap word has bits for only
2449 // one span, so no other goroutines are changing these
2451 n
/= wordsPerBitmapWord
;
2457 runtime_blockspecial(void *v
)
2459 uintptr
*b
, off
, shift
;
2464 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
;
2465 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2466 shift
= off
% wordsPerBitmapWord
;
2468 return (*b
& (bitSpecial
<<shift
)) != 0;
2472 runtime_setblockspecial(void *v
, bool s
)
2474 uintptr
*b
, off
, shift
, bits
, obits
;
2479 off
= (uintptr
*)v
- (uintptr
*)runtime_mheap
.arena_start
;
2480 b
= (uintptr
*)runtime_mheap
.arena_start
- off
/wordsPerBitmapWord
- 1;
2481 shift
= off
% wordsPerBitmapWord
;
2486 bits
= obits
| (bitSpecial
<<shift
);
2488 bits
= obits
& ~(bitSpecial
<<shift
);
2489 if(runtime_gomaxprocs
== 1) {
2493 // more than one goroutine is potentially running: use atomic op
2494 if(runtime_casp((void**)b
, (void*)obits
, (void*)bits
))
2501 runtime_MHeap_MapBits(MHeap
*h
)
2505 // Caller has added extra mappings to the arena.
2506 // Add extra mappings of bitmap words as needed.
2507 // We allocate extra bitmap pieces in chunks of bitmapChunk.
2513 n
= (h
->arena_used
- h
->arena_start
) / wordsPerBitmapWord
;
2514 n
= ROUND(n
, bitmapChunk
);
2515 if(h
->bitmap_mapped
>= n
)
2518 page_size
= getpagesize();
2519 n
= (n
+page_size
-1) & ~(page_size
-1);
2521 runtime_SysMap(h
->arena_start
- n
, n
- h
->bitmap_mapped
, &mstats
.gc_sys
);
2522 h
->bitmap_mapped
= n
;