2 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
3 * Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
4 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
5 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
7 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
8 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 * Permission is hereby granted to use or copy this program
11 * for any purpose, provided the above notices are retained on all copies.
12 * Permission to modify the code and to distribute modified code is granted,
13 * provided the above notices are retained, and a notice that the code was
14 * modified is included with the above copyright notice.
20 signed_word GC_mem_found
= 0;
21 /* Number of words of memory reclaimed */
23 static void report_leak(p
, sz
)
27 if (HDR(p
) -> hb_obj_kind
== PTRFREE
) {
28 GC_err_printf0("Leaked atomic object at ");
30 GC_err_printf0("Leaked composite object at ");
36 # define FOUND_FREE(hblk, word_no) \
38 report_leak((ptr_t)hblk + WORDS_TO_BYTES(word_no), \
39 HDR(hblk) -> hb_sz); \
49 * Test whether a block is completely empty, i.e. contains no marked
50 * objects. This does not require the block to be in physical
54 GC_bool
GC_block_empty(hhdr
)
57 register word
*p
= (word
*)(&(hhdr
-> hb_marks
[0]));
58 register word
* plim
=
59 (word
*)(&(hhdr
-> hb_marks
[MARK_BITS_SZ
]));
61 if (*p
++) return(FALSE
);
66 /* The following functions sometimes return a DONT_KNOW value. */
70 # define GC_block_nearly_full1(hhdr, pat1) DONT_KNOW
71 # define GC_block_nearly_full3(hhdr, pat1, pat2) DONT_KNOW
72 # define GC_block_nearly_full(hhdr) DONT_KNOW
76 * Test whether nearly all of the mark words consist of the same
79 #define FULL_THRESHOLD (MARK_BITS_SZ/16)
81 GC_bool
GC_block_nearly_full1(hhdr
, pat1
)
87 GC_ASSERT((MARK_BITS_SZ
& 1) == 0);
88 for (i
= 0; i
< MARK_BITS_SZ
; ++i
) {
89 if ((hhdr
-> hb_marks
[i
] | ~pat1
) != ONES
) {
90 if (++misses
> FULL_THRESHOLD
) return FALSE
;
97 * Test whether the same repeating 3 word pattern occurs in nearly
98 * all the mark bit slots.
99 * This is used as a heuristic, so we're a bit sloppy and ignore
100 * the last one or two words.
102 GC_bool
GC_block_nearly_full3(hhdr
, pat1
, pat2
, pat3
)
104 word pat1
, pat2
, pat3
;
109 if (MARK_BITS_SZ
< 4) {
112 for (i
= 0; i
< MARK_BITS_SZ
- 2; i
+= 3) {
113 if ((hhdr
-> hb_marks
[i
] | ~pat1
) != ONES
) {
114 if (++misses
> FULL_THRESHOLD
) return FALSE
;
116 if ((hhdr
-> hb_marks
[i
+1] | ~pat2
) != ONES
) {
117 if (++misses
> FULL_THRESHOLD
) return FALSE
;
119 if ((hhdr
-> hb_marks
[i
+2] | ~pat3
) != ONES
) {
120 if (++misses
> FULL_THRESHOLD
) return FALSE
;
126 /* Check whether a small object block is nearly full by looking at only */
128 /* We manually precomputed the mark bit patterns that need to be */
129 /* checked for, and we give up on the ones that are unlikely to occur, */
130 /* or have period > 3. */
131 /* This would be a lot easier with a mark bit per object instead of per */
132 /* word, but that would rewuire computing object numbers in the mark */
133 /* loop, which would require different data structures ... */
134 GC_bool
GC_block_nearly_full(hhdr
)
137 int sz
= hhdr
-> hb_sz
;
139 # if CPP_WORDSZ != 32 && CPP_WORDSZ != 64
140 return DONT_KNOW
; /* Shouldn't be used in any standard config. */
142 if (0 != HDR_WORDS
) return DONT_KNOW
;
143 /* Also shouldn't happen */
144 # if CPP_WORDSZ == 32
147 return GC_block_nearly_full1(hhdr
, 0xffffffffl
);
149 return GC_block_nearly_full1(hhdr
, 0x55555555l
);
151 return GC_block_nearly_full1(hhdr
, 0x11111111l
);
153 return GC_block_nearly_full3(hhdr
, 0x41041041l
,
157 return GC_block_nearly_full1(hhdr
, 0x01010101l
);
159 return GC_block_nearly_full3(hhdr
, 0x01001001l
,
163 return GC_block_nearly_full1(hhdr
, 0x00010001l
);
165 return GC_block_nearly_full1(hhdr
, 0x00000001l
);
170 # if CPP_WORDSZ == 64
173 return GC_block_nearly_full1(hhdr
, 0xffffffffffffffffl
);
175 return GC_block_nearly_full1(hhdr
, 0x5555555555555555l
);
177 return GC_block_nearly_full1(hhdr
, 0x1111111111111111l
);
179 return GC_block_nearly_full3(hhdr
, 0x1041041041041041l
,
181 0x0410410410410410l
);
183 return GC_block_nearly_full1(hhdr
, 0x0101010101010101l
);
185 return GC_block_nearly_full3(hhdr
, 0x1001001001001001l
,
187 0x0010010010010010l
);
189 return GC_block_nearly_full1(hhdr
, 0x0001000100010001l
);
191 return GC_block_nearly_full1(hhdr
, 0x0000000100000001l
);
197 #endif /* !SMALL_CONFIG */
200 # define INCR_WORDS(sz) n_words_found += (sz)
202 # define INCR_WORDS(sz)
205 * Restore unmarked small objects in h of size sz to the object
206 * free list. Returns the new list.
207 * Clears unmarked objects.
210 ptr_t
GC_reclaim_clear(hbp
, hhdr
, sz
, list
)
211 register struct hblk
*hbp
; /* ptr to current heap block */
216 register int word_no
;
217 register word
*p
, *q
, *plim
;
219 register int n_words_found
= 0;
222 p
= (word
*)(hbp
->hb_body
);
224 plim
= (word
*)((((word
)hbp
) + HBLKSIZE
)
225 - WORDS_TO_BYTES(sz
));
227 /* go through all words in block */
229 if( mark_bit_from_hdr(hhdr
, word_no
) ) {
233 /* object is available - put on list */
236 /* Clear object, advance p to next object in the process */
238 p
++; /* Skip link field */
239 # if defined(SMALL_CONFIG) && defined(ALIGN_DOUBLE)
240 /* We assert that sz must be even */
255 GC_mem_found
+= n_words_found
;
263 * A special case for 2 word composite objects (e.g. cons cells):
266 ptr_t
GC_reclaim_clear2(hbp
, hhdr
, list
)
267 register struct hblk
*hbp
; /* ptr to current heap block */
271 register word
* mark_word_addr
= &(hhdr
->hb_marks
[divWORDSZ(HDR_WORDS
)]);
272 register word
*p
, *plim
;
274 register int n_words_found
= 0;
276 register word mark_word
;
278 # define DO_OBJ(start_displ) \
279 if (!(mark_word & ((word)1 << start_displ))) { \
280 p[start_displ] = (word)list; \
281 list = (ptr_t)(p+start_displ); \
282 p[start_displ+1] = 0; \
286 p
= (word
*)(hbp
->hb_body
);
287 plim
= (word
*)(((word
)hbp
) + HBLKSIZE
);
289 /* go through all words in block */
291 mark_word
= *mark_word_addr
++;
292 for (i
= 0; i
< WORDSZ
; i
+= 8) {
302 GC_mem_found
+= n_words_found
;
309 * Another special case for 4 word composite objects:
312 ptr_t
GC_reclaim_clear4(hbp
, hhdr
, list
)
313 register struct hblk
*hbp
; /* ptr to current heap block */
317 register word
* mark_word_addr
= &(hhdr
->hb_marks
[divWORDSZ(HDR_WORDS
)]);
318 register word
*p
, *plim
;
320 register int n_words_found
= 0;
322 register word mark_word
;
323 # define DO_OBJ(start_displ) \
324 if (!(mark_word & ((word)1 << start_displ))) { \
325 p[start_displ] = (word)list; \
326 list = (ptr_t)(p+start_displ); \
327 p[start_displ+1] = 0; \
328 CLEAR_DOUBLE(p + start_displ + 2); \
332 p
= (word
*)(hbp
->hb_body
);
333 plim
= (word
*)(((word
)hbp
) + HBLKSIZE
);
335 /* go through all words in block */
337 mark_word
= *mark_word_addr
++;
346 # if CPP_WORDSZ == 64
359 GC_mem_found
+= n_words_found
;
365 #endif /* !SMALL_CONFIG */
367 /* The same thing, but don't clear objects: */
369 ptr_t
GC_reclaim_uninit(hbp
, hhdr
, sz
, list
)
370 register struct hblk
*hbp
; /* ptr to current heap block */
375 register int word_no
;
376 register word
*p
, *plim
;
378 register int n_words_found
= 0;
381 p
= (word
*)(hbp
->hb_body
);
383 plim
= (word
*)((((word
)hbp
) + HBLKSIZE
)
384 - WORDS_TO_BYTES(sz
));
386 /* go through all words in block */
388 if( !mark_bit_from_hdr(hhdr
, word_no
) ) {
390 /* object is available - put on list */
398 GC_mem_found
+= n_words_found
;
403 /* Don't really reclaim objects, just check for unmarked ones: */
405 void GC_reclaim_check(hbp
, hhdr
, sz
)
406 register struct hblk
*hbp
; /* ptr to current heap block */
410 register int word_no
;
411 register word
*p
, *plim
;
413 register int n_words_found
= 0;
416 p
= (word
*)(hbp
->hb_body
);
418 plim
= (word
*)((((word
)hbp
) + HBLKSIZE
)
419 - WORDS_TO_BYTES(sz
));
421 /* go through all words in block */
423 if( !mark_bit_from_hdr(hhdr
, word_no
) ) {
424 FOUND_FREE(hbp
, word_no
);
433 * Another special case for 2 word atomic objects:
436 ptr_t
GC_reclaim_uninit2(hbp
, hhdr
, list
)
437 register struct hblk
*hbp
; /* ptr to current heap block */
441 register word
* mark_word_addr
= &(hhdr
->hb_marks
[divWORDSZ(HDR_WORDS
)]);
442 register word
*p
, *plim
;
444 register int n_words_found
= 0;
446 register word mark_word
;
448 # define DO_OBJ(start_displ) \
449 if (!(mark_word & ((word)1 << start_displ))) { \
450 p[start_displ] = (word)list; \
451 list = (ptr_t)(p+start_displ); \
455 p
= (word
*)(hbp
->hb_body
);
456 plim
= (word
*)(((word
)hbp
) + HBLKSIZE
);
458 /* go through all words in block */
460 mark_word
= *mark_word_addr
++;
461 for (i
= 0; i
< WORDSZ
; i
+= 8) {
471 GC_mem_found
+= n_words_found
;
478 * Another special case for 4 word atomic objects:
481 ptr_t
GC_reclaim_uninit4(hbp
, hhdr
, list
)
482 register struct hblk
*hbp
; /* ptr to current heap block */
486 register word
* mark_word_addr
= &(hhdr
->hb_marks
[divWORDSZ(HDR_WORDS
)]);
487 register word
*p
, *plim
;
489 register int n_words_found
= 0;
491 register word mark_word
;
492 # define DO_OBJ(start_displ) \
493 if (!(mark_word & ((word)1 << start_displ))) { \
494 p[start_displ] = (word)list; \
495 list = (ptr_t)(p+start_displ); \
499 p
= (word
*)(hbp
->hb_body
);
500 plim
= (word
*)(((word
)hbp
) + HBLKSIZE
);
502 /* go through all words in block */
504 mark_word
= *mark_word_addr
++;
513 # if CPP_WORDSZ == 64
526 GC_mem_found
+= n_words_found
;
532 /* Finally the one word case, which never requires any clearing: */
534 ptr_t
GC_reclaim1(hbp
, hhdr
, list
)
535 register struct hblk
*hbp
; /* ptr to current heap block */
539 register word
* mark_word_addr
= &(hhdr
->hb_marks
[divWORDSZ(HDR_WORDS
)]);
540 register word
*p
, *plim
;
542 register int n_words_found
= 0;
544 register word mark_word
;
546 # define DO_OBJ(start_displ) \
547 if (!(mark_word & ((word)1 << start_displ))) { \
548 p[start_displ] = (word)list; \
549 list = (ptr_t)(p+start_displ); \
553 p
= (word
*)(hbp
->hb_body
);
554 plim
= (word
*)(((word
)hbp
) + HBLKSIZE
);
556 /* go through all words in block */
558 mark_word
= *mark_word_addr
++;
559 for (i
= 0; i
< WORDSZ
; i
+= 4) {
569 GC_mem_found
+= n_words_found
;
575 #endif /* !SMALL_CONFIG */
578 * Restore unmarked small objects in the block pointed to by hbp
579 * to the appropriate object free list.
580 * If entirely empty blocks are to be completely deallocated, then
581 * caller should perform that check.
583 void GC_reclaim_small_nonempty_block(hbp
, report_if_found
)
584 register struct hblk
*hbp
; /* ptr to current heap block */
585 int report_if_found
; /* Abort if a reclaimable object is found */
588 word sz
; /* size of objects in current block */
589 struct obj_kind
* ok
;
596 hhdr
-> hb_last_reclaimed
= (unsigned short) GC_gc_no
;
597 kind
= hhdr
-> hb_obj_kind
;
598 ok
= &GC_obj_kinds
[kind
];
599 flh
= &(ok
-> ok_freelist
[sz
]);
601 if (report_if_found
) {
602 GC_reclaim_check(hbp
, hhdr
, sz
);
603 } else if (ok
-> ok_init
) {
605 # ifndef SMALL_CONFIG
607 full
= GC_block_nearly_full1(hhdr
, 0xffffffffl
);
608 if (TRUE
== full
) goto out
;
609 if (FALSE
== full
) GC_write_hint(hbp
);
610 /* In the DONT_KNOW case, we let reclaim fault. */
611 *flh
= GC_reclaim1(hbp
, hhdr
, *flh
);
614 full
= GC_block_nearly_full1(hhdr
, 0x55555555l
);
615 if (TRUE
== full
) goto out
;
616 if (FALSE
== full
) GC_write_hint(hbp
);
617 *flh
= GC_reclaim_clear2(hbp
, hhdr
, *flh
);
620 full
= GC_block_nearly_full1(hhdr
, 0x11111111l
);
621 if (TRUE
== full
) goto out
;
622 if (FALSE
== full
) GC_write_hint(hbp
);
623 *flh
= GC_reclaim_clear4(hbp
, hhdr
, *flh
);
627 full
= GC_block_nearly_full(hhdr
);
628 if (TRUE
== full
) goto out
;
629 if (FALSE
== full
) GC_write_hint(hbp
);
630 *flh
= GC_reclaim_clear(hbp
, hhdr
, sz
, *flh
);
635 # ifndef SMALL_CONFIG
637 full
= GC_block_nearly_full1(hhdr
, 0xffffffffl
);
638 if (TRUE
== full
) goto out
;
639 if (FALSE
== full
) GC_write_hint(hbp
);
640 *flh
= GC_reclaim1(hbp
, hhdr
, *flh
);
643 full
= GC_block_nearly_full1(hhdr
, 0x55555555l
);
644 if (TRUE
== full
) goto out
;
645 if (FALSE
== full
) GC_write_hint(hbp
);
646 *flh
= GC_reclaim_uninit2(hbp
, hhdr
, *flh
);
649 full
= GC_block_nearly_full1(hhdr
, 0x11111111l
);
650 if (TRUE
== full
) goto out
;
651 if (FALSE
== full
) GC_write_hint(hbp
);
652 *flh
= GC_reclaim_uninit4(hbp
, hhdr
, *flh
);
656 full
= GC_block_nearly_full(hhdr
);
657 if (TRUE
== full
) goto out
;
658 if (FALSE
== full
) GC_write_hint(hbp
);
659 *flh
= GC_reclaim_uninit(hbp
, hhdr
, sz
, *flh
);
664 if (IS_UNCOLLECTABLE(kind
)) GC_set_hdr_marks(hhdr
);
668 * Restore an unmarked large object or an entirely empty blocks of small objects
669 * to the heap block free list.
670 * Otherwise enqueue the block for later processing
671 * by GC_reclaim_small_nonempty_block.
672 * If report_if_found is TRUE, then process any block immediately, and
673 * simply report free objects; do not actually reclaim them.
675 void GC_reclaim_block(hbp
, report_if_found
)
676 register struct hblk
*hbp
; /* ptr to current heap block */
677 word report_if_found
; /* Abort if a reclaimable object is found */
680 register word sz
; /* size of objects in current block */
681 register struct obj_kind
* ok
;
686 ok
= &GC_obj_kinds
[hhdr
-> hb_obj_kind
];
688 if( sz
> MAXOBJSZ
) { /* 1 big object */
689 if( !mark_bit_from_hdr(hhdr
, HDR_WORDS
) ) {
690 if (report_if_found
) {
691 FOUND_FREE(hbp
, HDR_WORDS
);
700 GC_bool empty
= GC_block_empty(hhdr
);
701 if (report_if_found
) {
702 GC_reclaim_small_nonempty_block(hbp
, (int)report_if_found
);
705 GC_mem_found
+= BYTES_TO_WORDS(HBLKSIZE
);
709 /* group of smaller objects, enqueue the real work */
710 rlh
= &(ok
-> ok_reclaim_list
[sz
]);
711 hhdr
-> hb_next
= *rlh
;
717 #if !defined(NO_DEBUGGING)
718 /* Routines to gather and print heap block info */
719 /* intended for debugging. Otherwise should be called */
721 static size_t number_of_blocks
;
722 static size_t total_bytes
;
724 /* Number of set bits in a word. Not performance critical. */
725 static int set_bits(n
)
729 register int result
= 0;
738 /* Return the number of set mark bits in the given header */
739 int GC_n_set_marks(hhdr
)
742 register int result
= 0;
745 for (i
= 0; i
< MARK_BITS_SZ
; i
++) {
746 result
+= set_bits(hhdr
-> hb_marks
[i
]);
752 void GC_print_block_descr(h
, dummy
)
756 register hdr
* hhdr
= HDR(h
);
757 register size_t bytes
= WORDS_TO_BYTES(hhdr
-> hb_sz
);
759 GC_printf3("(%lu:%lu,%lu)", (unsigned long)(hhdr
-> hb_obj_kind
),
760 (unsigned long)bytes
,
761 (unsigned long)(GC_n_set_marks(hhdr
)));
762 bytes
+= HDR_BYTES
+ HBLKSIZE
-1;
763 bytes
&= ~(HBLKSIZE
-1);
764 total_bytes
+= bytes
;
768 void GC_print_block_list()
770 GC_printf0("(kind(0=ptrfree,1=normal,2=unc.,3=stubborn):size_in_bytes, #_marks_set)\n");
771 number_of_blocks
= 0;
773 GC_apply_to_all_blocks(GC_print_block_descr
, (word
)0);
774 GC_printf2("\nblocks = %lu, bytes = %lu\n",
775 (unsigned long)number_of_blocks
,
776 (unsigned long)total_bytes
);
779 #endif /* NO_DEBUGGING */
782 * Perform GC_reclaim_block on the entire heap, after first clearing
783 * small object free lists (if we are not just looking for leaks).
785 void GC_start_reclaim(report_if_found
)
786 int report_if_found
; /* Abort if a GC_reclaimable object is found */
790 /* Clear reclaim- and free-lists */
791 for (kind
= 0; kind
< GC_n_kinds
; kind
++) {
794 register struct hblk
** rlp
;
795 register struct hblk
** rlim
;
796 register struct hblk
** rlist
= GC_obj_kinds
[kind
].ok_reclaim_list
;
798 if (rlist
== 0) continue; /* This kind not used. */
799 if (!report_if_found
) {
800 lim
= &(GC_obj_kinds
[kind
].ok_freelist
[MAXOBJSZ
+1]);
801 for( fop
= GC_obj_kinds
[kind
].ok_freelist
; fop
< lim
; fop
++ ) {
804 } /* otherwise free list objects are marked, */
805 /* and its safe to leave them */
806 rlim
= rlist
+ MAXOBJSZ
+1;
807 for( rlp
= rlist
; rlp
< rlim
; rlp
++ ) {
813 GC_printf0("GC_reclaim: current block sizes:\n");
814 GC_print_block_list();
817 /* Go through all heap blocks (in hblklist) and reclaim unmarked objects */
818 /* or enqueue the block for later processing. */
819 GC_apply_to_all_blocks(GC_reclaim_block
, (word
)report_if_found
);
822 /* This is a very stupid thing to do. We make it possible anyway, */
823 /* so that you can convince yourself that it really is very stupid. */
824 GC_reclaim_all((GC_stop_func
)0, FALSE
);
830 * Sweep blocks of the indicated object size and kind until either the
831 * appropriate free list is nonempty, or there are no more blocks to
834 void GC_continue_reclaim(sz
, kind
)
839 register struct hblk
* hbp
;
840 register struct obj_kind
* ok
= &(GC_obj_kinds
[kind
]);
841 struct hblk
** rlh
= ok
-> ok_reclaim_list
;
842 ptr_t
*flh
= &(ok
-> ok_freelist
[sz
]);
844 if (rlh
== 0) return; /* No blocks of this kind. */
846 while ((hbp
= *rlh
) != 0) {
848 *rlh
= hhdr
-> hb_next
;
849 GC_reclaim_small_nonempty_block(hbp
, FALSE
);
850 if (*flh
!= 0) break;
855 * Reclaim all small blocks waiting to be reclaimed.
856 * Abort and return FALSE when/if (*stop_func)() returns TRUE.
857 * If this returns TRUE, then it's safe to restart the world
858 * with incorrectly cleared mark bits.
859 * If ignore_old is TRUE, then reclaim only blocks that have been
860 * recently reclaimed, and discard the rest.
861 * Stop_func may be 0.
863 GC_bool
GC_reclaim_all(stop_func
, ignore_old
)
864 GC_stop_func stop_func
;
870 register struct hblk
* hbp
;
871 register struct obj_kind
* ok
;
875 CLOCK_TYPE start_time
;
876 CLOCK_TYPE done_time
;
878 GET_TIME(start_time
);
881 for (kind
= 0; kind
< GC_n_kinds
; kind
++) {
882 ok
= &(GC_obj_kinds
[kind
]);
883 rlp
= ok
-> ok_reclaim_list
;
884 if (rlp
== 0) continue;
885 for (sz
= 1; sz
<= MAXOBJSZ
; sz
++) {
887 while ((hbp
= *rlh
) != 0) {
888 if (stop_func
!= (GC_stop_func
)0 && (*stop_func
)()) {
892 *rlh
= hhdr
-> hb_next
;
893 if (!ignore_old
|| hhdr
-> hb_last_reclaimed
== GC_gc_no
- 1) {
894 /* It's likely we'll need it this time, too */
895 /* It's been touched recently, so this */
896 /* shouldn't trigger paging. */
897 GC_reclaim_small_nonempty_block(hbp
, FALSE
);
904 GC_printf1("Disposing of reclaim lists took %lu msecs\n",
905 MS_TIME_DIFF(done_time
,start_time
));