src/runtime/gc-internal.h

   1 /*
   2  * garbage collection - shared definitions for modules "inside" the GC system
   3  */
   4
   5 /*
   6  * This software is part of the SBCL system. See the README file for
   7  * more information.
   8  *
   9  * This software is derived from the CMU CL system, which was
  10  * written at Carnegie Mellon University and released into the
  11  * public domain. The software is in the public domain and is
  12  * provided with absolutely no warranty. See the COPYING and CREDITS
  13  * files for more information.
  14  */
  15
  16 #ifndef _GC_INTERNAL_H_
  17 #define _GC_INTERNAL_H_
  18
  19 #include "genesis/code.h"
  20 #include "genesis/simple-fun.h"
  21 #include "genesis/weak-pointer.h"
  22 #include "thread.h"
  23 #include "interr.h"
  24
  25 #ifdef LISP_FEATURE_GENCGC
  26 #include "gencgc-internal.h"
  27 #else
  28 #include "cheneygc-internal.h"
  29 #endif
  30
  31 /// Enable extra debug-only checks if DEBUG
  32 #ifdef DEBUG
  33 # define gc_dcheck(ex) gc_assert(ex)
  34 #else
  35 # define gc_dcheck(ex) ((void)0)
  36 #endif
  37
  38 /// Disable all assertions if NDEBUG
  39 #ifdef NDEBUG
  40 # define gc_assert(ex) ((void)0)
  41 # define gc_assert_verbose(ex, fmt, ...) ((void)0)
  42 #else
  43 # define gc_assert(ex)                                                 \
  44 do {                                                                   \
  45     if (!(ex)) gc_abort();                                             \
  46 } while (0)
  47 # define gc_assert_verbose(ex, fmt, ...)                               \
  48 do {                                                                   \
  49     if (!(ex)) {                                                       \
  50         fprintf(stderr, fmt, ## __VA_ARGS__);                          \
  51         gc_abort();                                                    \
  52     }                                                                  \
  53 } while (0)
  54 #endif
  55
  56 #define gc_abort()                                                     \
  57   lose("GC invariant lost, file \"%s\", line %d\n", __FILE__, __LINE__)
  58
  59 #define CEILING(x,y) (((x) + ((y) - 1)) & (~((y) - 1)))
  60
  61 /* FIXME: Shouldn't this be defined in sbcl.h? */
  62
  63 #if defined(LISP_FEATURE_SPARC) || defined(LISP_FEATURE_ARM)
  64 #define FUN_RAW_ADDR_OFFSET 0
  65 #else
  66 #define FUN_RAW_ADDR_OFFSET (offsetof(struct simple_fun, code) - FUN_POINTER_LOWTAG)
  67 #endif
  68
  69 static inline unsigned short
  70 #ifdef LISP_FEATURE_64_BIT
  71 code_n_funs(struct code* code) { return ((code)->header >> 32) & 0x7FFF; }
  72 #define FIRST_SIMPLE_FUN_OFFSET(code) ((code)->header >> 48)
  73 #else
  74 code_n_funs(struct code* code) { return fixnum_value((code)->n_entries) & 0x3FFF; }
  75 #define FIRST_SIMPLE_FUN_OFFSET(code) ((code)->n_entries >> 16)
  76 #endif
  77
  78 // Iterate over the native pointers to each function in 'code_var'
  79 // offsets are stored as the number of bytes into the instructions
  80 // portion of the code object at which the simple-fun object resides.
  81 // We use bytes, not words, because that's what the COMPUTE-FUN vop expects.
  82 // But the offsets could be compressed further if we chose to use words,
  83 // which might allow storing them as (unsigned-byte 16),
  84 // as long as provision is made for ultra huge simple-funs. (~ .5MB)
  85 //
  86 // Note that the second assignment to _offset_ is OK: while it technically
  87 // oversteps the bounds of the indices of the fun offsets, it can not
  88 // run off the end of the code.
  89 #define for_each_simple_fun(index_var,fun_var,code_var,assertp,guts)        \
  90   { int _nfuns_ = code_n_funs(code_var);                                    \
  91     if (_nfuns_ > 0) {                                                      \
  92       char *_insts_ = (char*)(code_var) +                                   \
  93         (code_header_words((code_var)->header)<<WORD_SHIFT);                \
  94       int index_var = 0;                                                    \
  95       int _offset_ = FIRST_SIMPLE_FUN_OFFSET(code_var);                     \
  96       do {                                                                  \
  97        struct simple_fun* fun_var = (struct simple_fun*)(_insts_+_offset_); \
  98        if (assertp)                                                         \
  99          gc_assert(widetag_of(fun_var->header)==SIMPLE_FUN_WIDETAG);        \
 100        guts ;                                                               \
 101        _offset_ = ((unsigned int*)_insts_)[index_var];                      \
 102       } while (++index_var < _nfuns_);                                      \
 103   }}
 104
 105 #define SIMPLE_FUN_SCAV_START(fun_ptr) &fun_ptr->name
 106 #define SIMPLE_FUN_SCAV_NWORDS(fun_ptr) ((lispobj*)fun_ptr->code - &fun_ptr->name)
 107
 108 /* values for the *_alloc_* parameters, also see the commentary for
 109  * struct page in gencgc-internal.h. These constants are used in gc-common,
 110  * so they can't easily be made gencgc-only */
 111 #define FREE_PAGE_FLAG 0
 112 #define BOXED_PAGE_FLAG 1
 113 #define UNBOXED_PAGE_FLAG 2
 114 #define OPEN_REGION_PAGE_FLAG 4
 115 #define CODE_PAGE_FLAG        (BOXED_PAGE_FLAG|UNBOXED_PAGE_FLAG)
 116
 117 // Gencgc distinguishes between "quick" and "ordinary" requests.
 118 // Even on cheneygc we need this flag, but it's actually just ignored.
 119 #define ALLOC_QUICK 1
 120
 121 #ifdef LISP_FEATURE_GENCGC
 122 #include "gencgc-alloc-region.h"
 123 void *
 124 gc_alloc_with_region(sword_t nbytes,int page_type_flag, struct alloc_region *my_region,
 125                      int quick_p);
 126 static inline void *
 127 gc_general_alloc(sword_t nbytes, int page_type_flag, int quick_p)
 128 {
 129     struct alloc_region *my_region;
 130 #ifdef LISP_FEATURE_SEGREGATED_CODE
 131     if (1 <= page_type_flag && page_type_flag <= 3) {
 132         my_region = &gc_alloc_regions[page_type_flag-1];
 133 #else
 134     if (UNBOXED_PAGE_FLAG == page_type_flag) {
 135         my_region = &unboxed_region;
 136     } else if (BOXED_PAGE_FLAG & page_type_flag) {
 137         my_region = &boxed_region;
 138 #endif
 139     } else {
 140         lose("bad page type flag: %d", page_type_flag);
 141     }
 142     return gc_alloc_with_region(nbytes, page_type_flag, my_region, quick_p);
 143 }
 144 #else
 145 extern void *gc_general_alloc(word_t nbytes,int page_type_flag,int quick_p);
 146 #endif
 147
 148 #define CHECK_COPY_PRECONDITIONS(object, nwords) \
 149     gc_dcheck(is_lisp_pointer(object)); \
 150     gc_dcheck(from_space_p(object)); \
 151     gc_dcheck((nwords & 0x01) == 0)
 152
 153 #define CHECK_COPY_POSTCONDITIONS(copy, lowtag) \
 154     gc_dcheck(lowtag_of(copy) == lowtag); \
 155     gc_dcheck(!from_space_p(copy));
 156
 157 static inline lispobj
 158 gc_general_copy_object(lispobj object, long nwords, int page_type_flag)
 159 {
 160     lispobj *new;
 161
 162     CHECK_COPY_PRECONDITIONS(object, nwords);
 163
 164     /* Allocate space. */
 165     new = gc_general_alloc(nwords*N_WORD_BYTES, page_type_flag, ALLOC_QUICK);
 166
 167     /* Copy the object. */
 168     memcpy(new,native_pointer(object),nwords*N_WORD_BYTES);
 169
 170     return make_lispobj(new, lowtag_of(object));
 171 }
 172
 173 extern sword_t (*scavtab[256])(lispobj *where, lispobj object);
 174 extern sword_t (*sizetab[256])(lispobj *where);
 175 #define OBJECT_SIZE(header,where) \
 176   (is_cons_half(header)?2:sizetab[widetag_of(header)](where))
 177
 178 extern struct weak_pointer *weak_pointers; /* in gc-common.c */
 179 extern struct hash_table *weak_hash_tables; /* in gc-common.c */
 180
 181 extern void heap_scavenge(lispobj *start, lispobj *limit);
 182 extern sword_t scavenge(lispobj *start, sword_t n_words);
 183 extern void scavenge_interrupt_contexts(struct thread *thread);
 184 extern void scav_weak_hash_tables(void);
 185 extern void scav_binding_stack(lispobj*, lispobj*);
 186 extern void scan_weak_hash_tables(void);
 187 extern void scan_weak_pointers(void);
 188
 189 lispobj  copy_large_unboxed_object(lispobj object, sword_t nwords);
 190 lispobj  copy_unboxed_object(lispobj object, sword_t nwords);
 191 lispobj  copy_large_object(lispobj object, sword_t nwords);
 192 lispobj  copy_object(lispobj object, sword_t nwords);
 193 struct simple_fun *code_fun_addr(struct code*, int);
 194
 195 lispobj *search_read_only_space(void *pointer);
 196 lispobj *search_static_space(void *pointer);
 197 lispobj *search_immobile_space(void *pointer);
 198 lispobj *search_dynamic_space(void *pointer);
 199
 200 lispobj *gc_search_space3(void *pointer, lispobj *start, void *limit);
 201 static inline lispobj *gc_search_space(lispobj *start, void *pointer) {
 202     return gc_search_space3(pointer,
 203                             start,
 204                             (void*)(1+((lispobj)pointer | LOWTAG_MASK)));
 205 }
 206 struct vector *symbol_name(lispobj*);
 207
 208 static inline int instruction_ptr_p(void *pointer, lispobj *start_addr)
 209 {
 210     return widetag_of(*start_addr) == CODE_HEADER_WIDETAG &&
 211         pointer >= (void*)(start_addr + code_header_words(*start_addr));
 212 }
 213 extern int properly_tagged_p_internal(lispobj pointer, lispobj *start_addr);
 214 static inline int properly_tagged_descriptor_p(void *pointer, lispobj *start_addr) {
 215   return is_lisp_pointer((lispobj)pointer) &&
 216     properly_tagged_p_internal((lispobj)pointer, start_addr);
 217 }
 218
 219 extern void scavenge_control_stack(struct thread *th);
 220 extern void scrub_control_stack(void);
 221 extern void scrub_thread_control_stack(struct thread *);
 222
 223 #include "fixnump.h"
 224
 225 #ifdef LISP_FEATURE_GENCGC
 226 #include "gencgc-internal.h"
 227 #else
 228 #include "cheneygc-internal.h"
 229 #endif
 230
 231 #if N_WORD_BITS == 32
 232 # define SIMPLE_ARRAY_WORD_WIDETAG SIMPLE_ARRAY_UNSIGNED_BYTE_32_WIDETAG
 233 #elif N_WORD_BITS == 64
 234 # define SIMPLE_ARRAY_WORD_WIDETAG SIMPLE_ARRAY_UNSIGNED_BYTE_64_WIDETAG
 235 #endif
 236
 237 extern void
 238 instance_scan(void (*proc)(), lispobj *instance_ptr, sword_t n_words, lispobj bitmap);
 239
 240 #include "genesis/bignum.h"
 241 extern boolean positive_bignum_logbitp(int,struct bignum*);
 242
 243 #ifdef LISP_FEATURE_IMMOBILE_SPACE
 244
 245 extern void fixup_immobile_refs(lispobj (*)(lispobj), lispobj, struct code*);
 246 extern lispobj fdefn_raw_referent(struct fdefn *fdefn);
 247
 248 static inline boolean immobile_space_p(lispobj obj)
 249 {
 250   return IMMOBILE_SPACE_START <= obj && obj < IMMOBILE_SPACE_END;
 251 }
 252
 253 typedef int low_page_index_t;
 254 static inline low_page_index_t find_immobile_page_index(void *addr)
 255 {
 256   if (addr >= (void*)IMMOBILE_SPACE_START) {
 257       // Must use full register size here to avoid truncation of quotient
 258       // and bogus result!
 259       page_index_t index =
 260           ((uintptr_t)addr -
 261            (uintptr_t)IMMOBILE_SPACE_START) / IMMOBILE_CARD_BYTES;
 262       if (index < (int)(IMMOBILE_SPACE_SIZE/IMMOBILE_CARD_BYTES))
 263           return index;
 264   }
 265   return -1;
 266 }
 267 int immobile_obj_younger_p(lispobj,generation_index_t);
 268 void promote_immobile_obj(lispobj*,int);
 269
 270 #define IMMOBILE_OBJ_VISITED_FLAG    0x10
 271 #define IMMOBILE_OBJ_GENERATION_MASK 0x0f // mask off the VISITED flag
 272
 273 #define IMMOBILE_VARYOBJ_SUBSPACE_START (IMMOBILE_SPACE_START+IMMOBILE_FIXEDOBJ_SUBSPACE_SIZE)
 274
 275 // Note: this does not work on a SIMPLE-FUN
 276 // because a simple-fun header does not contain a generation.
 277 #define __immobile_obj_generation(x) (__immobile_obj_gen_bits(x) & IMMOBILE_OBJ_GENERATION_MASK)
 278
 279 #ifdef LISP_FEATURE_LITTLE_ENDIAN
 280 static inline int immobile_obj_gen_bits(lispobj* pointer) // native pointer
 281 {
 282   if (widetag_of(*pointer) == SIMPLE_FUN_WIDETAG)
 283     pointer = fun_code_header(pointer);
 284   return ((generation_index_t*)pointer)[3];
 285 }
 286 // Faster way when we know that the object can't be a simple-fun,
 287 // such as when walking the immobile space.
 288 static inline int __immobile_obj_gen_bits(lispobj* pointer) // native pointer
 289 {
 290   return ((generation_index_t*)pointer)[3];
 291 }
 292 #else
 293 #error "Need to define immobile_obj_gen_bits() for big-endian"
 294 #endif /* little-endian */
 295
 296 static inline boolean immobile_filler_p(lispobj* obj) {
 297   return *(int*)obj == (2<<N_WIDETAG_BITS | CODE_HEADER_WIDETAG);
 298 }
 299
 300 #define set_instance_layout(instance_ptr,layout) \
 301   instance_ptr[0] = (layout << 32) | (instance_ptr[0] & 0xFFFFFFFF)
 302
 303 #endif /* immobile space */
 304
 305 #define WEAK_POINTER_NWORDS \
 306         CEILING((sizeof(struct weak_pointer) / sizeof(lispobj)), 2)
 307
 308 static inline boolean weak_pointer_breakable_p(struct weak_pointer *wp)
 309 {
 310     lispobj pointee = wp->value;
 311     // A broken weak-pointer's value slot has unbound-marker
 312     // which does not satisfy is_lisp_pointer().
 313     return is_lisp_pointer(pointee) && (from_space_p(pointee)
 314 #ifdef LISP_FEATURE_IMMOBILE_SPACE
 315          || (immobile_space_p(pointee) &&
 316              immobile_obj_gen_bits(native_pointer(pointee)) == from_space)
 317 #endif
 318             );
 319 }
 320
 321 /// Same as Lisp LOGBITP, except no negative bignums allowed.
 322 static inline boolean layout_bitmap_logbitp(int index, lispobj bitmap)
 323 {
 324     if (fixnump(bitmap))
 325       return (index < (N_WORD_BITS - N_FIXNUM_TAG_BITS))
 326           ? (bitmap >> (index+N_FIXNUM_TAG_BITS)) & 1
 327           : (sword_t)bitmap < 0;
 328     return positive_bignum_logbitp(index, (struct bignum*)native_pointer(bitmap));
 329 }
 330
 331 #endif /* _GC_INTERNAL_H_ */