From b68515ffa5f2c647600cbe53070c77ee3fb99cb3 Mon Sep 17 00:00:00 2001 From: Douglas Katzman Date: Wed, 16 Aug 2017 19:47:10 -0400 Subject: [PATCH] Untangle spaghetti in process_directory() - The code checked for excessive dynamic space size only *after* mapping dynamic space - And then checked for mismatched expected/actual address only *after* mapping at the wrong address. - And for a bogus space ID only *after* mapping it - And repeated the lossage message in 6 different places Other changes: - always assign anon_dynamic_space_start even if not on linux - abstract out the HPUX noise - delay allocation of gencgc page tables until all spaces have been loaded from the core file so that calloc() won't claim memory that would have been preferred for dynamic space, which is no longer made in allocate_spaces(). --- src/compiler/generic/genesis.lisp | 2 +- src/runtime/coreparse.c | 270 +++++++++++++++++--------------------- src/runtime/gencgc.c | 21 ++- src/runtime/marknsweepgc.c | 161 ++++++++++++----------- 4 files changed, 220 insertions(+), 234 deletions(-) diff --git a/src/compiler/generic/genesis.lisp b/src/compiler/generic/genesis.lisp index c26685ba1..2d007cdd1 100644 --- a/src/compiler/generic/genesis.lisp +++ b/src/compiler/generic/genesis.lisp @@ -189,7 +189,7 @@ (defconstant immobile-varyobj-core-space-id 5) (defvar *immobile-space-map* nil)) -(defconstant max-core-space-id 5) +(defconstant max-core-space-id (+ 3 #!+immobile-space 2)) (defconstant deflated-core-space-id-flag 8) ;;; a GENESIS-time representation of a memory space (e.g. read-only diff --git a/src/runtime/coreparse.c b/src/runtime/coreparse.c index 40a0493f8..c4bde90e1 100644 --- a/src/runtime/coreparse.c +++ b/src/runtime/coreparse.c @@ -171,12 +171,15 @@ lose: return -1; } -/* If more platforms doesn't support overlapping mmap rename this +#ifndef LISP_FEATURE_HPUX +#define load_core_bytes(fd, where, addr, len) os_map(fd, where, addr, len) +#else +#define load_core_bytes(fd, where, addr, len) copy_core_bytes(fd, where, addr, len) +/* If more platforms don't support overlapping mmap rename this * def to something like ifdef nommapoverlap */ /* currently hpux only */ -#ifdef LISP_FEATURE_HPUX -void copy_core_bytes(int fd, os_vm_offset_t offset, - os_vm_address_t addr, int len) +static void copy_core_bytes(int fd, os_vm_offset_t offset, + os_vm_address_t addr, int len) { unsigned char buf[4096]; int c,x; @@ -205,10 +208,13 @@ void copy_core_bytes(int fd, os_vm_offset_t offset, } #endif -#ifdef LISP_FEATURE_SB_CORE_COMPRESSION +#ifndef LISP_FEATURE_SB_CORE_COMPRESSION +# define inflate_core_bytes(fd,offset,addr,len) \ + lose("This runtime was not built with zlib-compressed core support... aborting\n") +#else # define ZLIB_BUFFER_SIZE (1u<<16) -void inflate_core_bytes(int fd, os_vm_offset_t offset, - os_vm_address_t addr, int len) +static void inflate_core_bytes(int fd, os_vm_offset_t offset, + os_vm_address_t addr, int len) { z_stream stream; unsigned char* buf = successful_malloc(ZLIB_BUFFER_SIZE); @@ -552,76 +558,100 @@ void relocate_heap(lispobj* want, lispobj* got, uword_t len) int merge_core_pages = -1; -#ifdef LISP_FEATURE_LINUX os_vm_address_t anon_dynamic_space_start; -#endif static void process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset) { extern void immobile_space_coreparse(uword_t,uword_t); - struct ndir_entry *entry; - int compressed; extern void write_protect_immobile_space(); + struct ndir_entry *entry; - FSHOW((stderr, "/process_directory(..), count=%d\n", count)); + struct { + uword_t len; + uword_t base; + } spaces[MAX_CORE_SPACE_ID+1] = { + {0, 0}, // blank for space ID 0 +#ifdef LISP_FEATURE_GENCGC + {0, DYNAMIC_SPACE_START}, +#else + {0, 0}, +#endif + // This order is determined by constants in compiler/generic/genesis + {0, STATIC_SPACE_START}, + {0, READ_ONLY_SPACE_START}, +#ifdef LISP_FEATURE_IMMOBILE_SPACE + {0, IMMOBILE_SPACE_START}, + {0, IMMOBILE_VARYOBJ_SUBSPACE_START} +#endif + }; for (entry = (struct ndir_entry *) ptr; --count>= 0; ++entry) { - - compressed = 0; sword_t id = entry->identifier; - if (id <= (MAX_CORE_SPACE_ID | DEFLATED_CORE_SPACE_ID_FLAG)) { - if (id & DEFLATED_CORE_SPACE_ID_FLAG) - compressed = 1; - id &= ~(DEFLATED_CORE_SPACE_ID_FLAG); + uword_t addr = (os_vm_page_size * entry->address); + int compressed = id & DEFLATED_CORE_SPACE_ID_FLAG; + id -= compressed; + if (id < 1 || id > MAX_CORE_SPACE_ID) + lose("unknown space ID %ld addr %p\n", id, addr); + +#ifndef LISP_FEATURE_RELOCATABLE_HEAP + int enforce_address = 1; +#else + // Only enforce other spaces' addresses + int enforce_address = id != DYNAMIC_CORE_SPACE_ID; +#endif + if (enforce_address) { + int fail; +#ifdef LISP_FEATURE_CHENEYGC + if (id == DYNAMIC_CORE_SPACE_ID) { + if ((fail = (addr != DYNAMIC_0_SPACE_START) && + (addr != DYNAMIC_1_SPACE_START)) != 0) + fprintf(stderr, "in core: %p; in runtime: %p or %p\n", + (void*)addr, + (void*)DYNAMIC_0_SPACE_START, + (void*)DYNAMIC_1_SPACE_START); + } else +#endif + if ((fail = (addr != spaces[id].base)) != 0) + fprintf(stderr, "in core: %p; in runtime: %p\n", + (void*)addr, (void*)spaces[id].base); + char *names[MAX_CORE_SPACE_ID] = { + "DYNAMIC", "STATIC", "READ_ONLY", "IMMOBILE", "IMMOBILE" + }; + if (fail) + lose("core/runtime address mismatch: %s_SPACE_START", names[id-1]); } - sword_t offset = os_vm_page_size * (1 + entry->data_page); - uword_t corefile_addr = os_vm_page_size * entry->address; - os_vm_address_t addr = (os_vm_address_t)corefile_addr; + spaces[id].base = (uword_t)addr; uword_t len = os_vm_page_size * entry->page_count; + spaces[id].len = len; + if (id == DYNAMIC_CORE_SPACE_ID && len > dynamic_space_size) { + lose("dynamic space too small for core: %luKiB required, %luKiB available.\n", + (unsigned long)len >> 10, + (unsigned long)dynamic_space_size >> 10); + } if (len != 0) { + uword_t __attribute__((unused)) aligned_start; #ifdef LISP_FEATURE_RELOCATABLE_HEAP - // If this entry is for the dynamic space, - // try to map the space now at the address - // specified in the core file. + // Try to map at address requested by the core file. if (id == DYNAMIC_CORE_SPACE_ID) { - uword_t aligned_start; - addr = (os_vm_address_t) - os_validate(MOVABLE, - (os_vm_address_t)maybe_fuzz_address(corefile_addr), - dynamic_space_size); - aligned_start = ((uword_t)addr + GENCGC_CARD_BYTES-1) - & ~(GENCGC_CARD_BYTES-1); - DYNAMIC_SPACE_START = aligned_start; - if (aligned_start > (uword_t)addr) { // not card-aligned - // This could happen only if the OS page size - // is smaller than the GC card size. - // Drop the final card and decrease the dynamic_space_size - // to make the absolute end come out the same. + addr = (uword_t)os_validate(MOVABLE, + (os_vm_address_t)maybe_fuzz_address(addr), + dynamic_space_size); + aligned_start = CEILING(addr, GENCGC_CARD_BYTES); + /* Misalignment can happen only if card size exceeds OS page. + * Drop one card to avoid overrunning the allocated space */ + if (aligned_start > addr) // not card-aligned dynamic_space_size -= GENCGC_CARD_BYTES; - --page_table_pages; - } - addr = (os_vm_address_t)aligned_start; - } -#endif - FSHOW((stderr, "/mapping %ld(0x%lx) bytes at 0x%lx\n", - len, len, (uword_t)addr)); - if (compressed) { -#ifdef LISP_FEATURE_SB_CORE_COMPRESSION - inflate_core_bytes(fd, offset + file_offset, addr, len); -#else - lose("This runtime was not built with zlib-compressed core support... aborting\n"); -#endif - } else { -#ifdef LISP_FEATURE_HPUX - copy_core_bytes(fd, offset + file_offset, addr, len); -#else - os_map(fd, offset + file_offset, addr, len); -#endif + DYNAMIC_SPACE_START = addr = aligned_start; } +#endif /* LISP_FEATURE_RELOCATABLE_HEAP */ + + sword_t offset = os_vm_page_size * (1 + entry->data_page); + if (compressed) + inflate_core_bytes(fd, offset + file_offset, (os_vm_address_t)addr, len); + else + load_core_bytes(fd, offset + file_offset, (os_vm_address_t)addr, len); } - // Compute this only after we possibly affected 'addr' if relocatable - lispobj *free_pointer = (lispobj *) addr + entry->nwords; #ifdef MADV_MERGEABLE if ((merge_core_pages == 1) @@ -629,105 +659,47 @@ process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset) madvise(addr, len, MADV_MERGEABLE); } #endif - FSHOW((stderr, "/space id = %ld, free pointer = %p\n", - id, (uword_t)free_pointer)); - switch (id) { - case DYNAMIC_CORE_SPACE_ID: - if (len > dynamic_space_size) { - fprintf(stderr, - "dynamic space too small for core: %luKiB required, %luKiB available.\n", - (unsigned long)len >> 10, - (unsigned long)dynamic_space_size >> 10); - exit(1); - } -#ifdef LISP_FEATURE_GENCGC -# ifdef LISP_FEATURE_RELOCATABLE_HEAP - if (DYNAMIC_SPACE_START != corefile_addr) - relocate_heap((lispobj*)corefile_addr, (lispobj*)addr, len); -# else - if (addr != (os_vm_address_t)DYNAMIC_SPACE_START) { - fprintf(stderr, "in core: %p; in runtime: %p \n", - (void*)addr, (void*)DYNAMIC_SPACE_START); - lose("core/runtime address mismatch: DYNAMIC_SPACE_START\n"); - } -# endif -#ifdef LISP_FEATURE_X86_64 - tune_asm_routines_for_microarch(); // before WPing immobile space -#endif -# ifdef LISP_FEATURE_IMMOBILE_SPACE - // Delayed until after dynamic space has been mapped - // so that writes into immobile space - // due to core relocation don't fault. - write_protect_immobile_space(); -# endif -#else - if ((addr != (os_vm_address_t)DYNAMIC_0_SPACE_START) && - (addr != (os_vm_address_t)DYNAMIC_1_SPACE_START)) { - fprintf(stderr, "in core: %p; in runtime: %p or %p\n", - (void*)addr, - (void*)DYNAMIC_0_SPACE_START, - (void*)DYNAMIC_1_SPACE_START); - lose("warning: core/runtime address mismatch: DYNAMIC_SPACE_START\n"); - } -#endif + if (id == DYNAMIC_CORE_SPACE_ID) { + /* 'addr' is the actual address if relocatable. + * For cheneygc, this will be whatever the GC was using + * at the time the core was saved. + * For gencgc we don't look at current_dynamic_space */ + current_dynamic_space = (lispobj *)addr; + + lispobj *free_pointer = (lispobj *) addr + entry->nwords; + /* FIXME: why not use set_alloc_pointer() ? */ #if defined(ALLOCATION_POINTER) SetSymbolValue(ALLOCATION_POINTER, (lispobj)free_pointer,0); #else dynamic_space_free_pointer = free_pointer; #endif - /* For stop-and-copy GC, this will be whatever the GC was - * using at the time. With GENCGC, this will always be - * space 0. (We checked above that for GENCGC, - * addr==DYNAMIC_SPACE_START.) */ - current_dynamic_space = (lispobj *)addr; -#ifdef LISP_FEATURE_LINUX - anon_dynamic_space_start = addr + len; - // This assertion is here because of the test in zero_pages_with_mmap() - // which trusts that if addr > anon_dynamic_space_start - // then addr did not come from any file mapping. + anon_dynamic_space_start = (os_vm_address_t)(addr + len); + /* This assertion safeguards the test in zero_pages_with_mmap() + * which trusts that if addr > anon_dynamic_space_start + * then addr did not come from any file mapping. */ gc_assert((lispobj)anon_dynamic_space_start > STATIC_SPACE_END); + } + } + +#ifdef LISP_FEATURE_GENCGC + immobile_space_coreparse(spaces[IMMOBILE_FIXEDOBJ_CORE_SPACE_ID].len, + spaces[IMMOBILE_VARYOBJ_CORE_SPACE_ID].len); +#endif +#ifdef LISP_FEATURE_RELOCATABLE_HEAP + if (DYNAMIC_SPACE_START != spaces[DYNAMIC_CORE_SPACE_ID].base) + relocate_heap((lispobj*)spaces[DYNAMIC_CORE_SPACE_ID].base, + (lispobj*)DYNAMIC_SPACE_START, + spaces[DYNAMIC_CORE_SPACE_ID].len); +#endif +#ifdef LISP_FEATURE_X86_64 + tune_asm_routines_for_microarch(); // before WPing immobile space #endif - break; - case STATIC_CORE_SPACE_ID: - if (addr != (os_vm_address_t)STATIC_SPACE_START) { - fprintf(stderr, "in core: %p - in runtime: %p\n", - (void*)addr, (void*)STATIC_SPACE_START); - lose("core/runtime address mismatch: STATIC_SPACE_START\n"); - } - break; - case READ_ONLY_CORE_SPACE_ID: - if (addr != (os_vm_address_t)READ_ONLY_SPACE_START) { - fprintf(stderr, "in core: %p - in runtime: %p\n", - (void*)addr, (void*)READ_ONLY_SPACE_START); - lose("core/runtime address mismatch: READ_ONLY_SPACE_START\n"); - } - break; #ifdef LISP_FEATURE_IMMOBILE_SPACE - // Immobile space is subdivided into fixed-size and variable-size. - // There is no margin between the two, though for efficiency - // they are written separately to eliminate waste in the core file. - case IMMOBILE_FIXEDOBJ_CORE_SPACE_ID: - if (addr != (os_vm_address_t)IMMOBILE_SPACE_START) { - fprintf(stderr, "in core: %p - in runtime: %p\n", - (void*)addr, (void*)IMMOBILE_SPACE_START); - lose("core/runtime address mismatch: IMMOBILE_SPACE_START\n"); - } - immobile_space_coreparse(IMMOBILE_SPACE_START, len); - break; - case IMMOBILE_VARYOBJ_CORE_SPACE_ID: - if (addr != (os_vm_address_t)IMMOBILE_VARYOBJ_SUBSPACE_START) { - fprintf(stderr, "in core: %p - in runtime: %p\n", - (void*)addr, (void*)IMMOBILE_VARYOBJ_SUBSPACE_START); - lose("core/runtime address mismatch: IMMOBILE_VARYOBJ_SUBSPACE_START\n"); - } - immobile_space_coreparse(IMMOBILE_VARYOBJ_SUBSPACE_START, len); - break; + /* Delayed until after dynamic space has been mapped so that writes + * to immobile space due to core relocation don't fault. */ + write_protect_immobile_space(); #endif - default: - lose("unknown space ID %ld addr %p\n", id, addr); - } - } } lispobj @@ -838,6 +810,10 @@ load_core_file(char *file, os_vm_offset_t file_offset) #ifdef LISP_FEATURE_GENCGC case PAGE_TABLE_CORE_ENTRY_TYPE_CODE: { + extern void gc_allocate_ptes(); + // Allocation of PTEs is delayed 'til now so that calloc() doesn't + // consume addresses that would have been taken by a mapped space. + gc_allocate_ptes(); os_vm_size_t remaining = *ptr; os_vm_size_t fdoffset = (*(ptr+1) + 1) * (os_vm_page_size); page_index_t page = 0, npages; diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c index f8b23b721..bd750f65d 100644 --- a/src/runtime/gencgc.c +++ b/src/runtime/gencgc.c @@ -2415,7 +2415,6 @@ static struct new_area new_areas_2[NUM_NEW_AREAS]; #ifdef LISP_FEATURE_IMMOBILE_SPACE extern unsigned int immobile_scav_queue_count; extern void - gc_init_immobile(), update_immobile_nursery_bits(), scavenge_immobile_roots(generation_index_t,generation_index_t), scavenge_immobile_newspace(), @@ -3785,14 +3784,24 @@ collect_garbage(generation_index_t last_gen) SHOW("returning from collect_garbage"); } +/* Initialization of gencgc metadata is split into three steps: + * 1. gc_init() - allocation of a fixed-address space via mmap(), + * failing which there's no reason to go on. (safepoint only) + * 2. gc_allocate_ptes() - page table entries + * 3. gencgc_pickup_dynamic() - calculation of scan start offsets + * Steps (2) and (3) are combined in self-build because there is + * no PAGE_TABLE_CORE_ENTRY_TYPE_CODE core entry. */ void gc_init(void) { - page_index_t i; - #if defined(LISP_FEATURE_SB_SAFEPOINT) alloc_gc_page(); #endif +} + +void gc_allocate_ptes() +{ + page_index_t i; /* Compute the number of pages needed for the dynamic space. * Dynamic space size should be aligned on page size. */ @@ -3811,9 +3820,6 @@ gc_init(void) * unnecessary and did hurt startup time. */ page_table = calloc(page_table_pages, sizeof(struct page)); gc_assert(page_table); -#ifdef LISP_FEATURE_IMMOBILE_SPACE - gc_init_immobile(); -#endif hopscotch_init(); #ifdef PIN_GRANULARITY_LISPOBJ @@ -3949,6 +3955,9 @@ gencgc_pickup_dynamic(void) void gc_initialize_pointers(void) { + /* !page_table_pages happens once only in self-build and not again */ + if (!page_table_pages) + gc_allocate_ptes(); gencgc_pickup_dynamic(); } diff --git a/src/runtime/marknsweepgc.c b/src/runtime/marknsweepgc.c index 6e9bf3dbd..0cd979720 100644 --- a/src/runtime/marknsweepgc.c +++ b/src/runtime/marknsweepgc.c @@ -1180,7 +1180,7 @@ sweep_immobile_space(int raise) compute_immobile_space_bound(); } -void gc_init_immobile() +static void gc_init_immobile() { #ifdef DEBUG logfile = stderr; @@ -1219,92 +1219,93 @@ static inline int immobile_obj_spacing(lispobj header_word, lispobj *obj, } // Set the characteristics of each used page at image startup time. -void immobile_space_coreparse(uword_t address, uword_t len) +void immobile_space_coreparse(uword_t fixedobj_len, uword_t varyobj_len) { int n_pages, word_idx, page; - - n_pages = (len + IMMOBILE_CARD_BYTES - 1) / IMMOBILE_CARD_BYTES; - if (address == IMMOBILE_SPACE_START) { - for (page = 0 ; page < n_pages ; ++page) { - lispobj* page_data = low_page_address(page); - for (word_idx = 0 ; word_idx < WORDS_PER_PAGE ; ++word_idx) { - lispobj* obj = page_data + word_idx; - lispobj header = *obj; - if (!fixnump(header)) { - gc_assert(other_immediate_lowtag_p(*obj)); - int size = sizetab[widetag_of(header)](obj); - fixedobj_pages[page].attr.parts.obj_size = size; - fixedobj_pages[page].attr.parts.obj_align - = immobile_obj_spacing(header, obj, size); - fixedobj_pages[page].gens |= 1 << __immobile_obj_gen_bits(obj); - if (ENABLE_PAGE_PROTECTION) - fixedobj_pages[page].attr.parts.flags = WRITE_PROTECT; - break; - } - } - } - } else if (address == IMMOBILE_VARYOBJ_SUBSPACE_START) { - lispobj* obj = (lispobj*)address; - lispobj* space_limit = (lispobj*)(address + len); - int n_words; - low_page_index_t last_page = 0; - // IMMOBILE_SPACE_FREE_POINTER should be current - // since static space is mmapped already. - lispobj* limit = (lispobj*)SYMBOL(IMMOBILE_SPACE_FREE_POINTER)->value; - gc_assert(limit != 0 /* would be zero if not mmapped yet */ - && limit < space_limit); - for ( ; obj < limit ; obj += n_words ) { - gc_assert(other_immediate_lowtag_p(obj[0])); - n_words = sizetab[widetag_of(*obj)](obj); - if (immobile_filler_p(obj)) { - // Holes were chained through the debug_info slot at save. - // Just update the head of the chain. - varyobj_holes = (lispobj)obj; - continue; - } - low_page_index_t first_page = find_immobile_page_index(obj); - last_page = find_immobile_page_index(obj+n_words-1); - // Only the page with this object header gets a bit in its gen mask. - VARYOBJ_PAGE_GENS(first_page) |= 1<<__immobile_obj_gen_bits(obj); - // For each page touched by this object, set the page's - // scan_start_offset, unless it was already set. - int page; - for (page = first_page ; page <= last_page ; ++page) { - if (!varyobj_page_scan_start_offset[page - FIRST_VARYOBJ_PAGE]) { - long offset = (char*)low_page_address(page+1) - (char*)obj; - varyobj_page_scan_start_offset[page - FIRST_VARYOBJ_PAGE] - = offset >> (WORD_SHIFT + 1); - } + uword_t address; + + gc_init_immobile(); + + address = IMMOBILE_SPACE_START; + n_pages = (fixedobj_len + IMMOBILE_CARD_BYTES - 1) / IMMOBILE_CARD_BYTES; + for (page = 0 ; page < n_pages ; ++page) { + lispobj* page_data = low_page_address(page); + for (word_idx = 0 ; word_idx < WORDS_PER_PAGE ; ++word_idx) { + lispobj* obj = page_data + word_idx; + lispobj header = *obj; + if (!fixnump(header)) { + gc_assert(other_immediate_lowtag_p(*obj)); + int size = sizetab[widetag_of(header)](obj); + fixedobj_pages[page].attr.parts.obj_size = size; + fixedobj_pages[page].attr.parts.obj_align + = immobile_obj_spacing(header, obj, size); + fixedobj_pages[page].gens |= 1 << __immobile_obj_gen_bits(obj); + if (ENABLE_PAGE_PROTECTION) + fixedobj_pages[page].attr.parts.flags = WRITE_PROTECT; + break; } } - // Write a padding object if necessary - if ((uword_t)limit & (IMMOBILE_CARD_BYTES-1)) { - int remainder = IMMOBILE_CARD_BYTES - - ((uword_t)limit & (IMMOBILE_CARD_BYTES-1)); - limit[0] = SIMPLE_ARRAY_FIXNUM_WIDETAG; - limit[1] = make_fixnum((remainder >> WORD_SHIFT) - 2); - int size = sizetab[SIMPLE_ARRAY_FIXNUM_WIDETAG](limit); - lispobj* padded_end = limit + size; - gc_assert(!((uword_t)padded_end & (IMMOBILE_CARD_BYTES-1))); - } - // Write-protect the pages occupied by the core file. - // (There can be no inter-generation pointers.) - if (ENABLE_PAGE_PROTECTION) { - int page; - for (page = FIRST_VARYOBJ_PAGE ; page <= last_page ; ++page) - varyobj_page_touched_bits[(page-FIRST_VARYOBJ_PAGE)/32] &= ~(1<<(page & 31)); + } + address = IMMOBILE_VARYOBJ_SUBSPACE_START; + n_pages = (varyobj_len + IMMOBILE_CARD_BYTES - 1) / IMMOBILE_CARD_BYTES; + lispobj* obj = (lispobj*)address; + lispobj* space_limit = (lispobj*)(address + varyobj_len); + int n_words; + low_page_index_t last_page = 0; + // IMMOBILE_SPACE_FREE_POINTER should be current + // since static space is mmapped already. + lispobj* limit = (lispobj*)SYMBOL(IMMOBILE_SPACE_FREE_POINTER)->value; + gc_assert(limit != 0 /* would be zero if not mmapped yet */ + && limit < space_limit); + for ( ; obj < limit ; obj += n_words ) { + gc_assert(other_immediate_lowtag_p(obj[0])); + n_words = sizetab[widetag_of(*obj)](obj); + if (immobile_filler_p(obj)) { + // Holes were chained through the debug_info slot at save. + // Just update the head of the chain. + varyobj_holes = (lispobj)obj; + continue; } - compute_immobile_space_bound(); - struct code* code = (struct code*)address; - while (!code_n_funs(code)) { - gc_assert(widetag_of(code->header) == CODE_HEADER_WIDETAG); - code = (struct code*) - (lispobj*)code + sizetab[CODE_HEADER_WIDETAG]((lispobj*)code); + low_page_index_t first_page = find_immobile_page_index(obj); + last_page = find_immobile_page_index(obj+n_words-1); + // Only the page with this object header gets a bit in its gen mask. + VARYOBJ_PAGE_GENS(first_page) |= 1<<__immobile_obj_gen_bits(obj); + // For each page touched by this object, set the page's + // scan_start_offset, unless it was already set. + int page; + for (page = first_page ; page <= last_page ; ++page) { + if (!varyobj_page_scan_start_offset[page - FIRST_VARYOBJ_PAGE]) { + long offset = (char*)low_page_address(page+1) - (char*)obj; + varyobj_page_scan_start_offset[page - FIRST_VARYOBJ_PAGE] + = offset >> (WORD_SHIFT + 1); + } } - ASM_ROUTINES_END = (unsigned)(uword_t)code; - } else { - lose("unknown immobile subspace"); } + // Write a padding object if necessary + if ((uword_t)limit & (IMMOBILE_CARD_BYTES-1)) { + int remainder = IMMOBILE_CARD_BYTES - + ((uword_t)limit & (IMMOBILE_CARD_BYTES-1)); + limit[0] = SIMPLE_ARRAY_FIXNUM_WIDETAG; + limit[1] = make_fixnum((remainder >> WORD_SHIFT) - 2); + int size = sizetab[SIMPLE_ARRAY_FIXNUM_WIDETAG](limit); + lispobj* padded_end = limit + size; + gc_assert(!((uword_t)padded_end & (IMMOBILE_CARD_BYTES-1))); + } + // Write-protect the pages occupied by the core file. + // (There can be no inter-generation pointers.) + if (ENABLE_PAGE_PROTECTION) { + int page; + for (page = FIRST_VARYOBJ_PAGE ; page <= last_page ; ++page) + varyobj_page_touched_bits[(page-FIRST_VARYOBJ_PAGE)/32] &= ~(1<<(page & 31)); + } + compute_immobile_space_bound(); + struct code* code = (struct code*)address; + while (!code_n_funs(code)) { + gc_assert(widetag_of(code->header) == CODE_HEADER_WIDETAG); + code = (struct code*) + (lispobj*)code + sizetab[CODE_HEADER_WIDETAG]((lispobj*)code); + } + ASM_ROUTINES_END = (unsigned)(uword_t)code; } // Demote pseudo-static to highest normal generation -- 2.11.4.GIT