| blob | 591d06a7f59fcc453ceb2cc53c674e9678d853ad |
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // See malloc.h for overview.
6 //
7 // TODO(rsc): double-check stats.
9 package runtime
10 #include <stddef.h>
11 #include <errno.h>
12 #include <stdlib.h>
13 #include "go-alloc.h"
14 #include "runtime.h"
15 #include "arch.h"
16 #include "malloc.h"
17 #include "interface.h"
18 #include "go-type.h"
20 // Map gccgo field names to gc field names.
21 // Eface aka __go_empty_interface.
22 #define type __type_descriptor
23 // Type aka __go_type_descriptor
24 #define kind __code
25 #define string __reflection
27 // GCCGO SPECIFIC CHANGE
28 //
29 // There is a long comment in runtime_mallocinit about where to put the heap
30 // on a 64-bit system. It makes assumptions that are not valid on linux/arm64
31 // -- it assumes user space can choose the lower 47 bits of a pointer, but on
32 // linux/arm64 we can only choose the lower 39 bits. This means the heap is
33 // roughly a quarter of the available address space and we cannot choose a bit
34 // pattern that all pointers will have -- luckily the GC is mostly precise
35 // these days so this doesn't matter all that much. The kernel (as of 3.13)
36 // will allocate address space starting either down from 0x7fffffffff or up
37 // from 0x2000000000, so we put the heap roughly in the middle of these two
38 // addresses to minimize the chance that a non-heap allocation will get in the
39 // way of the heap.
40 //
41 // This all means that there isn't much point in trying 256 different
42 // locations for the heap on such systems.
43 #ifdef __aarch64__
44 #define HeapBase(i) ((void*)(uintptr)(0x40ULL<<32))
45 #define HeapBaseOptions 1
46 #else
47 #define HeapBase(i) ((void*)(uintptr)(i<<40|0x00c0ULL<<32))
48 #define HeapBaseOptions 0x80
49 #endif
50 // END GCCGO SPECIFIC CHANGE
52 // Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K.
53 MHeap runtime_mheap;
54 MStats mstats;
56 int32 runtime_checking;
58 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go
60 extern volatile intgo runtime_MemProfileRate
61 __asm__ (GOSYM_PREFIX "runtime.MemProfileRate");
63 static MSpan* largealloc(uint32, uintptr*);
64 static void runtime_profilealloc(void *v, uintptr size);
65 static void settype(MSpan *s, void *v, uintptr typ);
67 // Allocate an object of at least size bytes.
68 // Small objects are allocated from the per-thread cache's free lists.
69 // Large objects (> 32 kB) are allocated straight from the heap.
70 // If the block will be freed with runtime_free(), typ must be 0.
71 void*
72 runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
73 {
74 M *m;
75 G *g;
76 int32 sizeclass;
77 uintptr tinysize, size1;
78 intgo rate;
79 MCache *c;
80 MSpan *s;
81 MLink *v, *next;
82 byte *tiny;
83 bool incallback;
85 if(size == 0) {
86 // All 0-length allocations use this pointer.
87 // The language does not require the allocations to
88 // have distinct values.
89 return &runtime_zerobase;
90 }
92 g = runtime_g();
93 m = g->m;
95 incallback = false;
96 if(m->mcache == nil && m->ncgo > 0) {
97 // For gccgo this case can occur when a cgo or SWIG function
98 // has an interface return type and the function
99 // returns a non-pointer, so memory allocation occurs
100 // after syscall.Cgocall but before syscall.CgocallDone.
101 // We treat it as a callback.
102 runtime_exitsyscall();
103 m = runtime_m();
104 incallback = true;
105 flag |= FlagNoInvokeGC;
106 }
108 if(runtime_gcwaiting() && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC)) {
109 runtime_gosched();
110 m = runtime_m();
111 }
112 if(m->mallocing)
113 runtime_throw("malloc/free - deadlock");
114 // Disable preemption during settype.
115 // We can not use m->mallocing for this, because settype calls mallocgc.
116 m->locks++;
117 m->mallocing = 1;
119 if(DebugTypeAtBlockEnd)
120 size += sizeof(uintptr);
122 c = m->mcache;
123 if(!runtime_debug.efence && size <= MaxSmallSize) {
124 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize) {
125 // Tiny allocator.
126 //
127 // Tiny allocator combines several tiny allocation requests
128 // into a single memory block. The resulting memory block
129 // is freed when all subobjects are unreachable. The subobjects
130 // must be FlagNoScan (don't have pointers), this ensures that
131 // the amount of potentially wasted memory is bounded.
132 //
133 // Size of the memory block used for combining (TinySize) is tunable.
134 // Current setting is 16 bytes, which relates to 2x worst case memory
135 // wastage (when all but one subobjects are unreachable).
136 // 8 bytes would result in no wastage at all, but provides less
137 // opportunities for combining.
138 // 32 bytes provides more opportunities for combining,
139 // but can lead to 4x worst case wastage.
140 // The best case winning is 8x regardless of block size.
141 //
142 // Objects obtained from tiny allocator must not be freed explicitly.
143 // So when an object will be freed explicitly, we ensure that
144 // its size >= TinySize.
145 //
146 // SetFinalizer has a special case for objects potentially coming
147 // from tiny allocator, it such case it allows to set finalizers
148 // for an inner byte of a memory block.
149 //
150 // The main targets of tiny allocator are small strings and
151 // standalone escaping variables. On a json benchmark
152 // the allocator reduces number of allocations by ~12% and
153 // reduces heap size by ~20%.
155 tinysize = c->tinysize;
156 if(size <= tinysize) {
157 tiny = c->tiny;
158 // Align tiny pointer for required (conservative) alignment.
159 if((size&7) == 0)
160 tiny = (byte*)ROUND((uintptr)tiny, 8);
161 else if((size&3) == 0)
162 tiny = (byte*)ROUND((uintptr)tiny, 4);
163 else if((size&1) == 0)
164 tiny = (byte*)ROUND((uintptr)tiny, 2);
165 size1 = size + (tiny - (byte*)c->tiny);
166 if(size1 <= tinysize) {
167 // The object fits into existing tiny block.
168 v = (MLink*)tiny;
169 c->tiny = (byte*)c->tiny + size1;
170 c->tinysize -= size1;
171 m->mallocing = 0;
172 m->locks--;
173 if(incallback)
174 runtime_entersyscall();
175 return v;
176 }
177 }
178 // Allocate a new TinySize block.
179 s = c->alloc[TinySizeClass];
180 if(s->freelist == nil)
181 s = runtime_MCache_Refill(c, TinySizeClass);
182 v = s->freelist;
183 next = v->next;
184 s->freelist = next;
185 s->ref++;
186 if(next != nil) // prefetching nil leads to a DTLB miss
187 PREFETCH(next);
188 ((uint64*)v)[0] = 0;
189 ((uint64*)v)[1] = 0;
190 // See if we need to replace the existing tiny block with the new one
191 // based on amount of remaining free space.
192 if(TinySize-size > tinysize) {
193 c->tiny = (byte*)v + size;
194 c->tinysize = TinySize - size;
195 }
196 size = TinySize;
197 goto done;
198 }
199 // Allocate from mcache free lists.
200 // Inlined version of SizeToClass().
201 if(size <= 1024-8)
202 sizeclass = runtime_size_to_class8[(size+7)>>3];
203 else
204 sizeclass = runtime_size_to_class128[(size-1024+127) >> 7];
205 size = runtime_class_to_size[sizeclass];
206 s = c->alloc[sizeclass];
207 if(s->freelist == nil)
208 s = runtime_MCache_Refill(c, sizeclass);
209 v = s->freelist;
210 next = v->next;
211 s->freelist = next;
212 s->ref++;
213 if(next != nil) // prefetching nil leads to a DTLB miss
214 PREFETCH(next);
215 if(!(flag & FlagNoZero)) {
216 v->next = nil;
217 // block is zeroed iff second word is zero ...
218 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0)
219 runtime_memclr((byte*)v, size);
220 }
221 done:
222 c->local_cachealloc += size;
223 } else {
224 // Allocate directly from heap.
225 s = largealloc(flag, &size);
226 v = (void*)(s->start << PageShift);
227 }
229 if(flag & FlagNoGC)
230 runtime_marknogc(v);
231 else if(!(flag & FlagNoScan))
232 runtime_markscan(v);
234 if(DebugTypeAtBlockEnd)
235 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ;
237 m->mallocing = 0;
238 // TODO: save type even if FlagNoScan? Potentially expensive but might help
239 // heap profiling/tracing.
240 if(UseSpanType && !(flag & FlagNoScan) && typ != 0)
241 settype(s, v, typ);
243 if(runtime_debug.allocfreetrace)
244 runtime_tracealloc(v, size, typ);
246 if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
247 if(size < (uintptr)rate && size < (uintptr)(uint32)c->next_sample)
248 c->next_sample -= size;
249 else
250 runtime_profilealloc(v, size);
251 }
253 m->locks--;
255 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc)
256 runtime_gc(0);
258 if(incallback)
259 runtime_entersyscall();
261 return v;
262 }
264 static MSpan*
265 largealloc(uint32 flag, uintptr *sizep)
266 {
267 uintptr npages, size;
268 MSpan *s;
269 void *v;
271 // Allocate directly from heap.
272 size = *sizep;
273 if(size + PageSize < size)
274 runtime_throw("out of memory");
275 npages = size >> PageShift;
276 if((size & PageMask) != 0)
277 npages++;
278 s = runtime_MHeap_Alloc(&runtime_mheap, npages, 0, 1, !(flag & FlagNoZero));
279 if(s == nil)
280 runtime_throw("out of memory");
281 s->limit = (uintptr)((byte*)(s->start<<PageShift) + size);
282 *sizep = npages<<PageShift;
283 v = (void*)(s->start << PageShift);
284 // setup for mark sweep
285 runtime_markspan(v, 0, 0, true);
286 return s;
287 }
289 static void
290 runtime_profilealloc(void *v, uintptr size)
291 {
292 uintptr rate;
293 int32 next;
294 MCache *c;
296 c = runtime_m()->mcache;
297 rate = runtime_MemProfileRate;
298 if(size < rate) {
299 // pick next profile time
300 // If you change this, also change allocmcache.
301 if(rate > 0x3fffffff) // make 2*rate not overflow
302 rate = 0x3fffffff;
303 next = runtime_fastrand1() % (2*rate);
304 // Subtract the "remainder" of the current allocation.
305 // Otherwise objects that are close in size to sampling rate
306 // will be under-sampled, because we consistently discard this remainder.
307 next -= (size - c->next_sample);
308 if(next < 0)
309 next = 0;
310 c->next_sample = next;
311 }
312 runtime_MProf_Malloc(v, size);
313 }
315 void*
316 __go_alloc(uintptr size)
317 {
318 return runtime_mallocgc(size, 0, FlagNoInvokeGC);
319 }
321 // Free the object whose base pointer is v.
322 void
323 __go_free(void *v)
324 {
325 M *m;
326 int32 sizeclass;
327 MSpan *s;
328 MCache *c;
329 uintptr size;
331 if(v == nil)
332 return;
334 // If you change this also change mgc0.c:/^sweep,
335 // which has a copy of the guts of free.
337 m = runtime_m();
338 if(m->mallocing)
339 runtime_throw("malloc/free - deadlock");
340 m->mallocing = 1;
342 if(!runtime_mlookup(v, nil, nil, &s)) {
343 runtime_printf("free %p: not an allocated block\n", v);
344 runtime_throw("free runtime_mlookup");
345 }
346 size = s->elemsize;
347 sizeclass = s->sizeclass;
348 // Objects that are smaller than TinySize can be allocated using tiny alloc,
349 // if then such object is combined with an object with finalizer, we will crash.
350 if(size < TinySize)
351 runtime_throw("freeing too small block");
353 if(runtime_debug.allocfreetrace)
354 runtime_tracefree(v, size);
356 // Ensure that the span is swept.
357 // If we free into an unswept span, we will corrupt GC bitmaps.
358 runtime_MSpan_EnsureSwept(s);
360 if(s->specials != nil)
361 runtime_freeallspecials(s, v, size);
363 c = m->mcache;
364 if(sizeclass == 0) {
365 // Large object.
366 s->needzero = 1;
367 // Must mark v freed before calling unmarkspan and MHeap_Free:
368 // they might coalesce v into other spans and change the bitmap further.
369 runtime_markfreed(v);
370 runtime_unmarkspan(v, 1<<PageShift);
371 // NOTE(rsc,dvyukov): The original implementation of efence
372 // in CL 22060046 used SysFree instead of SysFault, so that
373 // the operating system would eventually give the memory
374 // back to us again, so that an efence program could run
375 // longer without running out of memory. Unfortunately,
376 // calling SysFree here without any kind of adjustment of the
377 // heap data structures means that when the memory does
378 // come back to us, we have the wrong metadata for it, either in
379 // the MSpan structures or in the garbage collection bitmap.
380 // Using SysFault here means that the program will run out of
381 // memory fairly quickly in efence mode, but at least it won't
382 // have mysterious crashes due to confused memory reuse.
383 // It should be possible to switch back to SysFree if we also
384 // implement and then call some kind of MHeap_DeleteSpan.
385 if(runtime_debug.efence)
386 runtime_SysFault((void*)(s->start<<PageShift), size);
387 else
388 runtime_MHeap_Free(&runtime_mheap, s, 1);
389 c->local_nlargefree++;
390 c->local_largefree += size;
391 } else {
392 // Small object.
393 if(size > 2*sizeof(uintptr))
394 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed"
395 else if(size > sizeof(uintptr))
396 ((uintptr*)v)[1] = 0;
397 // Must mark v freed before calling MCache_Free:
398 // it might coalesce v and other blocks into a bigger span
399 // and change the bitmap further.
400 c->local_nsmallfree[sizeclass]++;
401 c->local_cachealloc -= size;
402 if(c->alloc[sizeclass] == s) {
403 // We own the span, so we can just add v to the freelist
404 runtime_markfreed(v);
405 ((MLink*)v)->next = s->freelist;
406 s->freelist = v;
407 s->ref--;
408 } else {
409 // Someone else owns this span. Add to free queue.
410 runtime_MCache_Free(c, v, sizeclass, size);
411 }
412 }
413 m->mallocing = 0;
414 }
416 int32
417 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
418 {
419 M *m;
420 uintptr n, i;
421 byte *p;
422 MSpan *s;
424 m = runtime_m();
426 m->mcache->local_nlookup++;
427 if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) {
428 // purge cache stats to prevent overflow
429 runtime_lock(&runtime_mheap);
430 runtime_purgecachedstats(m->mcache);
431 runtime_unlock(&runtime_mheap);
432 }
434 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
435 if(sp)
436 *sp = s;
437 if(s == nil) {
438 runtime_checkfreed(v, 1);
439 if(base)
440 *base = nil;
441 if(size)
442 *size = 0;
443 return 0;
444 }
446 p = (byte*)((uintptr)s->start<<PageShift);
447 if(s->sizeclass == 0) {
448 // Large object.
449 if(base)
450 *base = p;
451 if(size)
452 *size = s->npages<<PageShift;
453 return 1;
454 }
456 n = s->elemsize;
457 if(base) {
458 i = ((byte*)v - p)/n;
459 *base = p + i*n;
460 }
461 if(size)
462 *size = n;
464 return 1;
465 }
467 void
468 runtime_purgecachedstats(MCache *c)
469 {
470 MHeap *h;
471 int32 i;
473 // Protected by either heap or GC lock.
474 h = &runtime_mheap;
475 mstats.heap_alloc += (intptr)c->local_cachealloc;
476 c->local_cachealloc = 0;
477 mstats.nlookup += c->local_nlookup;
478 c->local_nlookup = 0;
479 h->largefree += c->local_largefree;
480 c->local_largefree = 0;
481 h->nlargefree += c->local_nlargefree;
482 c->local_nlargefree = 0;
483 for(i=0; i<(int32)nelem(c->local_nsmallfree); i++) {
484 h->nsmallfree[i] += c->local_nsmallfree[i];
485 c->local_nsmallfree[i] = 0;
486 }
487 }
489 extern uintptr runtime_sizeof_C_MStats
490 __asm__ (GOSYM_PREFIX "runtime.Sizeof_C_MStats");
492 // Size of the trailing by_size array differs between Go and C,
493 // _NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
494 // sizeof_C_MStats is what C thinks about size of Go struct.
496 // Initialized in mallocinit because it's defined in go/runtime/mem.go.
498 #define MaxArena32 (2U<<30)
500 void
501 runtime_mallocinit(void)
502 {
503 byte *p, *p1;
504 uintptr arena_size, bitmap_size, spans_size, p_size;
505 uintptr *pend;
506 uintptr end;
507 uintptr limit;
508 uint64 i;
509 bool reserved;
511 runtime_sizeof_C_MStats = sizeof(MStats) - (_NumSizeClasses - 61) * sizeof(mstats.by_size[0]);
513 p = nil;
514 p_size = 0;
515 arena_size = 0;
516 bitmap_size = 0;
517 spans_size = 0;
518 reserved = false;
520 // for 64-bit build
521 USED(p);
522 USED(p_size);
523 USED(arena_size);
524 USED(bitmap_size);
525 USED(spans_size);
527 runtime_InitSizes();
529 if(runtime_class_to_size[TinySizeClass] != TinySize)
530 runtime_throw("bad TinySizeClass");
532 // limit = runtime_memlimit();
533 // See https://code.google.com/p/go/issues/detail?id=5049
534 // TODO(rsc): Fix after 1.1.
535 limit = 0;
537 // Set up the allocation arena, a contiguous area of memory where
538 // allocated data will be found. The arena begins with a bitmap large
539 // enough to hold 4 bits per allocated word.
540 if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) {
541 // On a 64-bit machine, allocate from a single contiguous reservation.
542 // 128 GB (MaxMem) should be big enough for now.
543 //
544 // The code will work with the reservation at any address, but ask
545 // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f).
546 // Allocating a 128 GB region takes away 37 bits, and the amd64
547 // doesn't let us choose the top 17 bits, so that leaves the 11 bits
548 // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means
549 // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df.
550 // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid
551 // UTF-8 sequences, and they are otherwise as far away from
552 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0
553 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors
554 // on OS X during thread allocations. 0x00c0 causes conflicts with
555 // AddressSanitizer which reserves all memory up to 0x0100.
556 // These choices are both for debuggability and to reduce the
557 // odds of the conservative garbage collector not collecting memory
558 // because some non-pointer block of memory had a bit pattern
559 // that matched a memory address.
560 //
561 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB)
562 // but it hardly matters: e0 00 is not valid UTF-8 either.
563 //
564 // If this fails we fall back to the 32 bit memory mechanism
565 arena_size = MaxMem;
566 bitmap_size = arena_size / (sizeof(void*)*8/4);
567 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]);
568 spans_size = ROUND(spans_size, PageSize);
569 for(i = 0; i < HeapBaseOptions; i++) {
570 p = HeapBase(i);
571 p_size = bitmap_size + spans_size + arena_size + PageSize;
572 p = runtime_SysReserve(p, p_size, &reserved);
573 if(p != nil)
574 break;
575 }
576 }
577 if (p == nil) {
578 // On a 32-bit machine, we can't typically get away
579 // with a giant virtual address space reservation.
580 // Instead we map the memory information bitmap
581 // immediately after the data segment, large enough
582 // to handle another 2GB of mappings (256 MB),
583 // along with a reservation for another 512 MB of memory.
584 // When that gets used up, we'll start asking the kernel
585 // for any memory anywhere and hope it's in the 2GB
586 // following the bitmap (presumably the executable begins
587 // near the bottom of memory, so we'll have to use up
588 // most of memory before the kernel resorts to giving out
589 // memory before the beginning of the text segment).
590 //
591 // Alternatively we could reserve 512 MB bitmap, enough
592 // for 4GB of mappings, and then accept any memory the
593 // kernel threw at us, but normally that's a waste of 512 MB
594 // of address space, which is probably too much in a 32-bit world.
595 bitmap_size = MaxArena32 / (sizeof(void*)*8/4);
596 arena_size = 512<<20;
597 spans_size = MaxArena32 / PageSize * sizeof(runtime_mheap.spans[0]);
598 if(limit > 0 && arena_size+bitmap_size+spans_size > limit) {
599 bitmap_size = (limit / 9) & ~((1<<PageShift) - 1);
600 arena_size = bitmap_size * 8;
601 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]);
602 }
603 spans_size = ROUND(spans_size, PageSize);
605 // SysReserve treats the address we ask for, end, as a hint,
606 // not as an absolute requirement. If we ask for the end
607 // of the data segment but the operating system requires
608 // a little more space before we can start allocating, it will
609 // give out a slightly higher pointer. Except QEMU, which
610 // is buggy, as usual: it won't adjust the pointer upward.
611 // So adjust it upward a little bit ourselves: 1/4 MB to get
612 // away from the running binary image and then round up
613 // to a MB boundary.
615 end = 0;
616 pend = &__go_end;
617 if(pend != nil)
618 end = *pend;
619 p = (byte*)ROUND(end + (1<<18), 1<<20);
620 p_size = bitmap_size + spans_size + arena_size + PageSize;
621 p = runtime_SysReserve(p, p_size, &reserved);
622 if(p == nil)
623 runtime_throw("runtime: cannot reserve arena virtual address space");
624 }
626 // PageSize can be larger than OS definition of page size,
627 // so SysReserve can give us a PageSize-unaligned pointer.
628 // To overcome this we ask for PageSize more and round up the pointer.
629 p1 = (byte*)ROUND((uintptr)p, PageSize);
631 runtime_mheap.spans = (MSpan**)p1;
632 runtime_mheap.bitmap = p1 + spans_size;
633 runtime_mheap.arena_start = p1 + spans_size + bitmap_size;
634 runtime_mheap.arena_used = runtime_mheap.arena_start;
635 runtime_mheap.arena_end = p + p_size;
636 runtime_mheap.arena_reserved = reserved;
638 if(((uintptr)runtime_mheap.arena_start & (PageSize-1)) != 0)
639 runtime_throw("misrounded allocation in mallocinit");
641 // Initialize the rest of the allocator.
642 runtime_MHeap_Init(&runtime_mheap);
643 runtime_m()->mcache = runtime_allocmcache();
645 // See if it works.
646 runtime_free(runtime_malloc(TinySize));
647 }
649 void*
650 runtime_MHeap_SysAlloc(MHeap *h, uintptr n)
651 {
652 byte *p, *p_end;
653 uintptr p_size;
654 bool reserved;
657 if(n > (uintptr)(h->arena_end - h->arena_used)) {
658 // We are in 32-bit mode, maybe we didn't use all possible address space yet.
659 // Reserve some more space.
660 byte *new_end;
662 p_size = ROUND(n + PageSize, 256<<20);
663 new_end = h->arena_end + p_size;
664 if(new_end <= h->arena_start + MaxArena32) {
665 // TODO: It would be bad if part of the arena
666 // is reserved and part is not.
667 p = runtime_SysReserve(h->arena_end, p_size, &reserved);
668 if(p == h->arena_end) {
669 h->arena_end = new_end;
670 h->arena_reserved = reserved;
671 }
672 else if(p+p_size <= h->arena_start + MaxArena32) {
673 // Keep everything page-aligned.
674 // Our pages are bigger than hardware pages.
675 h->arena_end = p+p_size;
676 h->arena_used = p + (-(uintptr)p&(PageSize-1));
677 h->arena_reserved = reserved;
678 } else {
679 uint64 stat;
680 stat = 0;
681 runtime_SysFree(p, p_size, &stat);
682 }
683 }
684 }
685 if(n <= (uintptr)(h->arena_end - h->arena_used)) {
686 // Keep taking from our reservation.
687 p = h->arena_used;
688 runtime_SysMap(p, n, h->arena_reserved, &mstats.heap_sys);
689 h->arena_used += n;
690 runtime_MHeap_MapBits(h);
691 runtime_MHeap_MapSpans(h);
693 if(((uintptr)p & (PageSize-1)) != 0)
694 runtime_throw("misrounded allocation in MHeap_SysAlloc");
695 return p;
696 }
698 // If using 64-bit, our reservation is all we have.
699 if((uintptr)(h->arena_end - h->arena_start) >= MaxArena32)
700 return nil;
702 // On 32-bit, once the reservation is gone we can
703 // try to get memory at a location chosen by the OS
704 // and hope that it is in the range we allocated bitmap for.
705 p_size = ROUND(n, PageSize) + PageSize;
706 p = runtime_SysAlloc(p_size, &mstats.heap_sys);
707 if(p == nil)
708 return nil;
710 if(p < h->arena_start || (uintptr)(p+p_size - h->arena_start) >= MaxArena32) {
711 runtime_printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n",
712 p, h->arena_start, h->arena_start+MaxArena32);
713 runtime_SysFree(p, p_size, &mstats.heap_sys);
714 return nil;
715 }
717 p_end = p + p_size;
718 p += -(uintptr)p & (PageSize-1);
719 if(p+n > h->arena_used) {
720 h->arena_used = p+n;
721 if(p_end > h->arena_end)
722 h->arena_end = p_end;
723 runtime_MHeap_MapBits(h);
724 runtime_MHeap_MapSpans(h);
725 }
727 if(((uintptr)p & (PageSize-1)) != 0)
728 runtime_throw("misrounded allocation in MHeap_SysAlloc");
729 return p;
730 }
732 static struct
733 {
734 Lock;
735 byte* pos;
736 byte* end;
737 } persistent;
739 enum
740 {
741 PersistentAllocChunk = 256<<10,
742 PersistentAllocMaxBlock = 64<<10, // VM reservation granularity is 64K on windows
743 };
745 // Wrapper around SysAlloc that can allocate small chunks.
746 // There is no associated free operation.
747 // Intended for things like function/type/debug-related persistent data.
748 // If align is 0, uses default align (currently 8).
749 void*
750 runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat)
751 {
752 byte *p;
754 if(align != 0) {
755 if(align&(align-1))
756 runtime_throw("persistentalloc: align is not a power of 2");
757 if(align > PageSize)
758 runtime_throw("persistentalloc: align is too large");
759 } else
760 align = 8;
761 if(size >= PersistentAllocMaxBlock)
762 return runtime_SysAlloc(size, stat);
763 runtime_lock(&persistent);
764 persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align);
765 if(persistent.pos + size > persistent.end) {
766 persistent.pos = runtime_SysAlloc(PersistentAllocChunk, &mstats.other_sys);
767 if(persistent.pos == nil) {
768 runtime_unlock(&persistent);
769 runtime_throw("runtime: cannot allocate memory");
770 }
771 persistent.end = persistent.pos + PersistentAllocChunk;
772 }
773 p = persistent.pos;
774 persistent.pos += size;
775 runtime_unlock(&persistent);
776 if(stat != &mstats.other_sys) {
777 // reaccount the allocation against provided stat
778 runtime_xadd64(stat, size);
779 runtime_xadd64(&mstats.other_sys, -(uint64)size);
780 }
781 return p;
782 }
784 static void
785 settype(MSpan *s, void *v, uintptr typ)
786 {
787 uintptr size, ofs, j, t;
788 uintptr ntypes, nbytes2, nbytes3;
789 uintptr *data2;
790 byte *data3;
792 if(s->sizeclass == 0) {
793 s->types.compression = MTypes_Single;
794 s->types.data = typ;
795 return;
796 }
797 size = s->elemsize;
798 ofs = ((uintptr)v - (s->start<<PageShift)) / size;
800 switch(s->types.compression) {
801 case MTypes_Empty:
802 ntypes = (s->npages << PageShift) / size;
803 nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
804 data3 = runtime_mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC);
805 s->types.compression = MTypes_Bytes;
806 s->types.data = (uintptr)data3;
807 ((uintptr*)data3)[1] = typ;
808 data3[8*sizeof(uintptr) + ofs] = 1;
809 break;
811 case MTypes_Words:
812 ((uintptr*)s->types.data)[ofs] = typ;
813 break;
815 case MTypes_Bytes:
816 data3 = (byte*)s->types.data;
817 for(j=1; j<8; j++) {
818 if(((uintptr*)data3)[j] == typ) {
819 break;
820 }
821 if(((uintptr*)data3)[j] == 0) {
822 ((uintptr*)data3)[j] = typ;
823 break;
824 }
825 }
826 if(j < 8) {
827 data3[8*sizeof(uintptr) + ofs] = j;
828 } else {
829 ntypes = (s->npages << PageShift) / size;
830 nbytes2 = ntypes * sizeof(uintptr);
831 data2 = runtime_mallocgc(nbytes2, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC);
832 s->types.compression = MTypes_Words;
833 s->types.data = (uintptr)data2;
835 // Move the contents of data3 to data2. Then deallocate data3.
836 for(j=0; j<ntypes; j++) {
837 t = data3[8*sizeof(uintptr) + j];
838 t = ((uintptr*)data3)[t];
839 data2[j] = t;
840 }
841 data2[ofs] = typ;
842 }
843 break;
844 }
845 }
847 uintptr
848 runtime_gettype(void *v)
849 {
850 MSpan *s;
851 uintptr t, ofs;
852 byte *data;
854 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
855 if(s != nil) {
856 t = 0;
857 switch(s->types.compression) {
858 case MTypes_Empty:
859 break;
860 case MTypes_Single:
861 t = s->types.data;
862 break;
863 case MTypes_Words:
864 ofs = (uintptr)v - (s->start<<PageShift);
865 t = ((uintptr*)s->types.data)[ofs/s->elemsize];
866 break;
867 case MTypes_Bytes:
868 ofs = (uintptr)v - (s->start<<PageShift);
869 data = (byte*)s->types.data;
870 t = data[8*sizeof(uintptr) + ofs/s->elemsize];
871 t = ((uintptr*)data)[t];
872 break;
873 default:
874 runtime_throw("runtime_gettype: invalid compression kind");
875 }
876 if(0) {
877 runtime_printf("%p -> %d,%X\n", v, (int32)s->types.compression, (int64)t);
878 }
879 return t;
880 }
881 return 0;
882 }
884 // Runtime stubs.
886 void*
887 runtime_mal(uintptr n)
888 {
889 return runtime_mallocgc(n, 0, 0);
890 }
892 func new(typ *Type) (ret *uint8) {
893 ret = runtime_mallocgc(typ->__size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&kindNoPointers ? FlagNoScan : 0);
894 }
896 static void*
897 cnew(const Type *typ, intgo n, int32 objtyp)
898 {
899 if((objtyp&(PtrSize-1)) != objtyp)
900 runtime_throw("runtime: invalid objtyp");
901 if(n < 0 || (typ->__size > 0 && (uintptr)n > (MaxMem/typ->__size)))
902 runtime_panicstring("runtime: allocation size out of range");
903 return runtime_mallocgc(typ->__size*n, (uintptr)typ | objtyp, typ->kind&kindNoPointers ? FlagNoScan : 0);
904 }
906 // same as runtime_new, but callable from C
907 void*
908 runtime_cnew(const Type *typ)
909 {
910 return cnew(typ, 1, TypeInfo_SingleObject);
911 }
913 void*
914 runtime_cnewarray(const Type *typ, intgo n)
915 {
916 return cnew(typ, n, TypeInfo_Array);
917 }
919 func GC() {
920 runtime_gc(2); // force GC and do eager sweep
921 }
923 func SetFinalizer(obj Eface, finalizer Eface) {
924 byte *base;
925 uintptr size;
926 const FuncType *ft;
927 const Type *fint;
928 const PtrType *ot;
930 if(obj.__type_descriptor == nil) {
931 runtime_printf("runtime.SetFinalizer: first argument is nil interface\n");
932 goto throw;
933 }
934 if((obj.__type_descriptor->kind&kindMask) != GO_PTR) {
935 runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.__type_descriptor->__reflection);
936 goto throw;
937 }
938 ot = (const PtrType*)obj.type;
939 // As an implementation detail we do not run finalizers for zero-sized objects,
940 // because we use &runtime_zerobase for all such allocations.
941 if(ot->__element_type != nil && ot->__element_type->__size == 0)
942 return;
943 // The following check is required for cases when a user passes a pointer to composite literal,
944 // but compiler makes it a pointer to global. For example:
945 // var Foo = &Object{}
946 // func main() {
947 // runtime.SetFinalizer(Foo, nil)
948 // }
949 // See issue 7656.
950 if((byte*)obj.__object < runtime_mheap.arena_start || runtime_mheap.arena_used <= (byte*)obj.__object)
951 return;
952 if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) {
953 // As an implementation detail we allow to set finalizers for an inner byte
954 // of an object if it could come from tiny alloc (see mallocgc for details).
955 if(ot->__element_type == nil || (ot->__element_type->kind&kindNoPointers) == 0 || ot->__element_type->__size >= TinySize) {
956 runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block (%p)\n", obj.__object);
957 goto throw;
958 }
959 }
960 if(finalizer.__type_descriptor != nil) {
961 runtime_createfing();
962 if((finalizer.__type_descriptor->kind&kindMask) != GO_FUNC)
963 goto badfunc;
964 ft = (const FuncType*)finalizer.__type_descriptor;
965 if(ft->__dotdotdot || ft->__in.__count != 1)
966 goto badfunc;
967 fint = *(Type**)ft->__in.__values;
968 if(__go_type_descriptors_equal(fint, obj.__type_descriptor)) {
969 // ok - same type
970 } else if((fint->kind&kindMask) == GO_PTR && (fint->__uncommon == nil || fint->__uncommon->__name == nil || obj.type->__uncommon == nil || obj.type->__uncommon->__name == nil) && __go_type_descriptors_equal(((const PtrType*)fint)->__element_type, ((const PtrType*)obj.type)->__element_type)) {
971 // ok - not same type, but both pointers,
972 // one or the other is unnamed, and same element type, so assignable.
973 } else if((fint->kind&kindMask) == GO_INTERFACE && ((const InterfaceType*)fint)->__methods.__count == 0) {
974 // ok - satisfies empty interface
975 } else if((fint->kind&kindMask) == GO_INTERFACE && __go_convert_interface_2(fint, obj.__type_descriptor, 1) != nil) {
976 // ok - satisfies non-empty interface
977 } else
978 goto badfunc;
980 ot = (const PtrType*)obj.__type_descriptor;
981 if(!runtime_addfinalizer(obj.__object, *(FuncVal**)finalizer.__object, ft, ot)) {
982 runtime_printf("runtime.SetFinalizer: finalizer already set\n");
983 goto throw;
984 }
985 } else {
986 // NOTE: asking to remove a finalizer when there currently isn't one set is OK.
987 runtime_removefinalizer(obj.__object);
988 }
989 return;
991 badfunc:
992 runtime_printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.__type_descriptor->__reflection, *finalizer.__type_descriptor->__reflection);
993 throw:
994 runtime_throw("runtime.SetFinalizer");
995 }
997 func KeepAlive(x Eface) {
998 USED(x);
999 }