| blob | 9c8b8c1c74cd7a2cf7525f10306ce0ea1455089c |
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // See malloc.h for overview.
6 //
7 // TODO(rsc): double-check stats.
9 package runtime
10 #include <stddef.h>
11 #include <errno.h>
12 #include <stdlib.h>
13 #include "go-alloc.h"
14 #include "runtime.h"
15 #include "arch.h"
16 #include "malloc.h"
17 #include "interface.h"
18 #include "go-type.h"
19 #include "race.h"
21 // Map gccgo field names to gc field names.
22 // Eface aka __go_empty_interface.
23 #define type __type_descriptor
24 // Type aka __go_type_descriptor
25 #define kind __code
26 #define string __reflection
27 #define KindPtr GO_PTR
28 #define KindNoPointers GO_NO_POINTERS
30 // GCCGO SPECIFIC CHANGE
31 //
32 // There is a long comment in runtime_mallocinit about where to put the heap
33 // on a 64-bit system. It makes assumptions that are not valid on linux/arm64
34 // -- it assumes user space can choose the lower 47 bits of a pointer, but on
35 // linux/arm64 we can only choose the lower 39 bits. This means the heap is
36 // roughly a quarter of the available address space and we cannot choose a bit
37 // pattern that all pointers will have -- luckily the GC is mostly precise
38 // these days so this doesn't matter all that much. The kernel (as of 3.13)
39 // will allocate address space starting either down from 0x7fffffffff or up
40 // from 0x2000000000, so we put the heap roughly in the middle of these two
41 // addresses to minimize the chance that a non-heap allocation will get in the
42 // way of the heap.
43 //
44 // This all means that there isn't much point in trying 256 different
45 // locations for the heap on such systems.
46 #ifdef __aarch64__
47 #define HeapBase(i) ((void*)(uintptr)(0x40ULL<<32))
48 #define HeapBaseOptions 1
49 #else
50 #define HeapBase(i) ((void*)(uintptr)(i<<40|0x00c0ULL<<32))
51 #define HeapBaseOptions 0x80
52 #endif
53 // END GCCGO SPECIFIC CHANGE
55 // Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K.
56 MHeap runtime_mheap;
57 MStats mstats;
59 int32 runtime_checking;
61 extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go
63 extern volatile intgo runtime_MemProfileRate
64 __asm__ (GOSYM_PREFIX "runtime.MemProfileRate");
66 static void* largealloc(uint32, uintptr*);
67 static void profilealloc(void *v, uintptr size, uintptr typ);
69 // Allocate an object of at least size bytes.
70 // Small objects are allocated from the per-thread cache's free lists.
71 // Large objects (> 32 kB) are allocated straight from the heap.
72 // If the block will be freed with runtime_free(), typ must be 0.
73 void*
74 runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
75 {
76 M *m;
77 G *g;
78 int32 sizeclass;
79 uintptr tinysize, size1;
80 intgo rate;
81 MCache *c;
82 MCacheList *l;
83 MLink *v, *next;
84 byte *tiny;
85 bool incallback;
87 if(size == 0) {
88 // All 0-length allocations use this pointer.
89 // The language does not require the allocations to
90 // have distinct values.
91 return &runtime_zerobase;
92 }
94 m = runtime_m();
95 g = runtime_g();
97 incallback = false;
98 if(m->mcache == nil && g->ncgo > 0) {
99 // For gccgo this case can occur when a cgo or SWIG function
100 // has an interface return type and the function
101 // returns a non-pointer, so memory allocation occurs
102 // after syscall.Cgocall but before syscall.CgocallDone.
103 // We treat it as a callback.
104 runtime_exitsyscall();
105 m = runtime_m();
106 incallback = true;
107 flag |= FlagNoInvokeGC;
108 }
110 if(runtime_gcwaiting() && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC)) {
111 runtime_gosched();
112 m = runtime_m();
113 }
114 if(m->mallocing)
115 runtime_throw("malloc/free - deadlock");
116 // Disable preemption during settype_flush.
117 // We can not use m->mallocing for this, because settype_flush calls mallocgc.
118 m->locks++;
119 m->mallocing = 1;
121 if(DebugTypeAtBlockEnd)
122 size += sizeof(uintptr);
124 c = m->mcache;
125 if(!runtime_debug.efence && size <= MaxSmallSize) {
126 if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize) {
127 // Tiny allocator.
128 //
129 // Tiny allocator combines several tiny allocation requests
130 // into a single memory block. The resulting memory block
131 // is freed when all subobjects are unreachable. The subobjects
132 // must be FlagNoScan (don't have pointers), this ensures that
133 // the amount of potentially wasted memory is bounded.
134 //
135 // Size of the memory block used for combining (TinySize) is tunable.
136 // Current setting is 16 bytes, which relates to 2x worst case memory
137 // wastage (when all but one subobjects are unreachable).
138 // 8 bytes would result in no wastage at all, but provides less
139 // opportunities for combining.
140 // 32 bytes provides more opportunities for combining,
141 // but can lead to 4x worst case wastage.
142 // The best case winning is 8x regardless of block size.
143 //
144 // Objects obtained from tiny allocator must not be freed explicitly.
145 // So when an object will be freed explicitly, we ensure that
146 // its size >= TinySize.
147 //
148 // SetFinalizer has a special case for objects potentially coming
149 // from tiny allocator, it such case it allows to set finalizers
150 // for an inner byte of a memory block.
151 //
152 // The main targets of tiny allocator are small strings and
153 // standalone escaping variables. On a json benchmark
154 // the allocator reduces number of allocations by ~12% and
155 // reduces heap size by ~20%.
157 tinysize = c->tinysize;
158 if(size <= tinysize) {
159 tiny = c->tiny;
160 // Align tiny pointer for required (conservative) alignment.
161 if((size&7) == 0)
162 tiny = (byte*)ROUND((uintptr)tiny, 8);
163 else if((size&3) == 0)
164 tiny = (byte*)ROUND((uintptr)tiny, 4);
165 else if((size&1) == 0)
166 tiny = (byte*)ROUND((uintptr)tiny, 2);
167 size1 = size + (tiny - c->tiny);
168 if(size1 <= tinysize) {
169 // The object fits into existing tiny block.
170 v = (MLink*)tiny;
171 c->tiny += size1;
172 c->tinysize -= size1;
173 m->mallocing = 0;
174 m->locks--;
175 if(incallback)
176 runtime_entersyscall();
177 return v;
178 }
179 }
180 // Allocate a new TinySize block.
181 l = &c->list[TinySizeClass];
182 if(l->list == nil)
183 runtime_MCache_Refill(c, TinySizeClass);
184 v = l->list;
185 next = v->next;
186 if(next != nil) // prefetching nil leads to a DTLB miss
187 PREFETCH(next);
188 l->list = next;
189 l->nlist--;
190 ((uint64*)v)[0] = 0;
191 ((uint64*)v)[1] = 0;
192 // See if we need to replace the existing tiny block with the new one
193 // based on amount of remaining free space.
194 if(TinySize-size > tinysize) {
195 c->tiny = (byte*)v + size;
196 c->tinysize = TinySize - size;
197 }
198 size = TinySize;
199 goto done;
200 }
201 // Allocate from mcache free lists.
202 // Inlined version of SizeToClass().
203 if(size <= 1024-8)
204 sizeclass = runtime_size_to_class8[(size+7)>>3];
205 else
206 sizeclass = runtime_size_to_class128[(size-1024+127) >> 7];
207 size = runtime_class_to_size[sizeclass];
208 l = &c->list[sizeclass];
209 if(l->list == nil)
210 runtime_MCache_Refill(c, sizeclass);
211 v = l->list;
212 next = v->next;
213 if(next != nil) // prefetching nil leads to a DTLB miss
214 PREFETCH(next);
215 l->list = next;
216 l->nlist--;
217 if(!(flag & FlagNoZero)) {
218 v->next = nil;
219 // block is zeroed iff second word is zero ...
220 if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0)
221 runtime_memclr((byte*)v, size);
222 }
223 done:
224 c->local_cachealloc += size;
225 } else {
226 // Allocate directly from heap.
227 v = largealloc(flag, &size);
228 }
230 if(flag & FlagNoGC)
231 runtime_marknogc(v);
232 else if(!(flag & FlagNoScan))
233 runtime_markscan(v);
235 if(DebugTypeAtBlockEnd)
236 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ;
238 // TODO: save type even if FlagNoScan? Potentially expensive but might help
239 // heap profiling/tracing.
240 if(UseSpanType && !(flag & FlagNoScan) && typ != 0) {
241 uintptr *buf, i;
243 buf = m->settype_buf;
244 i = m->settype_bufsize;
245 buf[i++] = (uintptr)v;
246 buf[i++] = typ;
247 m->settype_bufsize = i;
248 }
250 m->mallocing = 0;
251 if(UseSpanType && !(flag & FlagNoScan) && typ != 0 && m->settype_bufsize == nelem(m->settype_buf))
252 runtime_settype_flush(m);
253 if(raceenabled)
254 runtime_racemalloc(v, size);
256 if(runtime_debug.allocfreetrace)
257 goto profile;
259 if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
260 if(size < (uintptr)rate && size < (uintptr)(uint32)c->next_sample)
261 c->next_sample -= size;
262 else {
263 profile:
264 profilealloc(v, size, typ);
265 }
266 }
268 m->locks--;
270 if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc)
271 runtime_gc(0);
273 if(incallback)
274 runtime_entersyscall();
276 return v;
277 }
279 static void*
280 largealloc(uint32 flag, uintptr *sizep)
281 {
282 uintptr npages, size;
283 MSpan *s;
284 void *v;
286 // Allocate directly from heap.
287 size = *sizep;
288 if(size + PageSize < size)
289 runtime_throw("out of memory");
290 npages = size >> PageShift;
291 if((size & PageMask) != 0)
292 npages++;
293 s = runtime_MHeap_Alloc(&runtime_mheap, npages, 0, 1, !(flag & FlagNoZero));
294 if(s == nil)
295 runtime_throw("out of memory");
296 s->limit = (byte*)(s->start<<PageShift) + size;
297 *sizep = npages<<PageShift;
298 v = (void*)(s->start << PageShift);
299 // setup for mark sweep
300 runtime_markspan(v, 0, 0, true);
301 return v;
302 }
304 static void
305 profilealloc(void *v, uintptr size, uintptr typ)
306 {
307 uintptr rate;
308 int32 next;
309 MCache *c;
311 c = runtime_m()->mcache;
312 rate = runtime_MemProfileRate;
313 if(size < rate) {
314 // pick next profile time
315 // If you change this, also change allocmcache.
316 if(rate > 0x3fffffff) // make 2*rate not overflow
317 rate = 0x3fffffff;
318 next = runtime_fastrand1() % (2*rate);
319 // Subtract the "remainder" of the current allocation.
320 // Otherwise objects that are close in size to sampling rate
321 // will be under-sampled, because we consistently discard this remainder.
322 next -= (size - c->next_sample);
323 if(next < 0)
324 next = 0;
325 c->next_sample = next;
326 }
327 runtime_MProf_Malloc(v, size, typ);
328 }
330 void*
331 __go_alloc(uintptr size)
332 {
333 return runtime_mallocgc(size, 0, FlagNoInvokeGC);
334 }
336 // Free the object whose base pointer is v.
337 void
338 __go_free(void *v)
339 {
340 M *m;
341 int32 sizeclass;
342 MSpan *s;
343 MCache *c;
344 uintptr size;
346 if(v == nil)
347 return;
349 // If you change this also change mgc0.c:/^sweep,
350 // which has a copy of the guts of free.
352 m = runtime_m();
353 if(m->mallocing)
354 runtime_throw("malloc/free - deadlock");
355 m->mallocing = 1;
357 if(!runtime_mlookup(v, nil, nil, &s)) {
358 runtime_printf("free %p: not an allocated block\n", v);
359 runtime_throw("free runtime_mlookup");
360 }
361 size = s->elemsize;
362 sizeclass = s->sizeclass;
363 // Objects that are smaller than TinySize can be allocated using tiny alloc,
364 // if then such object is combined with an object with finalizer, we will crash.
365 if(size < TinySize)
366 runtime_throw("freeing too small block");
368 if(raceenabled)
369 runtime_racefree(v);
371 // Ensure that the span is swept.
372 // If we free into an unswept span, we will corrupt GC bitmaps.
373 runtime_MSpan_EnsureSwept(s);
375 if(s->specials != nil)
376 runtime_freeallspecials(s, v, size);
378 c = m->mcache;
379 if(sizeclass == 0) {
380 // Large object.
381 s->needzero = 1;
382 // Must mark v freed before calling unmarkspan and MHeap_Free:
383 // they might coalesce v into other spans and change the bitmap further.
384 runtime_markfreed(v, size);
385 runtime_unmarkspan(v, 1<<PageShift);
386 if(runtime_debug.efence)
387 runtime_SysFree((void*)(s->start<<PageShift), size, &mstats.heap_sys);
388 else
389 runtime_MHeap_Free(&runtime_mheap, s, 1);
390 c->local_nlargefree++;
391 c->local_largefree += size;
392 } else {
393 // Small object.
394 if(size > 2*sizeof(uintptr))
395 ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed"
396 else if(size > sizeof(uintptr))
397 ((uintptr*)v)[1] = 0;
398 // Must mark v freed before calling MCache_Free:
399 // it might coalesce v and other blocks into a bigger span
400 // and change the bitmap further.
401 runtime_markfreed(v, size);
402 c->local_nsmallfree[sizeclass]++;
403 runtime_MCache_Free(c, v, sizeclass, size);
404 }
405 m->mallocing = 0;
406 }
408 int32
409 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
410 {
411 M *m;
412 uintptr n, i;
413 byte *p;
414 MSpan *s;
416 m = runtime_m();
418 m->mcache->local_nlookup++;
419 if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) {
420 // purge cache stats to prevent overflow
421 runtime_lock(&runtime_mheap);
422 runtime_purgecachedstats(m->mcache);
423 runtime_unlock(&runtime_mheap);
424 }
426 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
427 if(sp)
428 *sp = s;
429 if(s == nil) {
430 runtime_checkfreed(v, 1);
431 if(base)
432 *base = nil;
433 if(size)
434 *size = 0;
435 return 0;
436 }
438 p = (byte*)((uintptr)s->start<<PageShift);
439 if(s->sizeclass == 0) {
440 // Large object.
441 if(base)
442 *base = p;
443 if(size)
444 *size = s->npages<<PageShift;
445 return 1;
446 }
448 n = s->elemsize;
449 if(base) {
450 i = ((byte*)v - p)/n;
451 *base = p + i*n;
452 }
453 if(size)
454 *size = n;
456 return 1;
457 }
459 MCache*
460 runtime_allocmcache(void)
461 {
462 intgo rate;
463 MCache *c;
465 runtime_lock(&runtime_mheap);
466 c = runtime_FixAlloc_Alloc(&runtime_mheap.cachealloc);
467 runtime_unlock(&runtime_mheap);
468 runtime_memclr((byte*)c, sizeof(*c));
470 // Set first allocation sample size.
471 rate = runtime_MemProfileRate;
472 if(rate > 0x3fffffff) // make 2*rate not overflow
473 rate = 0x3fffffff;
474 if(rate != 0)
475 c->next_sample = runtime_fastrand1() % (2*rate);
477 return c;
478 }
480 void
481 runtime_freemcache(MCache *c)
482 {
483 runtime_MCache_ReleaseAll(c);
484 runtime_lock(&runtime_mheap);
485 runtime_purgecachedstats(c);
486 runtime_FixAlloc_Free(&runtime_mheap.cachealloc, c);
487 runtime_unlock(&runtime_mheap);
488 }
490 void
491 runtime_purgecachedstats(MCache *c)
492 {
493 MHeap *h;
494 int32 i;
496 // Protected by either heap or GC lock.
497 h = &runtime_mheap;
498 mstats.heap_alloc += c->local_cachealloc;
499 c->local_cachealloc = 0;
500 mstats.nlookup += c->local_nlookup;
501 c->local_nlookup = 0;
502 h->largefree += c->local_largefree;
503 c->local_largefree = 0;
504 h->nlargefree += c->local_nlargefree;
505 c->local_nlargefree = 0;
506 for(i=0; i<(int32)nelem(c->local_nsmallfree); i++) {
507 h->nsmallfree[i] += c->local_nsmallfree[i];
508 c->local_nsmallfree[i] = 0;
509 }
510 }
512 extern uintptr runtime_sizeof_C_MStats
513 __asm__ (GOSYM_PREFIX "runtime.Sizeof_C_MStats");
515 // Size of the trailing by_size array differs between Go and C,
516 // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
517 // sizeof_C_MStats is what C thinks about size of Go struct.
519 // Initialized in mallocinit because it's defined in go/runtime/mem.go.
521 #define MaxArena32 (2U<<30)
523 void
524 runtime_mallocinit(void)
525 {
526 byte *p;
527 uintptr arena_size, bitmap_size, spans_size;
528 extern byte _end[];
529 uintptr limit;
530 uint64 i;
532 runtime_sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeof(mstats.by_size[0]);
534 p = nil;
535 arena_size = 0;
536 bitmap_size = 0;
537 spans_size = 0;
539 // for 64-bit build
540 USED(p);
541 USED(arena_size);
542 USED(bitmap_size);
543 USED(spans_size);
545 runtime_InitSizes();
547 if(runtime_class_to_size[TinySizeClass] != TinySize)
548 runtime_throw("bad TinySizeClass");
550 // limit = runtime_memlimit();
551 // See https://code.google.com/p/go/issues/detail?id=5049
552 // TODO(rsc): Fix after 1.1.
553 limit = 0;
555 // Set up the allocation arena, a contiguous area of memory where
556 // allocated data will be found. The arena begins with a bitmap large
557 // enough to hold 4 bits per allocated word.
558 if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) {
559 // On a 64-bit machine, allocate from a single contiguous reservation.
560 // 128 GB (MaxMem) should be big enough for now.
561 //
562 // The code will work with the reservation at any address, but ask
563 // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f).
564 // Allocating a 128 GB region takes away 37 bits, and the amd64
565 // doesn't let us choose the top 17 bits, so that leaves the 11 bits
566 // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means
567 // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df.
568 // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid
569 // UTF-8 sequences, and they are otherwise as far away from
570 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0
571 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors
572 // on OS X during thread allocations. 0x00c0 causes conflicts with
573 // AddressSanitizer which reserves all memory up to 0x0100.
574 // These choices are both for debuggability and to reduce the
575 // odds of the conservative garbage collector not collecting memory
576 // because some non-pointer block of memory had a bit pattern
577 // that matched a memory address.
578 //
579 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB)
580 // but it hardly matters: e0 00 is not valid UTF-8 either.
581 //
582 // If this fails we fall back to the 32 bit memory mechanism
583 arena_size = MaxMem;
584 bitmap_size = arena_size / (sizeof(void*)*8/4);
585 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]);
586 spans_size = ROUND(spans_size, PageSize);
587 for(i = 0; i < HeapBaseOptions; i++) {
588 p = runtime_SysReserve(HeapBase(i), bitmap_size + spans_size + arena_size + PageSize);
589 if(p != nil)
590 break;
591 }
592 }
593 if (p == nil) {
594 // On a 32-bit machine, we can't typically get away
595 // with a giant virtual address space reservation.
596 // Instead we map the memory information bitmap
597 // immediately after the data segment, large enough
598 // to handle another 2GB of mappings (256 MB),
599 // along with a reservation for another 512 MB of memory.
600 // When that gets used up, we'll start asking the kernel
601 // for any memory anywhere and hope it's in the 2GB
602 // following the bitmap (presumably the executable begins
603 // near the bottom of memory, so we'll have to use up
604 // most of memory before the kernel resorts to giving out
605 // memory before the beginning of the text segment).
606 //
607 // Alternatively we could reserve 512 MB bitmap, enough
608 // for 4GB of mappings, and then accept any memory the
609 // kernel threw at us, but normally that's a waste of 512 MB
610 // of address space, which is probably too much in a 32-bit world.
611 bitmap_size = MaxArena32 / (sizeof(void*)*8/4);
612 arena_size = 512<<20;
613 spans_size = MaxArena32 / PageSize * sizeof(runtime_mheap.spans[0]);
614 if(limit > 0 && arena_size+bitmap_size+spans_size > limit) {
615 bitmap_size = (limit / 9) & ~((1<<PageShift) - 1);
616 arena_size = bitmap_size * 8;
617 spans_size = arena_size / PageSize * sizeof(runtime_mheap.spans[0]);
618 }
619 spans_size = ROUND(spans_size, PageSize);
621 // SysReserve treats the address we ask for, end, as a hint,
622 // not as an absolute requirement. If we ask for the end
623 // of the data segment but the operating system requires
624 // a little more space before we can start allocating, it will
625 // give out a slightly higher pointer. Except QEMU, which
626 // is buggy, as usual: it won't adjust the pointer upward.
627 // So adjust it upward a little bit ourselves: 1/4 MB to get
628 // away from the running binary image and then round up
629 // to a MB boundary.
630 p = (byte*)ROUND((uintptr)_end + (1<<18), 1<<20);
631 p = runtime_SysReserve(p, bitmap_size + spans_size + arena_size + PageSize);
632 if(p == nil)
633 runtime_throw("runtime: cannot reserve arena virtual address space");
634 }
636 // PageSize can be larger than OS definition of page size,
637 // so SysReserve can give us a PageSize-unaligned pointer.
638 // To overcome this we ask for PageSize more and round up the pointer.
639 p = (byte*)ROUND((uintptr)p, PageSize);
641 runtime_mheap.spans = (MSpan**)p;
642 runtime_mheap.bitmap = p + spans_size;
643 runtime_mheap.arena_start = p + spans_size + bitmap_size;
644 runtime_mheap.arena_used = runtime_mheap.arena_start;
645 runtime_mheap.arena_end = runtime_mheap.arena_start + arena_size;
647 // Initialize the rest of the allocator.
648 runtime_MHeap_Init(&runtime_mheap);
649 runtime_m()->mcache = runtime_allocmcache();
651 // See if it works.
652 runtime_free(runtime_malloc(TinySize));
653 }
655 void*
656 runtime_MHeap_SysAlloc(MHeap *h, uintptr n)
657 {
658 byte *p;
661 if(n > (uintptr)(h->arena_end - h->arena_used)) {
662 // We are in 32-bit mode, maybe we didn't use all possible address space yet.
663 // Reserve some more space.
664 byte *new_end;
665 uintptr needed;
667 needed = (uintptr)h->arena_used + n - (uintptr)h->arena_end;
668 needed = ROUND(needed, 256<<20);
669 new_end = h->arena_end + needed;
670 if(new_end <= h->arena_start + MaxArena32) {
671 p = runtime_SysReserve(h->arena_end, new_end - h->arena_end);
672 if(p == h->arena_end)
673 h->arena_end = new_end;
674 }
675 }
676 if(n <= (uintptr)(h->arena_end - h->arena_used)) {
677 // Keep taking from our reservation.
678 p = h->arena_used;
679 runtime_SysMap(p, n, &mstats.heap_sys);
680 h->arena_used += n;
681 runtime_MHeap_MapBits(h);
682 runtime_MHeap_MapSpans(h);
683 if(raceenabled)
684 runtime_racemapshadow(p, n);
685 return p;
686 }
688 // If using 64-bit, our reservation is all we have.
689 if(sizeof(void*) == 8 && (uintptr)h->bitmap >= 0xffffffffU)
690 return nil;
692 // On 32-bit, once the reservation is gone we can
693 // try to get memory at a location chosen by the OS
694 // and hope that it is in the range we allocated bitmap for.
695 p = runtime_SysAlloc(n, &mstats.heap_sys);
696 if(p == nil)
697 return nil;
699 if(p < h->arena_start || (uintptr)(p+n - h->arena_start) >= MaxArena32) {
700 runtime_printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n",
701 p, h->arena_start, h->arena_start+MaxArena32);
702 runtime_SysFree(p, n, &mstats.heap_sys);
703 return nil;
704 }
706 if(p+n > h->arena_used) {
707 h->arena_used = p+n;
708 if(h->arena_used > h->arena_end)
709 h->arena_end = h->arena_used;
710 runtime_MHeap_MapBits(h);
711 runtime_MHeap_MapSpans(h);
712 if(raceenabled)
713 runtime_racemapshadow(p, n);
714 }
716 return p;
717 }
719 static struct
720 {
721 Lock;
722 byte* pos;
723 byte* end;
724 } persistent;
726 enum
727 {
728 PersistentAllocChunk = 256<<10,
729 PersistentAllocMaxBlock = 64<<10, // VM reservation granularity is 64K on windows
730 };
732 // Wrapper around SysAlloc that can allocate small chunks.
733 // There is no associated free operation.
734 // Intended for things like function/type/debug-related persistent data.
735 // If align is 0, uses default align (currently 8).
736 void*
737 runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat)
738 {
739 byte *p;
741 if(align != 0) {
742 if(align&(align-1))
743 runtime_throw("persistentalloc: align is now a power of 2");
744 if(align > PageSize)
745 runtime_throw("persistentalloc: align is too large");
746 } else
747 align = 8;
748 if(size >= PersistentAllocMaxBlock)
749 return runtime_SysAlloc(size, stat);
750 runtime_lock(&persistent);
751 persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align);
752 if(persistent.pos + size > persistent.end) {
753 persistent.pos = runtime_SysAlloc(PersistentAllocChunk, &mstats.other_sys);
754 if(persistent.pos == nil) {
755 runtime_unlock(&persistent);
756 runtime_throw("runtime: cannot allocate memory");
757 }
758 persistent.end = persistent.pos + PersistentAllocChunk;
759 }
760 p = persistent.pos;
761 persistent.pos += size;
762 runtime_unlock(&persistent);
763 if(stat != &mstats.other_sys) {
764 // reaccount the allocation against provided stat
765 runtime_xadd64(stat, size);
766 runtime_xadd64(&mstats.other_sys, -(uint64)size);
767 }
768 return p;
769 }
771 static Lock settype_lock;
773 void
774 runtime_settype_flush(M *mp)
775 {
776 uintptr *buf, *endbuf;
777 uintptr size, ofs, j, t;
778 uintptr ntypes, nbytes2, nbytes3;
779 uintptr *data2;
780 byte *data3;
781 void *v;
782 uintptr typ, p;
783 MSpan *s;
785 buf = mp->settype_buf;
786 endbuf = buf + mp->settype_bufsize;
788 runtime_lock(&settype_lock);
789 while(buf < endbuf) {
790 v = (void*)*buf;
791 *buf = 0;
792 buf++;
793 typ = *buf;
794 buf++;
796 // (Manually inlined copy of runtime_MHeap_Lookup)
797 p = (uintptr)v>>PageShift;
798 p -= (uintptr)runtime_mheap.arena_start >> PageShift;
799 s = runtime_mheap.spans[p];
801 if(s->sizeclass == 0) {
802 s->types.compression = MTypes_Single;
803 s->types.data = typ;
804 continue;
805 }
807 size = s->elemsize;
808 ofs = ((uintptr)v - (s->start<<PageShift)) / size;
810 switch(s->types.compression) {
811 case MTypes_Empty:
812 ntypes = (s->npages << PageShift) / size;
813 nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
814 data3 = runtime_mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC);
815 s->types.compression = MTypes_Bytes;
816 s->types.data = (uintptr)data3;
817 ((uintptr*)data3)[1] = typ;
818 data3[8*sizeof(uintptr) + ofs] = 1;
819 break;
821 case MTypes_Words:
822 ((uintptr*)s->types.data)[ofs] = typ;
823 break;
825 case MTypes_Bytes:
826 data3 = (byte*)s->types.data;
827 for(j=1; j<8; j++) {
828 if(((uintptr*)data3)[j] == typ) {
829 break;
830 }
831 if(((uintptr*)data3)[j] == 0) {
832 ((uintptr*)data3)[j] = typ;
833 break;
834 }
835 }
836 if(j < 8) {
837 data3[8*sizeof(uintptr) + ofs] = j;
838 } else {
839 ntypes = (s->npages << PageShift) / size;
840 nbytes2 = ntypes * sizeof(uintptr);
841 data2 = runtime_mallocgc(nbytes2, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC);
842 s->types.compression = MTypes_Words;
843 s->types.data = (uintptr)data2;
845 // Move the contents of data3 to data2. Then deallocate data3.
846 for(j=0; j<ntypes; j++) {
847 t = data3[8*sizeof(uintptr) + j];
848 t = ((uintptr*)data3)[t];
849 data2[j] = t;
850 }
851 data2[ofs] = typ;
852 }
853 break;
854 }
855 }
856 runtime_unlock(&settype_lock);
858 mp->settype_bufsize = 0;
859 }
861 uintptr
862 runtime_gettype(void *v)
863 {
864 MSpan *s;
865 uintptr t, ofs;
866 byte *data;
868 s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
869 if(s != nil) {
870 t = 0;
871 switch(s->types.compression) {
872 case MTypes_Empty:
873 break;
874 case MTypes_Single:
875 t = s->types.data;
876 break;
877 case MTypes_Words:
878 ofs = (uintptr)v - (s->start<<PageShift);
879 t = ((uintptr*)s->types.data)[ofs/s->elemsize];
880 break;
881 case MTypes_Bytes:
882 ofs = (uintptr)v - (s->start<<PageShift);
883 data = (byte*)s->types.data;
884 t = data[8*sizeof(uintptr) + ofs/s->elemsize];
885 t = ((uintptr*)data)[t];
886 break;
887 default:
888 runtime_throw("runtime_gettype: invalid compression kind");
889 }
890 if(0) {
891 runtime_lock(&settype_lock);
892 runtime_printf("%p -> %d,%X\n", v, (int32)s->types.compression, (int64)t);
893 runtime_unlock(&settype_lock);
894 }
895 return t;
896 }
897 return 0;
898 }
900 // Runtime stubs.
902 void*
903 runtime_mal(uintptr n)
904 {
905 return runtime_mallocgc(n, 0, 0);
906 }
908 func new(typ *Type) (ret *uint8) {
909 ret = runtime_mallocgc(typ->__size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoScan : 0);
910 }
912 static void*
913 cnew(const Type *typ, intgo n, int32 objtyp)
914 {
915 if((objtyp&(PtrSize-1)) != objtyp)
916 runtime_throw("runtime: invalid objtyp");
917 if(n < 0 || (typ->__size > 0 && (uintptr)n > (MaxMem/typ->__size)))
918 runtime_panicstring("runtime: allocation size out of range");
919 return runtime_mallocgc(typ->__size*n, (uintptr)typ | objtyp, typ->kind&KindNoPointers ? FlagNoScan : 0);
920 }
922 // same as runtime_new, but callable from C
923 void*
924 runtime_cnew(const Type *typ)
925 {
926 return cnew(typ, 1, TypeInfo_SingleObject);
927 }
929 void*
930 runtime_cnewarray(const Type *typ, intgo n)
931 {
932 return cnew(typ, n, TypeInfo_Array);
933 }
935 func GC() {
936 runtime_gc(1);
937 }
939 func SetFinalizer(obj Eface, finalizer Eface) {
940 byte *base;
941 uintptr size;
942 const FuncType *ft;
943 const Type *fint;
944 const PtrType *ot;
946 if(obj.__type_descriptor == nil) {
947 runtime_printf("runtime.SetFinalizer: first argument is nil interface\n");
948 goto throw;
949 }
950 if(obj.__type_descriptor->__code != GO_PTR) {
951 runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.__type_descriptor->__reflection);
952 goto throw;
953 }
954 ot = (const PtrType*)obj.type;
955 // As an implementation detail we do not run finalizers for zero-sized objects,
956 // because we use &runtime_zerobase for all such allocations.
957 if(ot->__element_type != nil && ot->__element_type->__size == 0)
958 return;
959 if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) {
960 // As an implementation detail we allow to set finalizers for an inner byte
961 // of an object if it could come from tiny alloc (see mallocgc for details).
962 if(ot->__element_type == nil || (ot->__element_type->__code&GO_NO_POINTERS) == 0 || ot->__element_type->__size >= TinySize) {
963 runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
964 goto throw;
965 }
966 }
967 if(finalizer.__type_descriptor != nil) {
968 if(finalizer.__type_descriptor->__code != GO_FUNC)
969 goto badfunc;
970 ft = (const FuncType*)finalizer.__type_descriptor;
971 if(ft->__dotdotdot || ft->__in.__count != 1)
972 goto badfunc;
973 fint = *(Type**)ft->__in.__values;
974 if(__go_type_descriptors_equal(fint, obj.__type_descriptor)) {
975 // ok - same type
976 } else if(fint->__code == GO_PTR && (fint->__uncommon == nil || fint->__uncommon->__name == nil || obj.type->__uncommon == nil || obj.type->__uncommon->__name == nil) && __go_type_descriptors_equal(((const PtrType*)fint)->__element_type, ((const PtrType*)obj.type)->__element_type)) {
977 // ok - not same type, but both pointers,
978 // one or the other is unnamed, and same element type, so assignable.
979 } else if(fint->kind == GO_INTERFACE && ((const InterfaceType*)fint)->__methods.__count == 0) {
980 // ok - satisfies empty interface
981 } else if(fint->kind == GO_INTERFACE && __go_convert_interface_2(fint, obj.__type_descriptor, 1) != nil) {
982 // ok - satisfies non-empty interface
983 } else
984 goto badfunc;
986 ot = (const PtrType*)obj.__type_descriptor;
987 if(!runtime_addfinalizer(obj.__object, *(FuncVal**)finalizer.__object, ft, ot)) {
988 runtime_printf("runtime.SetFinalizer: finalizer already set\n");
989 goto throw;
990 }
991 } else {
992 // NOTE: asking to remove a finalizer when there currently isn't one set is OK.
993 runtime_removefinalizer(obj.__object);
994 }
995 return;
997 badfunc:
998 runtime_printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.__type_descriptor->__reflection, *finalizer.__type_descriptor->__reflection);
999 throw:
1000 runtime_throw("runtime.SetFinalizer");
1001 }