4 @maintainer Morgan McGuire, matrix@graphics3d.com
6 Note: every routine must call init() first.
8 There are two kinds of detection used in this file. At compile
9 time, the _MSC_VER #define is used to determine whether x86 assembly
10 can be used at all. At runtime, processor detection is used to
11 determine if we can safely call the routines that use that assembly.
13 @cite Rob Wyatt http://www.gamasutra.com/features/wyatts_world/19990709/processor_detection_01.htm
14 @cite Benjamin Jurke http://www.flipcode.com/cgi-bin/msg.cgi?showThread=COTD-ProcessorDetectionClass&forum=cotd&id=-1
15 @cite Michael Herf http://www.stereopsis.com/memcpy.html
21 #include "G3D/platform.h"
22 #include "G3D/System.h"
23 #include "G3D/debug.h"
24 #include "G3D/format.h"
26 #if defined(__OpenBSD__)
33 #include <sys/timeb.h>
34 #include "G3D/RegistryUtil.h"
36 #elif defined(G3D_LINUX)
42 #include <sys/types.h>
43 #include <sys/select.h>
46 #include <sys/ioctl.h>
50 // #include <assert.h>
52 #elif defined(G3D_OSX)
57 #include <sys/types.h>
58 #include <sys/sysctl.h>
59 #include <sys/select.h>
64 #include <mach-o/arch.h>
67 #include <CoreServices/CoreServices.h>
71 #include <xmmintrin.h>
76 static char versionCstr
[1024];
77 System::OutOfMemoryCallback
System::outOfMemoryCallback
= NULL
;
81 // Cannot use most G3D data structures or utility functions in here because
82 // they are not initialized.
84 static bool initialized
= false;
92 if ((G3D_VER
% 100) != 0) {
93 sprintf(versionCstr
, "G3D %d.%02d beta %d",
95 (G3D_VER
/ 100) % 100,
98 sprintf(versionCstr
, "G3D %d.%02d",
100 (G3D_VER
/ 100) % 100);
107 void System::memcpy(void* dst
, const void* src
, size_t numBytes
) {
108 ::memcpy(dst
, src
, numBytes
);
112 void System::memset(void* dst
, uint8 value
, size_t numBytes
) {
113 ::memset(dst
, value
, numBytes
);
120 ////////////////////////////////////////////////////////////////
124 /** Only store buffers up to these sizes (in bytes) in each pool->
125 Different pools have different management strategies.
127 A large block is preallocated for tiny buffers; they are used with
128 tremendous frequency. Other buffers are allocated as demanded.
130 enum {tinyBufferSize
= 128, smallBufferSize
= 1024, medBufferSize
= 4096};
133 Most buffers we're allowed to store.
134 64000 * 128 = 8 MB (preallocated)
135 1024 * 1024 = 1 MB (allocated on demand)
136 1024 * 4096 = 4 MB (allocated on demand)
138 enum {maxTinyBuffers
= 64000, maxSmallBuffers
= 1024, maxMedBuffers
= 1024};
147 inline MemBlock() : ptr(NULL
), bytes(0) {}
148 inline MemBlock(void* p
, size_t b
) : ptr(p
), bytes(b
) {}
151 MemBlock smallPool
[maxSmallBuffers
];
154 MemBlock medPool
[maxMedBuffers
];
157 /** The tiny pool is a single block of storage into which all tiny
158 objects are allocated. This provides better locality for
159 small objects and avoids the search time, since all tiny
160 blocks are exactly the same size. */
161 void* tinyPool
[maxTinyBuffers
];
164 /** Pointer to the data in the tiny pool */
168 CRITICAL_SECTION mutex
;
170 pthread_mutex_t mutex
;
173 /** Provide synchronization between threads */
176 EnterCriticalSection(&mutex
);
178 pthread_mutex_lock(&mutex
);
184 LeaveCriticalSection(&mutex
);
186 pthread_mutex_unlock(&mutex
);
191 Malloc out of the tiny heap.
193 inline void* tinyMalloc(size_t bytes
) {
194 // Note that we ignore the actual byte size
195 // and create a constant size block.
197 debugAssert(tinyBufferSize
>= bytes
);
201 if (tinyPoolSize
> 0) {
203 // Return the last one
204 ptr
= tinyPool
[tinyPoolSize
];
210 /** Returns true if this is a pointer into the tiny heap. */
211 bool inTinyHeap(void* ptr
) {
212 return (ptr
>= tinyHeap
) &&
213 (ptr
< (uint8
*)tinyHeap
+ maxTinyBuffers
* tinyBufferSize
);
216 void tinyFree(void* ptr
) {
217 debugAssert(tinyPoolSize
< maxTinyBuffers
);
219 // Put the pointer back into the free list
220 tinyPool
[tinyPoolSize
] = ptr
;
225 void flushPool(MemBlock
* pool
, int& poolSize
) {
226 for (int i
= 0; i
< poolSize
; ++i
) {
235 /** Allocate out of a specific pool-> Return NULL if no suitable
239 void* malloc(MemBlock
* pool
, int& poolSize
, size_t bytes
) {
241 // OPT: find the smallest block that satisfies the request.
243 // See if there's something we can use in the buffer pool->
244 // Search backwards since usually we'll re-use the last one.
245 for (int i
= (int)poolSize
- 1; i
>= 0; --i
) {
246 if (pool
[i
].bytes
>= bytes
) {
247 // We found a suitable entry in the pool->
249 // No need to offset the pointer; it is already offset
250 void* ptr
= pool
[i
].ptr
;
252 // Remove this element from the pool
254 pool
[i
] = pool
[poolSize
];
265 /** Count of memory allocations that have occurred. */
267 int mallocsFromTinyPool
;
268 int mallocsFromSmallPool
;
269 int mallocsFromMedPool
;
271 /** Amount of memory currently allocated (according to the application).
272 This does not count the memory still remaining in the buffer pool,
273 but does count extra memory required for rounding off to the size
275 Primarily useful for detecting leaks.*/
276 // TODO: make me an atomic int!
282 mallocsFromTinyPool
= 0;
283 mallocsFromSmallPool
= 0;
284 mallocsFromMedPool
= 0;
286 bytesAllocated
= true;
296 // Initialize the tiny heap as a bunch of pointers into one
297 // pre-allocated buffer.
298 tinyHeap
= ::malloc(maxTinyBuffers
* tinyBufferSize
);
299 for (int i
= 0; i
< maxTinyBuffers
; ++i
) {
300 tinyPool
[i
] = (uint8
*)tinyHeap
+ (tinyBufferSize
* i
);
302 tinyPoolSize
= maxTinyBuffers
;
305 InitializeCriticalSection(&mutex
);
307 pthread_mutex_init(&mutex
, NULL
);
315 DeleteCriticalSection(&mutex
);
317 // No destruction on pthreads
322 void* realloc(void* ptr
, size_t bytes
) {
324 return malloc(bytes
);
327 if (inTinyHeap(ptr
)) {
328 if (bytes
<= tinyBufferSize
) {
329 // The old pointer actually had enough space.
332 // Free the old pointer and malloc
334 void* newPtr
= malloc(bytes
);
335 System::memcpy(newPtr
, ptr
, tinyBufferSize
);
341 // In one of our heaps.
343 // See how big the block really was
344 size_t realSize
= ((uint32
*)ptr
)[-1];
345 if (bytes
<= realSize
) {
346 // The old block was big enough.
350 // Need to reallocate
351 void* newPtr
= malloc(bytes
);
352 System::memcpy(newPtr
, ptr
, realSize
);
359 void* malloc(size_t bytes
) {
363 if (bytes
<= tinyBufferSize
) {
365 void* ptr
= tinyMalloc(bytes
);
368 ++mallocsFromTinyPool
;
375 // Failure to allocate a tiny buffer is allowed to flow
376 // through to a small buffer
377 if (bytes
<= smallBufferSize
) {
379 void* ptr
= malloc(smallPool
, smallPoolSize
, bytes
);
382 ++mallocsFromSmallPool
;
387 } else if (bytes
<= medBufferSize
) {
388 // Note that a small allocation failure does *not* fall
389 // through into a medium allocation because that would
390 // waste the medium buffer's resources.
392 void* ptr
= malloc(medPool
, medPoolSize
, bytes
);
395 ++mallocsFromMedPool
;
401 bytesAllocated
+= 4 + (int) bytes
;
406 // Allocate 4 extra bytes for our size header (unfortunate,
407 // since malloc already added its own header).
408 void* ptr
= ::malloc(bytes
+ 4);
411 // Flush memory pools to try and recover space
412 flushPool(smallPool
, smallPoolSize
);
413 flushPool(medPool
, medPoolSize
);
414 ptr
= ::malloc(bytes
+ 4);
419 if ((System::outOfMemoryCallback
!= NULL
) &&
420 (System::outOfMemoryCallback(bytes
+ 4, true) == true)) {
421 // Re-attempt the malloc
422 ptr
= ::malloc(bytes
+ 4);
427 if (System::outOfMemoryCallback
!= NULL
) {
428 // Notify the application
429 System::outOfMemoryCallback(bytes
+ 4, false);
434 *(uint32
*)ptr
= (uint32
)bytes
;
436 return (uint8
*)ptr
+ 4;
440 void free(void* ptr
) {
442 // Free does nothing on null pointers
446 debugAssert(isValidPointer(ptr
));
448 if (inTinyHeap(ptr
)) {
455 uint32 bytes
= ((uint32
*)ptr
)[-1];
458 if (bytes
<= smallBufferSize
) {
459 if (smallPoolSize
< maxSmallBuffers
) {
460 smallPool
[smallPoolSize
] = MemBlock(ptr
, bytes
);
465 } else if (bytes
<= medBufferSize
) {
466 if (medPoolSize
< maxMedBuffers
) {
467 medPool
[medPoolSize
] = MemBlock(ptr
, bytes
);
473 bytesAllocated
-= bytes
+ 4;
476 // Free; the buffer pools are full or this is too big to store.
477 ::free((uint8
*)ptr
- 4);
480 std::string
performance() const {
481 if (totalMallocs
> 0) {
482 int pooled
= mallocsFromTinyPool
+
483 mallocsFromSmallPool
+
486 int total
= totalMallocs
;
488 return format("malloc performance: %5.1f%% <= %db, %5.1f%% <= %db, "
489 "%5.1f%% <= %db, %5.1f%% > %db",
490 100.0 * mallocsFromTinyPool
/ total
,
491 BufferPool::tinyBufferSize
,
492 100.0 * mallocsFromSmallPool
/ total
,
493 BufferPool::smallBufferSize
,
494 100.0 * mallocsFromMedPool
/ total
,
495 BufferPool::medBufferSize
,
496 100.0 * (1.0 - (double)pooled
/ total
),
497 BufferPool::medBufferSize
);
499 return "No System::malloc calls made yet.";
503 std::string
status() const {
504 return format("preallocated shared buffers: %5d/%d x %db",
505 maxTinyBuffers
- tinyPoolSize
, maxTinyBuffers
, tinyBufferSize
);
509 // Dynamically allocated because we need to ensure that
510 // the buffer pool is still around when the last global variable
512 static BufferPool
* bufferpool
= NULL
;
514 std::string
System::mallocPerformance() {
515 #ifndef NO_BUFFERPOOL
516 return bufferpool
->performance();
518 return "NO_BUFFERPOOL";
522 std::string
System::mallocStatus() {
523 #ifndef NO_BUFFERPOOL
524 return bufferpool
->status();
526 return "NO_BUFFERPOOL";
531 void System::resetMallocPerformanceCounters() {
532 #ifndef NO_BUFFERPOOL
533 bufferpool
->totalMallocs
= 0;
534 bufferpool
->mallocsFromMedPool
= 0;
535 bufferpool
->mallocsFromSmallPool
= 0;
536 bufferpool
->mallocsFromTinyPool
= 0;
541 #ifndef NO_BUFFERPOOL
542 inline void initMem() {
543 // Putting the test here ensures that the system is always
544 // initialized, even when globals are being allocated.
545 static bool initialized
= false;
547 bufferpool
= new BufferPool();
554 void* System::malloc(size_t bytes
) {
555 #ifndef NO_BUFFERPOOL
557 return bufferpool
->malloc(bytes
);
559 return ::malloc(bytes
);
563 void* System::calloc(size_t n
, size_t x
) {
564 #ifndef NO_BUFFERPOOL
565 void* b
= System::malloc(n
* x
);
566 System::memset(b
, 0, n
* x
);
569 return ::calloc(n
, x
);
574 void* System::realloc(void* block
, size_t bytes
) {
575 #ifndef NO_BUFFERPOOL
577 return bufferpool
->realloc(block
, bytes
);
579 return ::realloc(block
, bytes
);
584 void System::free(void* p
) {
585 #ifndef NO_BUFFERPOOL
593 void* System::alignedMalloc(size_t bytes
, size_t alignment
) {
594 alwaysAssertM(isPow2(alignment
), "alignment must be a power of 2");
596 // We must align to at least a word boundary.
597 alignment
= iMax((int)alignment
, sizeof(void *));
599 // Pad the allocation size with the alignment size and the
600 // size of the redirect pointer.
601 size_t totalBytes
= bytes
+ alignment
+ sizeof(intptr_t);
603 void* truePtr
= System::malloc(totalBytes
);
606 // malloc returned NULL
610 debugAssert(isValidHeapPointer(truePtr
));
612 // The blocks we return will not be valid Win32 debug heap
613 // pointers because they are offset
614 // debugAssert(_CrtIsValidPointer(truePtr, totalBytes, TRUE) );
617 // The return pointer will be the next aligned location (we must at least
618 // leave space for the redirect pointer, however).
619 char* alignedPtr
= ((char*)truePtr
)+ sizeof(intptr_t);
622 // 2^n - 1 has the form 1111... in binary.
623 uint32 bitMask
= (alignment
- 1);
625 // Advance forward until we reach an aligned location.
626 while ((((intptr_t)alignedPtr
) & bitMask
) != 0) {
627 alignedPtr
+= sizeof(void*);
630 alignedPtr
+= alignment
- (((intptr_t)alignedPtr
) & (alignment
- 1));
631 // assert((alignedPtr - truePtr) + bytes <= totalBytes);
634 debugAssert((alignedPtr
- truePtr
) + bytes
<= totalBytes
);
636 // Immediately before the aligned location, write the true array location
637 // so that we can free it correctly.
638 intptr_t* redirectPtr
= (intptr_t*)(alignedPtr
- sizeof(intptr_t));
639 redirectPtr
[0] = (intptr_t)truePtr
;
641 debugAssert(isValidHeapPointer(truePtr
));
644 debugAssert( _CrtIsValidPointer(alignedPtr
, bytes
, TRUE
) );
646 return (void*)alignedPtr
;
650 void System::alignedFree(void* _ptr
) {
655 char* alignedPtr
= (char*)_ptr
;
657 // Back up one word from the pointer the user passed in.
658 // We now have a pointer to a pointer to the true start
659 // of the memory block.
660 intptr_t* redirectPtr
= (intptr_t*)(alignedPtr
- sizeof(intptr_t));
662 // Dereference that pointer so that ptr = true start
663 void* truePtr
= (void*)(redirectPtr
[0]);
665 debugAssert(isValidHeapPointer(truePtr
));
666 System::free(truePtr
);