4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** This file contains the C functions that implement a memory
13 ** allocation subsystem for use by SQLite.
15 ** This version of the memory allocation subsystem omits all
16 ** use of malloc(). The SQLite user supplies a block of memory
17 ** before calling sqlite3_initialize() from which allocations
18 ** are made and returned by the xMalloc() and xRealloc()
19 ** implementations. Once sqlite3_initialize() has been called,
20 ** the amount of memory available to SQLite is fixed and cannot
23 ** This version of the memory allocation subsystem is included
24 ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
26 #include "sqliteInt.h"
29 ** This version of the memory allocator is only built into the library
30 ** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
31 ** mean that the library will use a memory-pool by default, just that
32 ** it is available. The mempool allocator is activated by calling
35 #ifdef SQLITE_ENABLE_MEMSYS3
38 ** Maximum size (in Mem3Blocks) of a "small" chunk.
44 ** Number of freelist hash slots
49 ** A memory allocation (also called a "chunk") consists of two or
50 ** more blocks where each block is 8 bytes. The first 8 bytes are
51 ** a header that is not returned to the user.
53 ** A chunk is two or more blocks that is either checked out or
54 ** free. The first block has format u.hdr. u.hdr.size4x is 4 times the
55 ** size of the allocation in blocks if the allocation is free.
56 ** The u.hdr.size4x&1 bit is true if the chunk is checked out and
57 ** false if the chunk is on the freelist. The u.hdr.size4x&2 bit
58 ** is true if the previous chunk is checked out and false if the
59 ** previous chunk is free. The u.hdr.prevSize field is the size of
60 ** the previous chunk in blocks if the previous chunk is on the
61 ** freelist. If the previous chunk is checked out, then
62 ** u.hdr.prevSize can be part of the data for that chunk and should
63 ** not be read or written.
65 ** We often identify a chunk by its index in mem3.aPool[]. When
66 ** this is done, the chunk index refers to the second block of
67 ** the chunk. In this way, the first chunk has an index of 1.
68 ** A chunk index of 0 means "no such chunk" and is the equivalent
71 ** The second block of free chunks is of the form u.list. The
72 ** two fields form a double-linked list of chunks of related sizes.
73 ** Pointers to the head of the list are stored in mem3.aiSmall[]
74 ** for smaller chunks and mem3.aiHash[] for larger chunks.
76 ** The second block of a chunk is user data if the chunk is checked
77 ** out. If a chunk is checked out, the user data may extend into
78 ** the u.hdr.prevSize value of the following chunk.
80 typedef struct Mem3Block Mem3Block
;
84 u32 prevSize
; /* Size of previous chunk in Mem3Block elements */
85 u32 size4x
; /* 4x the size of current chunk in Mem3Block elements */
88 u32 next
; /* Index in mem3.aPool[] of next free chunk */
89 u32 prev
; /* Index in mem3.aPool[] of previous free chunk */
95 ** All of the static variables used by this module are collected
96 ** into a single structure named "mem3". This is to keep the
97 ** static variables organized and to reduce namespace pollution
98 ** when this module is combined with other in the amalgamation.
100 static SQLITE_WSD
struct Mem3Global
{
102 ** Memory available for allocation. nPool is the size of the array
103 ** (in Mem3Blocks) pointed to by aPool less 2.
109 ** True if we are evaluating an out-of-memory callback.
114 ** Mutex to control access to the memory allocation subsystem.
116 sqlite3_mutex
*mutex
;
119 ** The minimum amount of free space that we have seen.
124 ** iKeyBlk is the index of the key chunk. Most new allocations
125 ** occur off of this chunk. szKeyBlk is the size (in Mem3Blocks)
126 ** of the current key chunk. iKeyBlk is 0 if there is no key chunk.
127 ** The key chunk is not in either the aiHash[] or aiSmall[].
133 ** Array of lists of free blocks according to the block size
134 ** for smaller chunks, or a hash on the block size for larger
137 u32 aiSmall
[MX_SMALL
-1]; /* For sizes 2 through MX_SMALL, inclusive */
138 u32 aiHash
[N_HASH
]; /* For sizes MX_SMALL+1 and larger */
139 } mem3
= { 97535575 };
141 #define mem3 GLOBAL(struct Mem3Global, mem3)
144 ** Unlink the chunk at mem3.aPool[i] from list it is currently
145 ** on. *pRoot is the list that i is a member of.
147 static void memsys3UnlinkFromList(u32 i
, u32
*pRoot
){
148 u32 next
= mem3
.aPool
[i
].u
.list
.next
;
149 u32 prev
= mem3
.aPool
[i
].u
.list
.prev
;
150 assert( sqlite3_mutex_held(mem3
.mutex
) );
154 mem3
.aPool
[prev
].u
.list
.next
= next
;
157 mem3
.aPool
[next
].u
.list
.prev
= prev
;
159 mem3
.aPool
[i
].u
.list
.next
= 0;
160 mem3
.aPool
[i
].u
.list
.prev
= 0;
164 ** Unlink the chunk at index i from
165 ** whatever list is currently a member of.
167 static void memsys3Unlink(u32 i
){
169 assert( sqlite3_mutex_held(mem3
.mutex
) );
170 assert( (mem3
.aPool
[i
-1].u
.hdr
.size4x
& 1)==0 );
172 size
= mem3
.aPool
[i
-1].u
.hdr
.size4x
/4;
173 assert( size
==mem3
.aPool
[i
+size
-1].u
.hdr
.prevSize
);
175 if( size
<= MX_SMALL
){
176 memsys3UnlinkFromList(i
, &mem3
.aiSmall
[size
-2]);
178 hash
= size
% N_HASH
;
179 memsys3UnlinkFromList(i
, &mem3
.aiHash
[hash
]);
184 ** Link the chunk at mem3.aPool[i] so that is on the list rooted
187 static void memsys3LinkIntoList(u32 i
, u32
*pRoot
){
188 assert( sqlite3_mutex_held(mem3
.mutex
) );
189 mem3
.aPool
[i
].u
.list
.next
= *pRoot
;
190 mem3
.aPool
[i
].u
.list
.prev
= 0;
192 mem3
.aPool
[*pRoot
].u
.list
.prev
= i
;
198 ** Link the chunk at index i into either the appropriate
199 ** small chunk list, or into the large chunk hash table.
201 static void memsys3Link(u32 i
){
203 assert( sqlite3_mutex_held(mem3
.mutex
) );
205 assert( (mem3
.aPool
[i
-1].u
.hdr
.size4x
& 1)==0 );
206 size
= mem3
.aPool
[i
-1].u
.hdr
.size4x
/4;
207 assert( size
==mem3
.aPool
[i
+size
-1].u
.hdr
.prevSize
);
209 if( size
<= MX_SMALL
){
210 memsys3LinkIntoList(i
, &mem3
.aiSmall
[size
-2]);
212 hash
= size
% N_HASH
;
213 memsys3LinkIntoList(i
, &mem3
.aiHash
[hash
]);
218 ** If the STATIC_MEM mutex is not already held, obtain it now. The mutex
219 ** will already be held (obtained by code in malloc.c) if
220 ** sqlite3GlobalConfig.bMemStat is true.
222 static void memsys3Enter(void){
223 if( sqlite3GlobalConfig
.bMemstat
==0 && mem3
.mutex
==0 ){
224 mem3
.mutex
= sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM
);
226 sqlite3_mutex_enter(mem3
.mutex
);
228 static void memsys3Leave(void){
229 sqlite3_mutex_leave(mem3
.mutex
);
233 ** Called when we are unable to satisfy an allocation of nBytes.
235 static void memsys3OutOfMemory(int nByte
){
236 if( !mem3
.alarmBusy
){
238 assert( sqlite3_mutex_held(mem3
.mutex
) );
239 sqlite3_mutex_leave(mem3
.mutex
);
240 sqlite3_release_memory(nByte
);
241 sqlite3_mutex_enter(mem3
.mutex
);
248 ** Chunk i is a free chunk that has been unlinked. Adjust its
249 ** size parameters for check-out and return a pointer to the
250 ** user portion of the chunk.
252 static void *memsys3Checkout(u32 i
, u32 nBlock
){
254 assert( sqlite3_mutex_held(mem3
.mutex
) );
256 assert( mem3
.aPool
[i
-1].u
.hdr
.size4x
/4==nBlock
);
257 assert( mem3
.aPool
[i
+nBlock
-1].u
.hdr
.prevSize
==nBlock
);
258 x
= mem3
.aPool
[i
-1].u
.hdr
.size4x
;
259 mem3
.aPool
[i
-1].u
.hdr
.size4x
= nBlock
*4 | 1 | (x
&2);
260 mem3
.aPool
[i
+nBlock
-1].u
.hdr
.prevSize
= nBlock
;
261 mem3
.aPool
[i
+nBlock
-1].u
.hdr
.size4x
|= 2;
262 return &mem3
.aPool
[i
];
266 ** Carve a piece off of the end of the mem3.iKeyBlk free chunk.
267 ** Return a pointer to the new allocation. Or, if the key chunk
268 ** is not large enough, return 0.
270 static void *memsys3FromKeyBlk(u32 nBlock
){
271 assert( sqlite3_mutex_held(mem3
.mutex
) );
272 assert( mem3
.szKeyBlk
>=nBlock
);
273 if( nBlock
>=mem3
.szKeyBlk
-1 ){
274 /* Use the entire key chunk */
275 void *p
= memsys3Checkout(mem3
.iKeyBlk
, mem3
.szKeyBlk
);
281 /* Split the key block. Return the tail. */
283 newi
= mem3
.iKeyBlk
+ mem3
.szKeyBlk
- nBlock
;
284 assert( newi
> mem3
.iKeyBlk
+1 );
285 mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.prevSize
= nBlock
;
286 mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.size4x
|= 2;
287 mem3
.aPool
[newi
-1].u
.hdr
.size4x
= nBlock
*4 + 1;
288 mem3
.szKeyBlk
-= nBlock
;
289 mem3
.aPool
[newi
-1].u
.hdr
.prevSize
= mem3
.szKeyBlk
;
290 x
= mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
& 2;
291 mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
= mem3
.szKeyBlk
*4 | x
;
292 if( mem3
.szKeyBlk
< mem3
.mnKeyBlk
){
293 mem3
.mnKeyBlk
= mem3
.szKeyBlk
;
295 return (void*)&mem3
.aPool
[newi
];
300 ** *pRoot is the head of a list of free chunks of the same size
301 ** or same size hash. In other words, *pRoot is an entry in either
302 ** mem3.aiSmall[] or mem3.aiHash[].
304 ** This routine examines all entries on the given list and tries
305 ** to coalesce each entries with adjacent free chunks.
307 ** If it sees a chunk that is larger than mem3.iKeyBlk, it replaces
308 ** the current mem3.iKeyBlk with the new larger chunk. In order for
309 ** this mem3.iKeyBlk replacement to work, the key chunk must be
310 ** linked into the hash tables. That is not the normal state of
311 ** affairs, of course. The calling routine must link the key
312 ** chunk before invoking this routine, then must unlink the (possibly
313 ** changed) key chunk once this routine has finished.
315 static void memsys3Merge(u32
*pRoot
){
316 u32 iNext
, prev
, size
, i
, x
;
318 assert( sqlite3_mutex_held(mem3
.mutex
) );
319 for(i
=*pRoot
; i
>0; i
=iNext
){
320 iNext
= mem3
.aPool
[i
].u
.list
.next
;
321 size
= mem3
.aPool
[i
-1].u
.hdr
.size4x
;
322 assert( (size
&1)==0 );
324 memsys3UnlinkFromList(i
, pRoot
);
325 assert( i
> mem3
.aPool
[i
-1].u
.hdr
.prevSize
);
326 prev
= i
- mem3
.aPool
[i
-1].u
.hdr
.prevSize
;
328 iNext
= mem3
.aPool
[prev
].u
.list
.next
;
331 size
= i
+ size
/4 - prev
;
332 x
= mem3
.aPool
[prev
-1].u
.hdr
.size4x
& 2;
333 mem3
.aPool
[prev
-1].u
.hdr
.size4x
= size
*4 | x
;
334 mem3
.aPool
[prev
+size
-1].u
.hdr
.prevSize
= size
;
340 if( size
>mem3
.szKeyBlk
){
342 mem3
.szKeyBlk
= size
;
348 ** Return a block of memory of at least nBytes in size.
349 ** Return NULL if unable.
351 ** This function assumes that the necessary mutexes, if any, are
352 ** already held by the caller. Hence "Unsafe".
354 static void *memsys3MallocUnsafe(int nByte
){
359 assert( sqlite3_mutex_held(mem3
.mutex
) );
360 assert( sizeof(Mem3Block
)==8 );
364 nBlock
= (nByte
+ 11)/8;
369 ** Look for an entry of the correct size in either the small
370 ** chunk table or in the large chunk hash table. This is
371 ** successful most of the time (about 9 times out of 10).
373 if( nBlock
<= MX_SMALL
){
374 i
= mem3
.aiSmall
[nBlock
-2];
376 memsys3UnlinkFromList(i
, &mem3
.aiSmall
[nBlock
-2]);
377 return memsys3Checkout(i
, nBlock
);
380 int hash
= nBlock
% N_HASH
;
381 for(i
=mem3
.aiHash
[hash
]; i
>0; i
=mem3
.aPool
[i
].u
.list
.next
){
382 if( mem3
.aPool
[i
-1].u
.hdr
.size4x
/4==nBlock
){
383 memsys3UnlinkFromList(i
, &mem3
.aiHash
[hash
]);
384 return memsys3Checkout(i
, nBlock
);
390 ** Try to satisfy the allocation by carving a piece off of the end
391 ** of the key chunk. This step usually works if step 1 fails.
393 if( mem3
.szKeyBlk
>=nBlock
){
394 return memsys3FromKeyBlk(nBlock
);
399 ** Loop through the entire memory pool. Coalesce adjacent free
400 ** chunks. Recompute the key chunk as the largest free chunk.
401 ** Then try again to satisfy the allocation by carving a piece off
402 ** of the end of the key chunk. This step happens very
405 for(toFree
=nBlock
*16; toFree
<(mem3
.nPool
*16); toFree
*= 2){
406 memsys3OutOfMemory(toFree
);
408 memsys3Link(mem3
.iKeyBlk
);
412 for(i
=0; i
<N_HASH
; i
++){
413 memsys3Merge(&mem3
.aiHash
[i
]);
415 for(i
=0; i
<MX_SMALL
-1; i
++){
416 memsys3Merge(&mem3
.aiSmall
[i
]);
419 memsys3Unlink(mem3
.iKeyBlk
);
420 if( mem3
.szKeyBlk
>=nBlock
){
421 return memsys3FromKeyBlk(nBlock
);
426 /* If none of the above worked, then we fail. */
431 ** Free an outstanding memory allocation.
433 ** This function assumes that the necessary mutexes, if any, are
434 ** already held by the caller. Hence "Unsafe".
436 static void memsys3FreeUnsafe(void *pOld
){
437 Mem3Block
*p
= (Mem3Block
*)pOld
;
440 assert( sqlite3_mutex_held(mem3
.mutex
) );
441 assert( p
>mem3
.aPool
&& p
<&mem3
.aPool
[mem3
.nPool
] );
443 assert( (mem3
.aPool
[i
-1].u
.hdr
.size4x
&1)==1 );
444 size
= mem3
.aPool
[i
-1].u
.hdr
.size4x
/4;
445 assert( i
+size
<=mem3
.nPool
+1 );
446 mem3
.aPool
[i
-1].u
.hdr
.size4x
&= ~1;
447 mem3
.aPool
[i
+size
-1].u
.hdr
.prevSize
= size
;
448 mem3
.aPool
[i
+size
-1].u
.hdr
.size4x
&= ~2;
451 /* Try to expand the key using the newly freed chunk */
453 while( (mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
&2)==0 ){
454 size
= mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.prevSize
;
455 mem3
.iKeyBlk
-= size
;
456 mem3
.szKeyBlk
+= size
;
457 memsys3Unlink(mem3
.iKeyBlk
);
458 x
= mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
& 2;
459 mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
= mem3
.szKeyBlk
*4 | x
;
460 mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.prevSize
= mem3
.szKeyBlk
;
462 x
= mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
& 2;
463 while( (mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.size4x
&1)==0 ){
464 memsys3Unlink(mem3
.iKeyBlk
+mem3
.szKeyBlk
);
465 mem3
.szKeyBlk
+= mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.size4x
/4;
466 mem3
.aPool
[mem3
.iKeyBlk
-1].u
.hdr
.size4x
= mem3
.szKeyBlk
*4 | x
;
467 mem3
.aPool
[mem3
.iKeyBlk
+mem3
.szKeyBlk
-1].u
.hdr
.prevSize
= mem3
.szKeyBlk
;
473 ** Return the size of an outstanding allocation, in bytes. The
474 ** size returned omits the 8-byte header overhead. This only
475 ** works for chunks that are currently checked out.
477 static int memsys3Size(void *p
){
480 pBlock
= (Mem3Block
*)p
;
481 assert( (pBlock
[-1].u
.hdr
.size4x
&1)!=0 );
482 return (pBlock
[-1].u
.hdr
.size4x
&~3)*2 - 4;
486 ** Round up a request size to the next valid allocation size.
488 static int memsys3Roundup(int n
){
492 return ((n
+11)&~7) - 4;
497 ** Allocate nBytes of memory.
499 static void *memsys3Malloc(int nBytes
){
501 assert( nBytes
>0 ); /* malloc.c filters out 0 byte requests */
503 p
= memsys3MallocUnsafe(nBytes
);
511 static void memsys3Free(void *pPrior
){
514 memsys3FreeUnsafe(pPrior
);
519 ** Change the size of an existing memory allocation
521 static void *memsys3Realloc(void *pPrior
, int nBytes
){
525 return sqlite3_malloc(nBytes
);
528 sqlite3_free(pPrior
);
531 nOld
= memsys3Size(pPrior
);
532 if( nBytes
<=nOld
&& nBytes
>=nOld
-128 ){
536 p
= memsys3MallocUnsafe(nBytes
);
539 memcpy(p
, pPrior
, nOld
);
541 memcpy(p
, pPrior
, nBytes
);
543 memsys3FreeUnsafe(pPrior
);
550 ** Initialize this module.
552 static int memsys3Init(void *NotUsed
){
553 UNUSED_PARAMETER(NotUsed
);
554 if( !sqlite3GlobalConfig
.pHeap
){
558 /* Store a pointer to the memory block in global structure mem3. */
559 assert( sizeof(Mem3Block
)==8 );
560 mem3
.aPool
= (Mem3Block
*)sqlite3GlobalConfig
.pHeap
;
561 mem3
.nPool
= (sqlite3GlobalConfig
.nHeap
/ sizeof(Mem3Block
)) - 2;
563 /* Initialize the key block. */
564 mem3
.szKeyBlk
= mem3
.nPool
;
565 mem3
.mnKeyBlk
= mem3
.szKeyBlk
;
567 mem3
.aPool
[0].u
.hdr
.size4x
= (mem3
.szKeyBlk
<<2) + 2;
568 mem3
.aPool
[mem3
.nPool
].u
.hdr
.prevSize
= mem3
.nPool
;
569 mem3
.aPool
[mem3
.nPool
].u
.hdr
.size4x
= 1;
575 ** Deinitialize this module.
577 static void memsys3Shutdown(void *NotUsed
){
578 UNUSED_PARAMETER(NotUsed
);
586 ** Open the file indicated and write a log of all unfreed memory
587 ** allocations into that log.
589 void sqlite3Memsys3Dump(const char *zFilename
){
594 if( zFilename
==0 || zFilename
[0]==0 ){
597 out
= fopen(zFilename
, "w");
599 fprintf(stderr
, "** Unable to output memory debug output log: %s **\n",
605 fprintf(out
, "CHUNKS:\n");
606 for(i
=1; i
<=mem3
.nPool
; i
+=size
/4){
607 size
= mem3
.aPool
[i
-1].u
.hdr
.size4x
;
609 fprintf(out
, "%p size error\n", &mem3
.aPool
[i
]);
613 if( (size
&1)==0 && mem3
.aPool
[i
+size
/4-1].u
.hdr
.prevSize
!=size
/4 ){
614 fprintf(out
, "%p tail size does not match\n", &mem3
.aPool
[i
]);
618 if( ((mem3
.aPool
[i
+size
/4-1].u
.hdr
.size4x
&2)>>1)!=(size
&1) ){
619 fprintf(out
, "%p tail checkout bit is incorrect\n", &mem3
.aPool
[i
]);
624 fprintf(out
, "%p %6d bytes checked out\n", &mem3
.aPool
[i
], (size
/4)*8-8);
626 fprintf(out
, "%p %6d bytes free%s\n", &mem3
.aPool
[i
], (size
/4)*8-8,
627 i
==mem3
.iKeyBlk
? " **key**" : "");
630 for(i
=0; i
<MX_SMALL
-1; i
++){
631 if( mem3
.aiSmall
[i
]==0 ) continue;
632 fprintf(out
, "small(%2d):", i
);
633 for(j
= mem3
.aiSmall
[i
]; j
>0; j
=mem3
.aPool
[j
].u
.list
.next
){
634 fprintf(out
, " %p(%d)", &mem3
.aPool
[j
],
635 (mem3
.aPool
[j
-1].u
.hdr
.size4x
/4)*8-8);
639 for(i
=0; i
<N_HASH
; i
++){
640 if( mem3
.aiHash
[i
]==0 ) continue;
641 fprintf(out
, "hash(%2d):", i
);
642 for(j
= mem3
.aiHash
[i
]; j
>0; j
=mem3
.aPool
[j
].u
.list
.next
){
643 fprintf(out
, " %p(%d)", &mem3
.aPool
[j
],
644 (mem3
.aPool
[j
-1].u
.hdr
.size4x
/4)*8-8);
648 fprintf(out
, "key=%d\n", mem3
.iKeyBlk
);
649 fprintf(out
, "nowUsed=%d\n", mem3
.nPool
*8 - mem3
.szKeyBlk
*8);
650 fprintf(out
, "mxUsed=%d\n", mem3
.nPool
*8 - mem3
.mnKeyBlk
*8);
651 sqlite3_mutex_leave(mem3
.mutex
);
658 UNUSED_PARAMETER(zFilename
);
663 ** This routine is the only routine in this file with external
666 ** Populate the low-level memory allocation function pointers in
667 ** sqlite3GlobalConfig.m with pointers to the routines in this file. The
668 ** arguments specify the block of memory to manage.
670 ** This routine is only called by sqlite3_config(), and therefore
671 ** is not required to be threadsafe (it is not).
673 const sqlite3_mem_methods
*sqlite3MemGetMemsys3(void){
674 static const sqlite3_mem_methods mempoolMethods
= {
684 return &mempoolMethods
;
687 #endif /* SQLITE_ENABLE_MEMSYS3 */