src/malloc.c

   1 /*
   2 ** 2001 September 15
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 *************************************************************************
  12 **
  13 ** Memory allocation functions used throughout sqlite.
  14 */
  15 #include "sqliteInt.h"
  16 #include <stdarg.h>
  17
  18 /*
  19 ** Attempt to release up to n bytes of non-essential memory currently
  20 ** held by SQLite. An example of non-essential memory is memory used to
  21 ** cache database pages that are not currently in use.
  22 */
  23 int sqlite3_release_memory(int n){
  24 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  25   return sqlite3PcacheReleaseMemory(n);
  26 #else
  27   /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine
  28   ** is a no-op returning zero if SQLite is not compiled with
  29   ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */
  30   UNUSED_PARAMETER(n);
  31   return 0;
  32 #endif
  33 }
  34
  35 /*
  36 ** Default value of the hard heap limit.  0 means "no limit".
  37 */
  38 #ifndef SQLITE_MAX_MEMORY
  39 # define SQLITE_MAX_MEMORY 0
  40 #endif
  41
  42 /*
  43 ** State information local to the memory allocation subsystem.
  44 */
  45 static SQLITE_WSD struct Mem0Global {
  46   sqlite3_mutex *mutex;         /* Mutex to serialize access */
  47   sqlite3_int64 alarmThreshold; /* The soft heap limit */
  48   sqlite3_int64 hardLimit;      /* The hard upper bound on memory */
  49
  50   /*
  51   ** True if heap is nearly "full" where "full" is defined by the
  52   ** sqlite3_soft_heap_limit() setting.
  53   */
  54   int nearlyFull;
  55 } mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 };
  56
  57 #define mem0 GLOBAL(struct Mem0Global, mem0)
  58
  59 /*
  60 ** Return the memory allocator mutex. sqlite3_status() needs it.
  61 */
  62 sqlite3_mutex *sqlite3MallocMutex(void){
  63   return mem0.mutex;
  64 }
  65
  66 #ifndef SQLITE_OMIT_DEPRECATED
  67 /*
  68 ** Deprecated external interface.  It used to set an alarm callback
  69 ** that was invoked when memory usage grew too large.  Now it is a
  70 ** no-op.
  71 */
  72 int sqlite3_memory_alarm(
  73   void(*xCallback)(void *pArg, sqlite3_int64 used,int N),
  74   void *pArg,
  75   sqlite3_int64 iThreshold
  76 ){
  77   (void)xCallback;
  78   (void)pArg;
  79   (void)iThreshold;
  80   return SQLITE_OK;
  81 }
  82 #endif
  83
  84 /*
  85 ** Set the soft heap-size limit for the library.  An argument of
  86 ** zero disables the limit.  A negative argument is a no-op used to
  87 ** obtain the return value.
  88 **
  89 ** The return value is the value of the heap limit just before this
  90 ** interface was called.
  91 **
  92 ** If the hard heap limit is enabled, then the soft heap limit cannot
  93 ** be disabled nor raised above the hard heap limit.
  94 */
  95 sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){
  96   sqlite3_int64 priorLimit;
  97   sqlite3_int64 excess;
  98   sqlite3_int64 nUsed;
  99 #ifndef SQLITE_OMIT_AUTOINIT
 100   int rc = sqlite3_initialize();
 101   if( rc ) return -1;
 102 #endif
 103   sqlite3_mutex_enter(mem0.mutex);
 104   priorLimit = mem0.alarmThreshold;
 105   if( n<0 ){
 106     sqlite3_mutex_leave(mem0.mutex);
 107     return priorLimit;
 108   }
 109   if( mem0.hardLimit>0 && (n>mem0.hardLimit || n==0) ){
 110     n = mem0.hardLimit;
 111   }
 112   mem0.alarmThreshold = n;
 113   nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
 114   AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed);
 115   sqlite3_mutex_leave(mem0.mutex);
 116   excess = sqlite3_memory_used() - n;
 117   if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff));
 118   return priorLimit;
 119 }
 120 void sqlite3_soft_heap_limit(int n){
 121   if( n<0 ) n = 0;
 122   sqlite3_soft_heap_limit64(n);
 123 }
 124
 125 /*
 126 ** Set the hard heap-size limit for the library. An argument of zero
 127 ** disables the hard heap limit.  A negative argument is a no-op used
 128 ** to obtain the return value without affecting the hard heap limit.
 129 **
 130 ** The return value is the value of the hard heap limit just prior to
 131 ** calling this interface.
 132 **
 133 ** Setting the hard heap limit will also activate the soft heap limit
 134 ** and constrain the soft heap limit to be no more than the hard heap
 135 ** limit.
 136 */
 137 sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){
 138   sqlite3_int64 priorLimit;
 139 #ifndef SQLITE_OMIT_AUTOINIT
 140   int rc = sqlite3_initialize();
 141   if( rc ) return -1;
 142 #endif
 143   sqlite3_mutex_enter(mem0.mutex);
 144   priorLimit = mem0.hardLimit;
 145   if( n>=0 ){
 146     mem0.hardLimit = n;
 147     if( n<mem0.alarmThreshold || mem0.alarmThreshold==0 ){
 148       mem0.alarmThreshold = n;
 149     }
 150   }
 151   sqlite3_mutex_leave(mem0.mutex);
 152   return priorLimit;
 153 }
 154
 155
 156 /*
 157 ** Initialize the memory allocation subsystem.
 158 */
 159 int sqlite3MallocInit(void){
 160   int rc;
 161   if( sqlite3GlobalConfig.m.xMalloc==0 ){
 162     sqlite3MemSetDefault();
 163   }
 164   memset(&mem0, 0, sizeof(mem0));
 165   mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM);
 166   if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512
 167       || sqlite3GlobalConfig.nPage<=0 ){
 168     sqlite3GlobalConfig.pPage = 0;
 169     sqlite3GlobalConfig.szPage = 0;
 170   }
 171   rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData);
 172   if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0));
 173 /* BEGIN SQLCIPHER */
 174 #ifdef SQLITE_HAS_CODEC
 175   /* install wrapping functions for memory management
 176      that will wipe all memory allocated by SQLite
 177      when freed */
 178   if( rc==SQLITE_OK ) {
 179     extern void sqlcipher_init_memmethods(void);
 180     sqlcipher_init_memmethods();
 181   }
 182 #endif
 183 /* END SQLCIPHER */
 184   return rc;
 185 }
 186
 187 /*
 188 ** Return true if the heap is currently under memory pressure - in other
 189 ** words if the amount of heap used is close to the limit set by
 190 ** sqlite3_soft_heap_limit().
 191 */
 192 int sqlite3HeapNearlyFull(void){
 193   return AtomicLoad(&mem0.nearlyFull);
 194 }
 195
 196 /*
 197 ** Deinitialize the memory allocation subsystem.
 198 */
 199 void sqlite3MallocEnd(void){
 200   if( sqlite3GlobalConfig.m.xShutdown ){
 201     sqlite3GlobalConfig.m.xShutdown(sqlite3GlobalConfig.m.pAppData);
 202   }
 203   memset(&mem0, 0, sizeof(mem0));
 204 }
 205
 206 /*
 207 ** Return the amount of memory currently checked out.
 208 */
 209 sqlite3_int64 sqlite3_memory_used(void){
 210   sqlite3_int64 res, mx;
 211   sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0);
 212   return res;
 213 }
 214
 215 /*
 216 ** Return the maximum amount of memory that has ever been
 217 ** checked out since either the beginning of this process
 218 ** or since the most recent reset.
 219 */
 220 sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
 221   sqlite3_int64 res, mx;
 222   sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag);
 223   return mx;
 224 }
 225
 226 /*
 227 ** Trigger the alarm
 228 */
 229 static void sqlite3MallocAlarm(int nByte){
 230   if( mem0.alarmThreshold<=0 ) return;
 231   sqlite3_mutex_leave(mem0.mutex);
 232   sqlite3_release_memory(nByte);
 233   sqlite3_mutex_enter(mem0.mutex);
 234 }
 235
 236 /*
 237 ** Do a memory allocation with statistics and alarms.  Assume the
 238 ** lock is already held.
 239 */
 240 static void mallocWithAlarm(int n, void **pp){
 241   void *p;
 242   int nFull;
 243   assert( sqlite3_mutex_held(mem0.mutex) );
 244   assert( n>0 );
 245
 246   /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal
 247   ** implementation of malloc_good_size(), which must be called in debug
 248   ** mode and specifically when the DMD "Dark Matter Detector" is enabled
 249   ** or else a crash results.  Hence, do not attempt to optimize out the
 250   ** following xRoundup() call. */
 251   nFull = sqlite3GlobalConfig.m.xRoundup(n);
 252
 253   sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n);
 254   if( mem0.alarmThreshold>0 ){
 255     sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
 256     if( nUsed >= mem0.alarmThreshold - nFull ){
 257       AtomicStore(&mem0.nearlyFull, 1);
 258       sqlite3MallocAlarm(nFull);
 259       if( mem0.hardLimit ){
 260         nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
 261         if( nUsed >= mem0.hardLimit - nFull ){
 262           *pp = 0;
 263           return;
 264         }
 265       }
 266     }else{
 267       AtomicStore(&mem0.nearlyFull, 0);
 268     }
 269   }
 270   p = sqlite3GlobalConfig.m.xMalloc(nFull);
 271 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
 272   if( p==0 && mem0.alarmThreshold>0 ){
 273     sqlite3MallocAlarm(nFull);
 274     p = sqlite3GlobalConfig.m.xMalloc(nFull);
 275   }
 276 #endif
 277   if( p ){
 278     nFull = sqlite3MallocSize(p);
 279     sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull);
 280     sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1);
 281   }
 282   *pp = p;
 283 }
 284
 285 /*
 286 ** Allocate memory.  This routine is like sqlite3_malloc() except that it
 287 ** assumes the memory subsystem has already been initialized.
 288 */
 289 void *sqlite3Malloc(u64 n){
 290   void *p;
 291   if( n==0 || n>=0x7fffff00 ){
 292     /* A memory allocation of a number of bytes which is near the maximum
 293     ** signed integer value might cause an integer overflow inside of the
 294     ** xMalloc().  Hence we limit the maximum size to 0x7fffff00, giving
 295     ** 255 bytes of overhead.  SQLite itself will never use anything near
 296     ** this amount.  The only way to reach the limit is with sqlite3_malloc() */
 297     p = 0;
 298   }else if( sqlite3GlobalConfig.bMemstat ){
 299     sqlite3_mutex_enter(mem0.mutex);
 300     mallocWithAlarm((int)n, &p);
 301     sqlite3_mutex_leave(mem0.mutex);
 302   }else{
 303     p = sqlite3GlobalConfig.m.xMalloc((int)n);
 304   }
 305   assert( EIGHT_BYTE_ALIGNMENT(p) );  /* IMP: R-11148-40995 */
 306   return p;
 307 }
 308
 309 /*
 310 ** This version of the memory allocation is for use by the application.
 311 ** First make sure the memory subsystem is initialized, then do the
 312 ** allocation.
 313 */
 314 void *sqlite3_malloc(int n){
 315 #ifndef SQLITE_OMIT_AUTOINIT
 316   if( sqlite3_initialize() ) return 0;
 317 #endif
 318   return n<=0 ? 0 : sqlite3Malloc(n);
 319 }
 320 void *sqlite3_malloc64(sqlite3_uint64 n){
 321 #ifndef SQLITE_OMIT_AUTOINIT
 322   if( sqlite3_initialize() ) return 0;
 323 #endif
 324   return sqlite3Malloc(n);
 325 }
 326
 327 /*
 328 ** TRUE if p is a lookaside memory allocation from db
 329 */
 330 #ifndef SQLITE_OMIT_LOOKASIDE
 331 static int isLookaside(sqlite3 *db, void *p){
 332   return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd);
 333 }
 334 #else
 335 #define isLookaside(A,B) 0
 336 #endif
 337
 338 /*
 339 ** Return the size of a memory allocation previously obtained from
 340 ** sqlite3Malloc() or sqlite3_malloc().
 341 */
 342 int sqlite3MallocSize(void *p){
 343   assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
 344   return sqlite3GlobalConfig.m.xSize(p);
 345 }
 346 static int lookasideMallocSize(sqlite3 *db, void *p){
 347 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 348   return p<db->lookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL;
 349 #else
 350   return db->lookaside.szTrue;
 351 #endif
 352 }
 353 int sqlite3DbMallocSize(sqlite3 *db, void *p){
 354   assert( p!=0 );
 355 #ifdef SQLITE_DEBUG
 356   if( db==0 || !isLookaside(db,p) ){
 357     if( db==0 ){
 358       assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
 359       assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
 360     }else{
 361       assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 362       assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 363     }
 364   }
 365 #endif
 366   if( db ){
 367     if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
 368 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 369       if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
 370         assert( sqlite3_mutex_held(db->mutex) );
 371         return LOOKASIDE_SMALL;
 372       }
 373 #endif
 374       if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
 375         assert( sqlite3_mutex_held(db->mutex) );
 376         return db->lookaside.szTrue;
 377       }
 378     }
 379   }
 380   return sqlite3GlobalConfig.m.xSize(p);
 381 }
 382 sqlite3_uint64 sqlite3_msize(void *p){
 383   assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
 384   assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
 385   return p ? sqlite3GlobalConfig.m.xSize(p) : 0;
 386 }
 387
 388 /*
 389 ** Free memory previously obtained from sqlite3Malloc().
 390 */
 391 void sqlite3_free(void *p){
 392   if( p==0 ) return;  /* IMP: R-49053-54554 */
 393   assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
 394   assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
 395   if( sqlite3GlobalConfig.bMemstat ){
 396     sqlite3_mutex_enter(mem0.mutex);
 397     sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, sqlite3MallocSize(p));
 398     sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1);
 399     sqlite3GlobalConfig.m.xFree(p);
 400     sqlite3_mutex_leave(mem0.mutex);
 401   }else{
 402     sqlite3GlobalConfig.m.xFree(p);
 403   }
 404 }
 405
 406 /*
 407 ** Add the size of memory allocation "p" to the count in
 408 ** *db->pnBytesFreed.
 409 */
 410 static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){
 411   *db->pnBytesFreed += sqlite3DbMallocSize(db,p);
 412 }
 413
 414 /*
 415 ** Free memory that might be associated with a particular database
 416 ** connection.  Calling sqlite3DbFree(D,X) for X==0 is a harmless no-op.
 417 ** The sqlite3DbFreeNN(D,X) version requires that X be non-NULL.
 418 */
 419 void sqlite3DbFreeNN(sqlite3 *db, void *p){
 420   assert( db==0 || sqlite3_mutex_held(db->mutex) );
 421   assert( p!=0 );
 422   if( db ){
 423     if( db->pnBytesFreed ){
 424       measureAllocationSize(db, p);
 425       return;
 426     }
 427     if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
 428 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 429       if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
 430         LookasideSlot *pBuf = (LookasideSlot*)p;
 431 #ifdef SQLITE_DEBUG
 432         memset(p, 0xaa, LOOKASIDE_SMALL);  /* Trash freed content */
 433 #endif
 434         pBuf->pNext = db->lookaside.pSmallFree;
 435         db->lookaside.pSmallFree = pBuf;
 436         return;
 437       }
 438 #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
 439       if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
 440         LookasideSlot *pBuf = (LookasideSlot*)p;
 441 #ifdef SQLITE_DEBUG
 442         memset(p, 0xaa, db->lookaside.szTrue);  /* Trash freed content */
 443 #endif
 444         pBuf->pNext = db->lookaside.pFree;
 445         db->lookaside.pFree = pBuf;
 446         return;
 447       }
 448     }
 449   }
 450   assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 451   assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 452   assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) );
 453   sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
 454   sqlite3_free(p);
 455 }
 456 void sqlite3DbFree(sqlite3 *db, void *p){
 457   assert( db==0 || sqlite3_mutex_held(db->mutex) );
 458   if( p ) sqlite3DbFreeNN(db, p);
 459 }
 460
 461 /*
 462 ** Change the size of an existing memory allocation
 463 */
 464 void *sqlite3Realloc(void *pOld, u64 nBytes){
 465   int nOld, nNew, nDiff;
 466   void *pNew;
 467   assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) );
 468   assert( sqlite3MemdebugNoType(pOld, (u8)~MEMTYPE_HEAP) );
 469   if( pOld==0 ){
 470     return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */
 471   }
 472   if( nBytes==0 ){
 473     sqlite3_free(pOld); /* IMP: R-26507-47431 */
 474     return 0;
 475   }
 476   if( nBytes>=0x7fffff00 ){
 477     /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */
 478     return 0;
 479   }
 480   nOld = sqlite3MallocSize(pOld);
 481   /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second
 482   ** argument to xRealloc is always a value returned by a prior call to
 483   ** xRoundup. */
 484   nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes);
 485   if( nOld==nNew ){
 486     pNew = pOld;
 487   }else if( sqlite3GlobalConfig.bMemstat ){
 488     sqlite3_mutex_enter(mem0.mutex);
 489     sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes);
 490     nDiff = nNew - nOld;
 491     if( nDiff>0 && sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >=
 492           mem0.alarmThreshold-nDiff ){
 493       sqlite3MallocAlarm(nDiff);
 494     }
 495     pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
 496 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
 497     if( pNew==0 && mem0.alarmThreshold>0 ){
 498       sqlite3MallocAlarm((int)nBytes);
 499       pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
 500     }
 501 #endif
 502     if( pNew ){
 503       nNew = sqlite3MallocSize(pNew);
 504       sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld);
 505     }
 506     sqlite3_mutex_leave(mem0.mutex);
 507   }else{
 508     pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
 509   }
 510   assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */
 511   return pNew;
 512 }
 513
 514 /*
 515 ** The public interface to sqlite3Realloc.  Make sure that the memory
 516 ** subsystem is initialized prior to invoking sqliteRealloc.
 517 */
 518 void *sqlite3_realloc(void *pOld, int n){
 519 #ifndef SQLITE_OMIT_AUTOINIT
 520   if( sqlite3_initialize() ) return 0;
 521 #endif
 522   if( n<0 ) n = 0;  /* IMP: R-26507-47431 */
 523   return sqlite3Realloc(pOld, n);
 524 }
 525 void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){
 526 #ifndef SQLITE_OMIT_AUTOINIT
 527   if( sqlite3_initialize() ) return 0;
 528 #endif
 529   return sqlite3Realloc(pOld, n);
 530 }
 531
 532
 533 /*
 534 ** Allocate and zero memory.
 535 */
 536 void *sqlite3MallocZero(u64 n){
 537   void *p = sqlite3Malloc(n);
 538   if( p ){
 539     memset(p, 0, (size_t)n);
 540   }
 541   return p;
 542 }
 543
 544 /*
 545 ** Allocate and zero memory.  If the allocation fails, make
 546 ** the mallocFailed flag in the connection pointer.
 547 */
 548 void *sqlite3DbMallocZero(sqlite3 *db, u64 n){
 549   void *p;
 550   testcase( db==0 );
 551   p = sqlite3DbMallocRaw(db, n);
 552   if( p ) memset(p, 0, (size_t)n);
 553   return p;
 554 }
 555
 556
 557 /* Finish the work of sqlite3DbMallocRawNN for the unusual and
 558 ** slower case when the allocation cannot be fulfilled using lookaside.
 559 */
 560 static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){
 561   void *p;
 562   assert( db!=0 );
 563   p = sqlite3Malloc(n);
 564   if( !p ) sqlite3OomFault(db);
 565   sqlite3MemdebugSetType(p,
 566          (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP);
 567   return p;
 568 }
 569
 570 /*
 571 ** Allocate memory, either lookaside (if possible) or heap.
 572 ** If the allocation fails, set the mallocFailed flag in
 573 ** the connection pointer.
 574 **
 575 ** If db!=0 and db->mallocFailed is true (indicating a prior malloc
 576 ** failure on the same database connection) then always return 0.
 577 ** Hence for a particular database connection, once malloc starts
 578 ** failing, it fails consistently until mallocFailed is reset.
 579 ** This is an important assumption.  There are many places in the
 580 ** code that do things like this:
 581 **
 582 **         int *a = (int*)sqlite3DbMallocRaw(db, 100);
 583 **         int *b = (int*)sqlite3DbMallocRaw(db, 200);
 584 **         if( b ) a[10] = 9;
 585 **
 586 ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed
 587 ** that all prior mallocs (ex: "a") worked too.
 588 **
 589 ** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is
 590 ** not a NULL pointer.
 591 */
 592 void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){
 593   void *p;
 594   if( db ) return sqlite3DbMallocRawNN(db, n);
 595   p = sqlite3Malloc(n);
 596   sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
 597   return p;
 598 }
 599 void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
 600 #ifndef SQLITE_OMIT_LOOKASIDE
 601   LookasideSlot *pBuf;
 602   assert( db!=0 );
 603   assert( sqlite3_mutex_held(db->mutex) );
 604   assert( db->pnBytesFreed==0 );
 605   if( n>db->lookaside.sz ){
 606     if( !db->lookaside.bDisable ){
 607       db->lookaside.anStat[1]++;
 608     }else if( db->mallocFailed ){
 609       return 0;
 610     }
 611     return dbMallocRawFinish(db, n);
 612   }
 613 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 614   if( n<=LOOKASIDE_SMALL ){
 615     if( (pBuf = db->lookaside.pSmallFree)!=0 ){
 616       db->lookaside.pSmallFree = pBuf->pNext;
 617       db->lookaside.anStat[0]++;
 618       return (void*)pBuf;
 619     }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){
 620       db->lookaside.pSmallInit = pBuf->pNext;
 621       db->lookaside.anStat[0]++;
 622       return (void*)pBuf;
 623     }
 624   }
 625 #endif
 626   if( (pBuf = db->lookaside.pFree)!=0 ){
 627     db->lookaside.pFree = pBuf->pNext;
 628     db->lookaside.anStat[0]++;
 629     return (void*)pBuf;
 630   }else if( (pBuf = db->lookaside.pInit)!=0 ){
 631     db->lookaside.pInit = pBuf->pNext;
 632     db->lookaside.anStat[0]++;
 633     return (void*)pBuf;
 634   }else{
 635     db->lookaside.anStat[2]++;
 636   }
 637 #else
 638   assert( db!=0 );
 639   assert( sqlite3_mutex_held(db->mutex) );
 640   assert( db->pnBytesFreed==0 );
 641   if( db->mallocFailed ){
 642     return 0;
 643   }
 644 #endif
 645   return dbMallocRawFinish(db, n);
 646 }
 647
 648 /* Forward declaration */
 649 static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n);
 650
 651 /*
 652 ** Resize the block of memory pointed to by p to n bytes. If the
 653 ** resize fails, set the mallocFailed flag in the connection object.
 654 */
 655 void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){
 656   assert( db!=0 );
 657   if( p==0 ) return sqlite3DbMallocRawNN(db, n);
 658   assert( sqlite3_mutex_held(db->mutex) );
 659   if( ((uptr)p)<(uptr)db->lookaside.pEnd ){
 660 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 661     if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){
 662       if( n<=LOOKASIDE_SMALL ) return p;
 663     }else
 664 #endif
 665     if( ((uptr)p)>=(uptr)db->lookaside.pStart ){
 666       if( n<=db->lookaside.szTrue ) return p;
 667     }
 668   }
 669   return dbReallocFinish(db, p, n);
 670 }
 671 static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){
 672   void *pNew = 0;
 673   assert( db!=0 );
 674   assert( p!=0 );
 675   if( db->mallocFailed==0 ){
 676     if( isLookaside(db, p) ){
 677       pNew = sqlite3DbMallocRawNN(db, n);
 678       if( pNew ){
 679         memcpy(pNew, p, lookasideMallocSize(db, p));
 680         sqlite3DbFree(db, p);
 681       }
 682     }else{
 683       assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 684       assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
 685       sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
 686       pNew = sqlite3Realloc(p, n);
 687       if( !pNew ){
 688         sqlite3OomFault(db);
 689       }
 690       sqlite3MemdebugSetType(pNew,
 691             (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP));
 692     }
 693   }
 694   return pNew;
 695 }
 696
 697 /*
 698 ** Attempt to reallocate p.  If the reallocation fails, then free p
 699 ** and set the mallocFailed flag in the database connection.
 700 */
 701 void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){
 702   void *pNew;
 703   pNew = sqlite3DbRealloc(db, p, n);
 704   if( !pNew ){
 705     sqlite3DbFree(db, p);
 706   }
 707   return pNew;
 708 }
 709
 710 /*
 711 ** Make a copy of a string in memory obtained from sqliteMalloc(). These
 712 ** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This
 713 ** is because when memory debugging is turned on, these two functions are
 714 ** called via macros that record the current file and line number in the
 715 ** ThreadData structure.
 716 */
 717 char *sqlite3DbStrDup(sqlite3 *db, const char *z){
 718   char *zNew;
 719   size_t n;
 720   if( z==0 ){
 721     return 0;
 722   }
 723   n = strlen(z) + 1;
 724   zNew = sqlite3DbMallocRaw(db, n);
 725   if( zNew ){
 726     memcpy(zNew, z, n);
 727   }
 728   return zNew;
 729 }
 730 char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){
 731   char *zNew;
 732   assert( db!=0 );
 733   assert( z!=0 || n==0 );
 734   assert( (n&0x7fffffff)==n );
 735   zNew = z ? sqlite3DbMallocRawNN(db, n+1) : 0;
 736   if( zNew ){
 737     memcpy(zNew, z, (size_t)n);
 738     zNew[n] = 0;
 739   }
 740   return zNew;
 741 }
 742
 743 /*
 744 ** The text between zStart and zEnd represents a phrase within a larger
 745 ** SQL statement.  Make a copy of this phrase in space obtained form
 746 ** sqlite3DbMalloc().  Omit leading and trailing whitespace.
 747 */
 748 char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){
 749   int n;
 750   while( sqlite3Isspace(zStart[0]) ) zStart++;
 751   n = (int)(zEnd - zStart);
 752   while( ALWAYS(n>0) && sqlite3Isspace(zStart[n-1]) ) n--;
 753   return sqlite3DbStrNDup(db, zStart, n);
 754 }
 755
 756 /*
 757 ** Free any prior content in *pz and replace it with a copy of zNew.
 758 */
 759 void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){
 760   sqlite3DbFree(db, *pz);
 761   *pz = sqlite3DbStrDup(db, zNew);
 762 }
 763
 764 /*
 765 ** Call this routine to record the fact that an OOM (out-of-memory) error
 766 ** has happened.  This routine will set db->mallocFailed, and also
 767 ** temporarily disable the lookaside memory allocator and interrupt
 768 ** any running VDBEs.
 769 */
 770 void sqlite3OomFault(sqlite3 *db){
 771   if( db->mallocFailed==0 && db->bBenignMalloc==0 ){
 772     db->mallocFailed = 1;
 773     if( db->nVdbeExec>0 ){
 774       AtomicStore(&db->u1.isInterrupted, 1);
 775     }
 776     DisableLookaside;
 777     if( db->pParse ){
 778       db->pParse->rc = SQLITE_NOMEM_BKPT;
 779     }
 780   }
 781 }
 782
 783 /*
 784 ** This routine reactivates the memory allocator and clears the
 785 ** db->mallocFailed flag as necessary.
 786 **
 787 ** The memory allocator is not restarted if there are running
 788 ** VDBEs.
 789 */
 790 void sqlite3OomClear(sqlite3 *db){
 791   if( db->mallocFailed && db->nVdbeExec==0 ){
 792     db->mallocFailed = 0;
 793     AtomicStore(&db->u1.isInterrupted, 0);
 794     assert( db->lookaside.bDisable>0 );
 795     EnableLookaside;
 796   }
 797 }
 798
 799 /*
 800 ** Take actions at the end of an API call to indicate an OOM error
 801 */
 802 static SQLITE_NOINLINE int apiOomError(sqlite3 *db){
 803   sqlite3OomClear(db);
 804   sqlite3Error(db, SQLITE_NOMEM);
 805   return SQLITE_NOMEM_BKPT;
 806 }
 807
 808 /*
 809 ** This function must be called before exiting any API function (i.e.
 810 ** returning control to the user) that has called sqlite3_malloc or
 811 ** sqlite3_realloc.
 812 **
 813 ** The returned value is normally a copy of the second argument to this
 814 ** function. However, if a malloc() failure has occurred since the previous
 815 ** invocation SQLITE_NOMEM is returned instead.
 816 **
 817 ** If an OOM as occurred, then the connection error-code (the value
 818 ** returned by sqlite3_errcode()) is set to SQLITE_NOMEM.
 819 */
 820 int sqlite3ApiExit(sqlite3* db, int rc){
 821   /* If the db handle must hold the connection handle mutex here.
 822   ** Otherwise the read (and possible write) of db->mallocFailed
 823   ** is unsafe, as is the call to sqlite3Error().
 824   */
 825   assert( db!=0 );
 826   assert( sqlite3_mutex_held(db->mutex) );
 827   if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){
 828     return apiOomError(db);
 829   }
 830   return rc & db->errMask;
 831 }