ext/lsm1/lsm-test/lsmtest_tdb4.c

   1
   2 /*
   3 ** This file contains the TestDb bt wrapper.
   4 */
   5
   6 #include "lsmtest_tdb.h"
   7 #include "lsmtest.h"
   8 #include <unistd.h>
   9 #include "bt.h"
  10
  11 #include <pthread.h>
  12
  13 typedef struct BtDb BtDb;
  14 typedef struct BtFile BtFile;
  15
  16 /* Background checkpointer interface (see implementations below). */
  17 typedef struct bt_ckpter bt_ckpter;
  18 static int bgc_attach(BtDb *pDb, const char*);
  19 static int bgc_detach(BtDb *pDb);
  20
  21 /*
  22 ** Each database or log file opened by a database handle is wrapped by
  23 ** an object of the following type.
  24 */
  25 struct BtFile {
  26   BtDb *pBt;                      /* Database handle that opened this file */
  27   bt_env *pVfs;                   /* Underlying VFS */
  28   bt_file *pFile;                 /* File handle belonging to underlying VFS */
  29   int nSectorSize;                /* Size of sectors in bytes */
  30   int nSector;                    /* Allocated size of nSector array */
  31   u8 **apSector;                  /* Original sector data */
  32 };
  33
  34 /*
  35 ** nCrashSync:
  36 **   If this value is non-zero, then a "crash-test" is running. If
  37 **   nCrashSync==1, then the crash is simulated during the very next
  38 **   call to the xSync() VFS method (on either the db or log file).
  39 **   If nCrashSync==2, the following call to xSync(), and so on.
  40 **
  41 ** bCrash:
  42 **   After a crash is simulated, this variable is set. Any subsequent
  43 **   attempts to write to a file or modify the file system in any way
  44 **   fail once this is set. All the caller can do is close the connection.
  45 **
  46 ** bFastInsert:
  47 **   If this variable is set to true, then a BT_CONTROL_FAST_INSERT_OP
  48 **   control is issued before each callto BtReplace() or BtCsrOpen().
  49 */
  50 struct BtDb {
  51   TestDb base;                    /* Base class */
  52   bt_db *pBt;                     /* bt database handle */
  53   sqlite4_env *pEnv;              /* SQLite environment (for malloc/free) */
  54   bt_env *pVfs;                   /* Underlying VFS */
  55   int bFastInsert;                /* True to use fast-insert */
  56
  57   /* Space for bt_fetch() results */
  58   u8 *aBuffer;                    /* Space to store results */
  59   int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
  60   int nRef;
  61
  62   /* Background checkpointer used by mt connections */
  63   bt_ckpter *pCkpter;
  64
  65   /* Stuff used for crash test simulation */
  66   BtFile *apFile[2];              /* Database and log files used by pBt */
  67   bt_env env;                     /* Private VFS for this object */
  68   int nCrashSync;                 /* Number of syncs until crash (see above) */
  69   int bCrash;                     /* True once a crash has been simulated */
  70 };
  71
  72 static int btVfsFullpath(
  73   sqlite4_env *pEnv,
  74   bt_env *pVfs,
  75   const char *z,
  76   char **pzOut
  77 ){
  78   BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
  79   if( pBt->bCrash ) return SQLITE4_IOERR;
  80   return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut);
  81 }
  82
  83 static int btVfsOpen(
  84   sqlite4_env *pEnv,
  85   bt_env *pVfs,
  86   const char *zFile,
  87   int flags, bt_file **ppFile
  88 ){
  89   BtFile *p;
  90   BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
  91   int rc;
  92
  93   if( pBt->bCrash ) return SQLITE4_IOERR;
  94
  95   p = (BtFile*)testMalloc(sizeof(BtFile));
  96   if( !p ) return SQLITE4_NOMEM;
  97   if( flags & BT_OPEN_DATABASE ){
  98     pBt->apFile[0] = p;
  99   }else if( flags & BT_OPEN_LOG ){
 100     pBt->apFile[1] = p;
 101   }
 102   if( (flags & BT_OPEN_SHARED)==0 ){
 103     p->pBt = pBt;
 104   }
 105   p->pVfs = pBt->pVfs;
 106
 107   rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile);
 108   if( rc!=SQLITE4_OK ){
 109     testFree(p);
 110     p = 0;
 111   }else{
 112     pBt->nRef++;
 113   }
 114
 115   *ppFile = (bt_file*)p;
 116   return rc;
 117 }
 118
 119 static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){
 120   BtFile *p = (BtFile*)pFile;
 121   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 122   return p->pVfs->xSize(p->pFile, piRes);
 123 }
 124
 125 static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){
 126   BtFile *p = (BtFile*)pFile;
 127   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 128   return p->pVfs->xRead(p->pFile, iOff, pBuf, nBuf);
 129 }
 130
 131 static int btFlushSectors(BtFile *p, int iFile){
 132   sqlite4_int64 iSz;
 133   int rc;
 134   int i;
 135   u8 *aTmp = 0;
 136
 137   rc = p->pBt->pVfs->xSize(p->pFile, &iSz);
 138   for(i=0; rc==SQLITE4_OK && i<p->nSector; i++){
 139     if( p->pBt->bCrash && p->apSector[i] ){
 140
 141       /* The system is simulating a crash. There are three choices for
 142       ** this sector:
 143       **
 144       **   1) Leave it as it is (simulating a successful write),
 145       **   2) Restore the original data (simulating a lost write),
 146       **   3) Populate the disk sector with garbage data.
 147       */
 148       sqlite4_int64 iSOff = p->nSectorSize*i;
 149       int nWrite = MIN(p->nSectorSize, iSz - iSOff);
 150
 151       if( nWrite ){
 152         u8 *aWrite = 0;
 153         int iOpt = (testPrngValue(i) % 3) + 1;
 154         if( iOpt==1 ){
 155           aWrite = p->apSector[i];
 156         }else if( iOpt==3 ){
 157           if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize);
 158           aWrite = aTmp;
 159           testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32));
 160         }
 161
 162 #if 0
 163 fprintf(stderr, "handle sector %d of %s with %s\n", i,
 164     iFile==0 ? "db" : "log",
 165     iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit"
 166 );
 167 fflush(stderr);
 168 #endif
 169
 170         if( aWrite ){
 171           rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite);
 172         }
 173       }
 174     }
 175     testFree(p->apSector[i]);
 176     p->apSector[i] = 0;
 177   }
 178
 179   testFree(aTmp);
 180   return rc;
 181 }
 182
 183 static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){
 184   int rc;
 185   sqlite4_int64 iSz;              /* Size of file on disk */
 186   int iFirst;                     /* First sector affected */
 187   int iSector;                    /* Current sector */
 188   int iLast;                      /* Last sector affected */
 189
 190   if( p->nSectorSize==0 ){
 191     p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile);
 192     if( p->nSectorSize<512 ) p->nSectorSize = 512;
 193   }
 194   iLast = (iOff+nBuf-1) / p->nSectorSize;
 195   iFirst = iOff / p->nSectorSize;
 196
 197   rc = p->pBt->pVfs->xSize(p->pFile, &iSz);
 198   for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){
 199     int nRead;
 200     sqlite4_int64 iSOff = iSector * p->nSectorSize;
 201     u8 *aBuf = testMalloc(p->nSectorSize);
 202     nRead = MIN(p->nSectorSize, (iSz - iSOff));
 203     if( nRead>0 ){
 204       rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead);
 205     }
 206
 207     while( rc==SQLITE4_OK && iSector>=p->nSector ){
 208       int nNew = p->nSector + 32;
 209       u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*));
 210       memcpy(apNew, p->apSector, p->nSector*sizeof(u8*));
 211       testFree(p->apSector);
 212       p->apSector = apNew;
 213       p->nSector = nNew;
 214     }
 215
 216     p->apSector[iSector] = aBuf;
 217   }
 218
 219   return rc;
 220 }
 221
 222 static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){
 223   BtFile *p = (BtFile*)pFile;
 224   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 225   if( p->pBt && p->pBt->nCrashSync ){
 226     btSaveSectors(p, iOff, nBuf);
 227   }
 228   return p->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf);
 229 }
 230
 231 static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){
 232   BtFile *p = (BtFile*)pFile;
 233   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 234   return p->pVfs->xTruncate(p->pFile, iOff);
 235 }
 236
 237 static int btVfsSync(bt_file *pFile){
 238   int rc = SQLITE4_OK;
 239   BtFile *p = (BtFile*)pFile;
 240   BtDb *pBt = p->pBt;
 241
 242   if( pBt ){
 243     if( pBt->bCrash ) return SQLITE4_IOERR;
 244     if( pBt->nCrashSync ){
 245       pBt->nCrashSync--;
 246       pBt->bCrash = (pBt->nCrashSync==0);
 247       if( pBt->bCrash ){
 248         btFlushSectors(pBt->apFile[0], 0);
 249         btFlushSectors(pBt->apFile[1], 1);
 250         rc = SQLITE4_IOERR;
 251       }else{
 252         btFlushSectors(p, 0);
 253       }
 254     }
 255   }
 256
 257   if( rc==SQLITE4_OK ){
 258     rc = p->pVfs->xSync(p->pFile);
 259   }
 260   return rc;
 261 }
 262
 263 static int btVfsSectorSize(bt_file *pFile){
 264   BtFile *p = (BtFile*)pFile;
 265   return p->pVfs->xSectorSize(p->pFile);
 266 }
 267
 268 static void btDeref(BtDb *p){
 269   p->nRef--;
 270   assert( p->nRef>=0 );
 271   if( p->nRef<=0 ) testFree(p);
 272 }
 273
 274 static int btVfsClose(bt_file *pFile){
 275   BtFile *p = (BtFile*)pFile;
 276   BtDb *pBt = p->pBt;
 277   int rc;
 278   if( pBt ){
 279     btFlushSectors(p, 0);
 280     if( p==pBt->apFile[0] ) pBt->apFile[0] = 0;
 281     if( p==pBt->apFile[1] ) pBt->apFile[1] = 0;
 282   }
 283   testFree(p->apSector);
 284   rc = p->pVfs->xClose(p->pFile);
 285 #if 0
 286   btDeref(p->pBt);
 287 #endif
 288   testFree(p);
 289   return rc;
 290 }
 291
 292 static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){
 293   BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
 294   if( pBt->bCrash ) return SQLITE4_IOERR;
 295   return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile);
 296 }
 297
 298 static int btVfsLock(bt_file *pFile, int iLock, int eType){
 299   BtFile *p = (BtFile*)pFile;
 300   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 301   return p->pVfs->xLock(p->pFile, iLock, eType);
 302 }
 303
 304 static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){
 305   BtFile *p = (BtFile*)pFile;
 306   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 307   return p->pVfs->xTestLock(p->pFile, iLock, nLock, eType);
 308 }
 309
 310 static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){
 311   BtFile *p = (BtFile*)pFile;
 312   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 313   return p->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut);
 314 }
 315
 316 static void btVfsShmBarrier(bt_file *pFile){
 317   BtFile *p = (BtFile*)pFile;
 318   return p->pVfs->xShmBarrier(p->pFile);
 319 }
 320
 321 static int btVfsShmUnmap(bt_file *pFile, int bDelete){
 322   BtFile *p = (BtFile*)pFile;
 323   if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR;
 324   return p->pVfs->xShmUnmap(p->pFile, bDelete);
 325 }
 326
 327 static int bt_close(TestDb *pTestDb){
 328   BtDb *p = (BtDb*)pTestDb;
 329   int rc = sqlite4BtClose(p->pBt);
 330   free(p->aBuffer);
 331   if( p->apFile[0] ) p->apFile[0]->pBt = 0;
 332   if( p->apFile[1] ) p->apFile[1]->pBt = 0;
 333   bgc_detach(p);
 334   testFree(p);
 335   return rc;
 336 }
 337
 338 static int btMinTransaction(BtDb *p, int iMin, int *piLevel){
 339   int iLevel;
 340   int rc = SQLITE4_OK;
 341
 342   iLevel = sqlite4BtTransactionLevel(p->pBt);
 343   if( iLevel<iMin ){
 344     rc = sqlite4BtBegin(p->pBt, iMin);
 345     *piLevel = iLevel;
 346   }else{
 347     *piLevel = -1;
 348   }
 349
 350   return rc;
 351 }
 352 static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){
 353   int rc = rcin;
 354   if( iLevel>=0 ){
 355     if( rc==SQLITE4_OK ){
 356       rc = sqlite4BtCommit(p->pBt, iLevel);
 357     }else{
 358       sqlite4BtRollback(p->pBt, iLevel);
 359     }
 360     assert( iLevel==sqlite4BtTransactionLevel(p->pBt) );
 361   }
 362   return rc;
 363 }
 364
 365 static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){
 366   BtDb *p = (BtDb*)pTestDb;
 367   int iLevel;
 368   int rc;
 369
 370   rc = btMinTransaction(p, 2, &iLevel);
 371   if( rc==SQLITE4_OK ){
 372     if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0);
 373     rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV);
 374     rc = btRestoreTransaction(p, iLevel, rc);
 375   }
 376   return rc;
 377 }
 378
 379 static int bt_delete(TestDb *pTestDb, void *pK, int nK){
 380   return bt_write(pTestDb, pK, nK, 0, -1);
 381 }
 382
 383 static int bt_delete_range(
 384   TestDb *pTestDb,
 385   void *pKey1, int nKey1,
 386   void *pKey2, int nKey2
 387 ){
 388   BtDb *p = (BtDb*)pTestDb;
 389   bt_cursor *pCsr = 0;
 390   int rc = SQLITE4_OK;
 391   int iLevel;
 392
 393   rc = btMinTransaction(p, 2, &iLevel);
 394   if( rc==SQLITE4_OK ){
 395     if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0);
 396     rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
 397   }
 398   while( rc==SQLITE4_OK ){
 399     const void *pK;
 400     int n;
 401     int nCmp;
 402     int res;
 403
 404     rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE);
 405     if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
 406     if( rc!=SQLITE4_OK ) break;
 407
 408     rc = sqlite4BtCsrKey(pCsr, &pK, &n);
 409     if( rc!=SQLITE4_OK ) break;
 410
 411     nCmp = MIN(n, nKey1);
 412     res = memcmp(pKey1, pK, nCmp);
 413     assert( res<0 || (res==0 && nKey1<=n) );
 414     if( res==0 && nKey1==n ){
 415       rc = sqlite4BtCsrNext(pCsr);
 416       if( rc!=SQLITE4_OK ) break;
 417       rc = sqlite4BtCsrKey(pCsr, &pK, &n);
 418       if( rc!=SQLITE4_OK ) break;
 419     }
 420
 421     nCmp = MIN(n, nKey2);
 422     res = memcmp(pKey2, pK, nCmp);
 423     if( res<0 || (res==0 && nKey2<=n) ) break;
 424
 425     rc = sqlite4BtDelete(pCsr);
 426   }
 427   if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
 428
 429   sqlite4BtCsrClose(pCsr);
 430
 431   rc = btRestoreTransaction(p, iLevel, rc);
 432   return rc;
 433 }
 434
 435 static int bt_fetch(
 436   TestDb *pTestDb,
 437   void *pK, int nK,
 438   void **ppVal, int *pnVal
 439 ){
 440   BtDb *p = (BtDb*)pTestDb;
 441   bt_cursor *pCsr = 0;
 442   int iLevel;
 443   int rc = SQLITE4_OK;
 444
 445   iLevel = sqlite4BtTransactionLevel(p->pBt);
 446   if( iLevel==0 ){
 447     rc = sqlite4BtBegin(p->pBt, 1);
 448     if( rc!=SQLITE4_OK ) return rc;
 449   }
 450
 451   if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0);
 452   rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
 453   if( rc==SQLITE4_OK ){
 454     rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ);
 455     if( rc==SQLITE4_OK ){
 456       const void *pV = 0;
 457       int nV = 0;
 458       rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
 459       if( rc==SQLITE4_OK ){
 460         if( nV>p->nBuffer ){
 461           free(p->aBuffer);
 462           p->aBuffer = (u8*)malloc(nV*2);
 463           p->nBuffer = nV*2;
 464         }
 465         memcpy(p->aBuffer, pV, nV);
 466         *pnVal = nV;
 467         *ppVal = (void*)(p->aBuffer);
 468       }
 469
 470     }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){
 471       *ppVal = 0;
 472       *pnVal = -1;
 473       rc = SQLITE4_OK;
 474     }
 475     sqlite4BtCsrClose(pCsr);
 476   }
 477
 478   if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0);
 479   return rc;
 480 }
 481
 482 static int bt_scan(
 483   TestDb *pTestDb,
 484   void *pCtx,
 485   int bReverse,
 486   void *pFirst, int nFirst,
 487   void *pLast, int nLast,
 488   void (*xCallback)(void *, void *, int , void *, int)
 489 ){
 490   BtDb *p = (BtDb*)pTestDb;
 491   bt_cursor *pCsr = 0;
 492   int rc;
 493   int iLevel;
 494
 495   rc = btMinTransaction(p, 1, &iLevel);
 496
 497   if( rc==SQLITE4_OK ){
 498     if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0);
 499     rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
 500   }
 501   if( rc==SQLITE4_OK ){
 502     if( bReverse ){
 503       if( pLast ){
 504         rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE);
 505       }else{
 506         rc = sqlite4BtCsrLast(pCsr);
 507       }
 508     }else{
 509       rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE);
 510     }
 511     if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
 512
 513     while( rc==SQLITE4_OK ){
 514       const void *pK = 0; int nK = 0;
 515       const void *pV = 0; int nV = 0;
 516
 517       rc = sqlite4BtCsrKey(pCsr, &pK, &nK);
 518       if( rc==SQLITE4_OK ){
 519         rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
 520       }
 521
 522       if( rc!=SQLITE4_OK ) break;
 523       if( bReverse ){
 524         if( pFirst ){
 525           int res;
 526           int nCmp = MIN(nK, nFirst);
 527           res = memcmp(pFirst, pK, nCmp);
 528           if( res>0 || (res==0 && nK<nFirst) ) break;
 529         }
 530       }else{
 531         if( pLast ){
 532           int res;
 533           int nCmp = MIN(nK, nLast);
 534           res = memcmp(pLast, pK, nCmp);
 535           if( res<0 || (res==0 && nK>nLast) ) break;
 536         }
 537       }
 538
 539       xCallback(pCtx, (void*)pK, nK, (void*)pV, nV);
 540       if( bReverse ){
 541         rc = sqlite4BtCsrPrev(pCsr);
 542       }else{
 543         rc = sqlite4BtCsrNext(pCsr);
 544       }
 545     }
 546     if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
 547
 548     sqlite4BtCsrClose(pCsr);
 549   }
 550
 551   rc = btRestoreTransaction(p, iLevel, rc);
 552   return rc;
 553 }
 554
 555 static int bt_begin(TestDb *pTestDb, int iLvl){
 556   BtDb *p = (BtDb*)pTestDb;
 557   int rc = sqlite4BtBegin(p->pBt, iLvl);
 558   return rc;
 559 }
 560
 561 static int bt_commit(TestDb *pTestDb, int iLvl){
 562   BtDb *p = (BtDb*)pTestDb;
 563   int rc = sqlite4BtCommit(p->pBt, iLvl);
 564   return rc;
 565 }
 566
 567 static int bt_rollback(TestDb *pTestDb, int iLvl){
 568   BtDb *p = (BtDb*)pTestDb;
 569   int rc = sqlite4BtRollback(p->pBt, iLvl);
 570   return rc;
 571 }
 572
 573 static int testParseOption(
 574   const char **pzIn,              /* IN/OUT: pointer to next option */
 575   const char **pzOpt,             /* OUT: nul-terminated option name */
 576   const char **pzArg,             /* OUT: nul-terminated option argument */
 577   char *pSpace                    /* Temporary space for output params */
 578 ){
 579   const char *p = *pzIn;
 580   const char *pStart;
 581   int n;
 582
 583   char *pOut = pSpace;
 584
 585   while( *p==' ' ) p++;
 586   pStart = p;
 587   while( *p && *p!='=' ) p++;
 588   if( *p==0 ) return 1;
 589
 590   n = (p - pStart);
 591   memcpy(pOut, pStart, n);
 592   *pzOpt = pOut;
 593   pOut += n;
 594   *pOut++ = '\0';
 595
 596   p++;
 597   pStart = p;
 598   while( *p && *p!=' ' ) p++;
 599   n = (p - pStart);
 600
 601   memcpy(pOut, pStart, n);
 602   *pzArg = pOut;
 603   pOut += n;
 604   *pOut++ = '\0';
 605
 606   *pzIn = p;
 607   return 0;
 608 }
 609
 610 static int testParseInt(const char *z, int *piVal){
 611   int i = 0;
 612   const char *p = z;
 613
 614   while( *p>='0' && *p<='9' ){
 615     i = i*10 + (*p - '0');
 616     p++;
 617   }
 618   if( *p=='K' || *p=='k' ){
 619     i = i * 1024;
 620     p++;
 621   }else if( *p=='M' || *p=='m' ){
 622     i = i * 1024 * 1024;
 623     p++;
 624   }
 625
 626   if( *p ) return SQLITE4_ERROR;
 627   *piVal = i;
 628   return SQLITE4_OK;
 629 }
 630
 631 static int testBtConfigure(BtDb *pDb, const char *zCfg, int *pbMt){
 632   int rc = SQLITE4_OK;
 633
 634   if( zCfg ){
 635     struct CfgParam {
 636       const char *zParam;
 637       int eParam;
 638     } aParam[] = {
 639       { "safety",         BT_CONTROL_SAFETY },
 640       { "autockpt",       BT_CONTROL_AUTOCKPT },
 641       { "multiproc",      BT_CONTROL_MULTIPROC },
 642       { "blksz",          BT_CONTROL_BLKSZ },
 643       { "pagesz",         BT_CONTROL_PAGESZ },
 644       { "mt",             -1 },
 645       { "fastinsert",     -2 },
 646       { 0, 0 }
 647     };
 648     const char *z = zCfg;
 649     int n = strlen(z);
 650     char *aSpace;
 651     const char *zOpt;
 652     const char *zArg;
 653
 654     aSpace = (char*)testMalloc(n+2);
 655     while( rc==SQLITE4_OK && 0==testParseOption(&z, &zOpt, &zArg, aSpace) ){
 656       int i;
 657       int iVal;
 658       rc = testArgSelect(aParam, "param", zOpt, &i);
 659       if( rc!=SQLITE4_OK ) break;
 660
 661       rc = testParseInt(zArg, &iVal);
 662       if( rc!=SQLITE4_OK ) break;
 663
 664       switch( aParam[i].eParam ){
 665         case -1:
 666           *pbMt = iVal;
 667           break;
 668         case -2:
 669           pDb->bFastInsert = 1;
 670           break;
 671         default:
 672           rc = sqlite4BtControl(pDb->pBt, aParam[i].eParam, (void*)&iVal);
 673           break;
 674       }
 675     }
 676     testFree(aSpace);
 677   }
 678
 679   return rc;
 680 }
 681
 682
 683 int test_bt_open(
 684   const char *zSpec,
 685   const char *zFilename,
 686   int bClear,
 687   TestDb **ppDb
 688 ){
 689
 690   static const DatabaseMethods SqlMethods = {
 691     bt_close,
 692     bt_write,
 693     bt_delete,
 694     bt_delete_range,
 695     bt_fetch,
 696     bt_scan,
 697     bt_begin,
 698     bt_commit,
 699     bt_rollback
 700   };
 701   BtDb *p = 0;
 702   bt_db *pBt = 0;
 703   int rc;
 704   sqlite4_env *pEnv = sqlite4_env_default();
 705
 706   if( bClear && zFilename && zFilename[0] ){
 707     char *zLog = sqlite3_mprintf("%s-wal", zFilename);
 708     unlink(zFilename);
 709     unlink(zLog);
 710     sqlite3_free(zLog);
 711   }
 712
 713   rc = sqlite4BtNew(pEnv, 0, &pBt);
 714   if( rc==SQLITE4_OK ){
 715     int mt = 0;                   /* True for multi-threaded connection */
 716
 717     p = (BtDb*)testMalloc(sizeof(BtDb));
 718     p->base.pMethods = &SqlMethods;
 719     p->pBt = pBt;
 720     p->pEnv = pEnv;
 721     p->nRef = 1;
 722
 723     p->env.pVfsCtx = (void*)p;
 724     p->env.xFullpath = btVfsFullpath;
 725     p->env.xOpen = btVfsOpen;
 726     p->env.xSize = btVfsSize;
 727     p->env.xRead = btVfsRead;
 728     p->env.xWrite = btVfsWrite;
 729     p->env.xTruncate = btVfsTruncate;
 730     p->env.xSync = btVfsSync;
 731     p->env.xSectorSize = btVfsSectorSize;
 732     p->env.xClose = btVfsClose;
 733     p->env.xUnlink = btVfsUnlink;
 734     p->env.xLock = btVfsLock;
 735     p->env.xTestLock = btVfsTestLock;
 736     p->env.xShmMap = btVfsShmMap;
 737     p->env.xShmBarrier = btVfsShmBarrier;
 738     p->env.xShmUnmap = btVfsShmUnmap;
 739
 740     sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs);
 741     sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env);
 742
 743     rc = testBtConfigure(p, zSpec, &mt);
 744     if( rc==SQLITE4_OK ){
 745       rc = sqlite4BtOpen(pBt, zFilename);
 746     }
 747
 748     if( rc==SQLITE4_OK && mt ){
 749       int nAuto = 0;
 750       rc = bgc_attach(p, zSpec);
 751       sqlite4BtControl(pBt, BT_CONTROL_AUTOCKPT, (void*)&nAuto);
 752     }
 753   }
 754
 755   if( rc!=SQLITE4_OK && p ){
 756     bt_close(&p->base);
 757   }
 758
 759   *ppDb = &p->base;
 760   return rc;
 761 }
 762
 763 int test_fbt_open(
 764   const char *zSpec,
 765   const char *zFilename,
 766   int bClear,
 767   TestDb **ppDb
 768 ){
 769   return test_bt_open("fast=1", zFilename, bClear, ppDb);
 770 }
 771
 772 int test_fbts_open(
 773   const char *zSpec,
 774   const char *zFilename,
 775   int bClear,
 776   TestDb **ppDb
 777 ){
 778   return test_bt_open("fast=1 blksz=32K pagesz=512", zFilename, bClear, ppDb);
 779 }
 780
 781
 782 void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){
 783   BtDb *p = (BtDb*)pTestDb;
 784   assert( pTestDb->pMethods->xClose==bt_close );
 785   assert( p->bCrash==0 );
 786   p->nCrashSync = iSync;
 787 }
 788
 789 bt_db *tdb_bt(TestDb *pDb){
 790   if( pDb->pMethods->xClose==bt_close ){
 791     return ((BtDb *)pDb)->pBt;
 792   }
 793   return 0;
 794 }
 795
 796 /*************************************************************************
 797 ** Beginning of code for background checkpointer.
 798 */
 799
 800 struct bt_ckpter {
 801   sqlite4_buffer file;            /* File name */
 802   sqlite4_buffer spec;            /* Options */
 803   int nLogsize;                   /* Minimum log size to checkpoint */
 804   int nRef;                       /* Number of clients */
 805
 806   int bDoWork;                    /* Set by client threads */
 807   pthread_t ckpter_thread;        /* Checkpointer thread */
 808   pthread_cond_t ckpter_cond;     /* Condition var the ckpter waits on */
 809   pthread_mutex_t ckpter_mutex;   /* Mutex used with ckpter_cond */
 810
 811   bt_ckpter *pNext;               /* Next object in list at gBgc.pCkpter */
 812 };
 813
 814 static struct GlobalBackgroundCheckpointer {
 815   bt_ckpter *pCkpter;             /* Linked list of checkpointers */
 816 } gBgc;
 817
 818 static void *bgc_main(void *pArg){
 819   BtDb *pDb = 0;
 820   int rc;
 821   int mt;
 822   bt_ckpter *pCkpter = (bt_ckpter*)pArg;
 823
 824   rc = test_bt_open("", (char*)pCkpter->file.p, 0, (TestDb**)&pDb);
 825   assert( rc==SQLITE4_OK );
 826   rc = testBtConfigure(pDb, (char*)pCkpter->spec.p, &mt);
 827
 828   while( pCkpter->nRef>0 ){
 829     bt_db *db = pDb->pBt;
 830     int nLog = 0;
 831
 832     sqlite4BtBegin(db, 1);
 833     sqlite4BtCommit(db, 0);
 834     sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog);
 835
 836     if( nLog>=pCkpter->nLogsize ){
 837       int rc;
 838       bt_checkpoint ckpt;
 839       memset(&ckpt, 0, sizeof(bt_checkpoint));
 840       ckpt.nFrameBuffer = nLog/2;
 841       rc = sqlite4BtControl(db, BT_CONTROL_CHECKPOINT, (void*)&ckpt);
 842       assert( rc==SQLITE4_OK );
 843       sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog);
 844     }
 845
 846     /* The thread will wake up when it is signaled either because another
 847     ** thread has created some work for this one or because the connection
 848     ** is being closed.  */
 849     pthread_mutex_lock(&pCkpter->ckpter_mutex);
 850     if( pCkpter->bDoWork==0 ){
 851       pthread_cond_wait(&pCkpter->ckpter_cond, &pCkpter->ckpter_mutex);
 852     }
 853     pCkpter->bDoWork = 0;
 854     pthread_mutex_unlock(&pCkpter->ckpter_mutex);
 855   }
 856
 857   if( pDb ) bt_close((TestDb*)pDb);
 858   return 0;
 859 }
 860
 861 static void bgc_logsize_cb(void *pCtx, int nLogsize){
 862   bt_ckpter *p = (bt_ckpter*)pCtx;
 863   if( nLogsize>=p->nLogsize ){
 864     pthread_mutex_lock(&p->ckpter_mutex);
 865     p->bDoWork = 1;
 866     pthread_cond_signal(&p->ckpter_cond);
 867     pthread_mutex_unlock(&p->ckpter_mutex);
 868   }
 869 }
 870
 871 static int bgc_attach(BtDb *pDb, const char *zSpec){
 872   int rc;
 873   int n;
 874   bt_info info;
 875   bt_ckpter *pCkpter;
 876
 877   /* Figure out the full path to the database opened by handle pDb. */
 878   info.eType = BT_INFO_FILENAME;
 879   info.pgno = 0;
 880   sqlite4_buffer_init(&info.output, 0);
 881   rc = sqlite4BtControl(pDb->pBt, BT_CONTROL_INFO, (void*)&info);
 882   if( rc!=SQLITE4_OK ) return rc;
 883
 884   sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV));
 885
 886   /* Search for an existing bt_ckpter object. */
 887   n = info.output.n;
 888   for(pCkpter=gBgc.pCkpter; pCkpter; pCkpter=pCkpter->pNext){
 889     if( n==pCkpter->file.n && 0==memcmp(info.output.p, pCkpter->file.p, n) ){
 890       break;
 891     }
 892   }
 893
 894   /* Failed to find a suitable checkpointer. Create a new one. */
 895   if( pCkpter==0 ){
 896     bt_logsizecb cb;
 897
 898     pCkpter = testMalloc(sizeof(bt_ckpter));
 899     memcpy(&pCkpter->file, &info.output, sizeof(sqlite4_buffer));
 900     info.output.p = 0;
 901     pCkpter->pNext = gBgc.pCkpter;
 902     pCkpter->nLogsize = 1000;
 903     gBgc.pCkpter = pCkpter;
 904     pCkpter->nRef = 1;
 905
 906     sqlite4_buffer_init(&pCkpter->spec, 0);
 907     rc = sqlite4_buffer_set(&pCkpter->spec, zSpec, strlen(zSpec)+1);
 908     assert( rc==SQLITE4_OK );
 909
 910     /* Kick off the checkpointer thread. */
 911     if( rc==0 ) rc = pthread_cond_init(&pCkpter->ckpter_cond, 0);
 912     if( rc==0 ) rc = pthread_mutex_init(&pCkpter->ckpter_mutex, 0);
 913     if( rc==0 ){
 914       rc = pthread_create(&pCkpter->ckpter_thread, 0, bgc_main, (void*)pCkpter);
 915     }
 916     assert( rc==0 ); /* todo: Fix this */
 917
 918     /* Set up the logsize callback for the client thread */
 919     cb.pCtx = (void*)pCkpter;
 920     cb.xLogsize = bgc_logsize_cb;
 921     sqlite4BtControl(pDb->pBt, BT_CONTROL_LOGSIZECB, (void*)&cb);
 922   }else{
 923     pCkpter->nRef++;
 924   }
 925
 926   /* Assuming a checkpointer was encountered or effected, attach the
 927   ** connection to it.  */
 928   if( pCkpter ){
 929     pDb->pCkpter = pCkpter;
 930   }
 931
 932   sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV));
 933   sqlite4_buffer_clear(&info.output);
 934   return rc;
 935 }
 936
 937 static int bgc_detach(BtDb *pDb){
 938   int rc = SQLITE4_OK;
 939   bt_ckpter *pCkpter = pDb->pCkpter;
 940   if( pCkpter ){
 941     int bShutdown = 0;            /* True if this is the last reference */
 942
 943     sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV));
 944     pCkpter->nRef--;
 945     if( pCkpter->nRef==0 ){
 946       bt_ckpter **pp;
 947
 948       *pp = pCkpter->pNext;
 949       for(pp=&gBgc.pCkpter; *pp!=pCkpter; pp=&((*pp)->pNext));
 950       bShutdown = 1;
 951     }
 952     sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV));
 953
 954     if( bShutdown ){
 955       void *pDummy;
 956
 957       /* Signal the checkpointer thread. */
 958       pthread_mutex_lock(&pCkpter->ckpter_mutex);
 959       pCkpter->bDoWork = 1;
 960       pthread_cond_signal(&pCkpter->ckpter_cond);
 961       pthread_mutex_unlock(&pCkpter->ckpter_mutex);
 962
 963       /* Join the checkpointer thread. */
 964       pthread_join(pCkpter->ckpter_thread, &pDummy);
 965       pthread_cond_destroy(&pCkpter->ckpter_cond);
 966       pthread_mutex_destroy(&pCkpter->ckpter_mutex);
 967
 968       sqlite4_buffer_clear(&pCkpter->file);
 969       sqlite4_buffer_clear(&pCkpter->spec);
 970       testFree(pCkpter);
 971     }
 972
 973     pDb->pCkpter = 0;
 974   }
 975   return rc;
 976 }
 977
 978 /*
 979 ** End of background checkpointer.
 980 *************************************************************************/
 981
 982