ext/recover/dbdata.c

   1 /*
   2 ** 2019-04-17
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This file contains an implementation of two eponymous virtual tables,
  14 ** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the
  15 ** "sqlite_dbpage" eponymous virtual table be available.
  16 **
  17 ** SQLITE_DBDATA:
  18 **   sqlite_dbdata is used to extract data directly from a database b-tree
  19 **   page and its associated overflow pages, bypassing the b-tree layer.
  20 **   The table schema is equivalent to:
  21 **
  22 **     CREATE TABLE sqlite_dbdata(
  23 **       pgno INTEGER,
  24 **       cell INTEGER,
  25 **       field INTEGER,
  26 **       value ANY,
  27 **       schema TEXT HIDDEN
  28 **     );
  29 **
  30 **   IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE
  31 **   FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND
  32 **   "schema".
  33 **
  34 **   Each page of the database is inspected. If it cannot be interpreted as
  35 **   a b-tree page, or if it is a b-tree page containing 0 entries, the
  36 **   sqlite_dbdata table contains no rows for that page.  Otherwise, the
  37 **   table contains one row for each field in the record associated with
  38 **   each cell on the page. For intkey b-trees, the key value is stored in
  39 **   field -1.
  40 **
  41 **   For example, for the database:
  42 **
  43 **     CREATE TABLE t1(a, b);     -- root page is page 2
  44 **     INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five');
  45 **     INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten');
  46 **
  47 **   the sqlite_dbdata table contains, as well as from entries related to
  48 **   page 1, content equivalent to:
  49 **
  50 **     INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES
  51 **         (2, 0, -1, 5     ),
  52 **         (2, 0,  0, 'v'   ),
  53 **         (2, 0,  1, 'five'),
  54 **         (2, 1, -1, 10    ),
  55 **         (2, 1,  0, 'x'   ),
  56 **         (2, 1,  1, 'ten' );
  57 **
  58 **   If database corruption is encountered, this module does not report an
  59 **   error. Instead, it attempts to extract as much data as possible and
  60 **   ignores the corruption.
  61 **
  62 ** SQLITE_DBPTR:
  63 **   The sqlite_dbptr table has the following schema:
  64 **
  65 **     CREATE TABLE sqlite_dbptr(
  66 **       pgno INTEGER,
  67 **       child INTEGER,
  68 **       schema TEXT HIDDEN
  69 **     );
  70 **
  71 **   It contains one entry for each b-tree pointer between a parent and
  72 **   child page in the database.
  73 */
  74
  75 #if !defined(SQLITEINT_H)
  76 #include "sqlite3.h"
  77
  78 typedef unsigned char u8;
  79 typedef unsigned int u32;
  80
  81 #endif
  82 #include <string.h>
  83 #include <assert.h>
  84
  85 #ifndef SQLITE_OMIT_VIRTUALTABLE
  86
  87 #define DBDATA_PADDING_BYTES 100
  88
  89 typedef struct DbdataTable DbdataTable;
  90 typedef struct DbdataCursor DbdataCursor;
  91
  92 /* Cursor object */
  93 struct DbdataCursor {
  94   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
  95   sqlite3_stmt *pStmt;            /* For fetching database pages */
  96
  97   int iPgno;                      /* Current page number */
  98   u8 *aPage;                      /* Buffer containing page */
  99   int nPage;                      /* Size of aPage[] in bytes */
 100   int nCell;                      /* Number of cells on aPage[] */
 101   int iCell;                      /* Current cell number */
 102   int bOnePage;                   /* True to stop after one page */
 103   int szDb;
 104   sqlite3_int64 iRowid;
 105
 106   /* Only for the sqlite_dbdata table */
 107   u8 *pRec;                       /* Buffer containing current record */
 108   sqlite3_int64 nRec;             /* Size of pRec[] in bytes */
 109   sqlite3_int64 nHdr;             /* Size of header in bytes */
 110   int iField;                     /* Current field number */
 111   u8 *pHdrPtr;
 112   u8 *pPtr;
 113   u32 enc;                        /* Text encoding */
 114
 115   sqlite3_int64 iIntkey;          /* Integer key value */
 116 };
 117
 118 /* Table object */
 119 struct DbdataTable {
 120   sqlite3_vtab base;              /* Base class.  Must be first */
 121   sqlite3 *db;                    /* The database connection */
 122   sqlite3_stmt *pStmt;            /* For fetching database pages */
 123   int bPtr;                       /* True for sqlite3_dbptr table */
 124 };
 125
 126 /* Column and schema definitions for sqlite_dbdata */
 127 #define DBDATA_COLUMN_PGNO        0
 128 #define DBDATA_COLUMN_CELL        1
 129 #define DBDATA_COLUMN_FIELD       2
 130 #define DBDATA_COLUMN_VALUE       3
 131 #define DBDATA_COLUMN_SCHEMA      4
 132 #define DBDATA_SCHEMA             \
 133       "CREATE TABLE x("           \
 134       "  pgno INTEGER,"           \
 135       "  cell INTEGER,"           \
 136       "  field INTEGER,"          \
 137       "  value ANY,"              \
 138       "  schema TEXT HIDDEN"      \
 139       ")"
 140
 141 /* Column and schema definitions for sqlite_dbptr */
 142 #define DBPTR_COLUMN_PGNO         0
 143 #define DBPTR_COLUMN_CHILD        1
 144 #define DBPTR_COLUMN_SCHEMA       2
 145 #define DBPTR_SCHEMA              \
 146       "CREATE TABLE x("           \
 147       "  pgno INTEGER,"           \
 148       "  child INTEGER,"          \
 149       "  schema TEXT HIDDEN"      \
 150       ")"
 151
 152 /*
 153 ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual
 154 ** table.
 155 */
 156 static int dbdataConnect(
 157   sqlite3 *db,
 158   void *pAux,
 159   int argc, const char *const*argv,
 160   sqlite3_vtab **ppVtab,
 161   char **pzErr
 162 ){
 163   DbdataTable *pTab = 0;
 164   int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA);
 165
 166   (void)argc;
 167   (void)argv;
 168   (void)pzErr;
 169   sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
 170   if( rc==SQLITE_OK ){
 171     pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable));
 172     if( pTab==0 ){
 173       rc = SQLITE_NOMEM;
 174     }else{
 175       memset(pTab, 0, sizeof(DbdataTable));
 176       pTab->db = db;
 177       pTab->bPtr = (pAux!=0);
 178     }
 179   }
 180
 181   *ppVtab = (sqlite3_vtab*)pTab;
 182   return rc;
 183 }
 184
 185 /*
 186 ** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table.
 187 */
 188 static int dbdataDisconnect(sqlite3_vtab *pVtab){
 189   DbdataTable *pTab = (DbdataTable*)pVtab;
 190   if( pTab ){
 191     sqlite3_finalize(pTab->pStmt);
 192     sqlite3_free(pVtab);
 193   }
 194   return SQLITE_OK;
 195 }
 196
 197 /*
 198 ** This function interprets two types of constraints:
 199 **
 200 **       schema=?
 201 **       pgno=?
 202 **
 203 ** If neither are present, idxNum is set to 0. If schema=? is present,
 204 ** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit
 205 ** in idxNum is set.
 206 **
 207 ** If both parameters are present, schema is in position 0 and pgno in
 208 ** position 1.
 209 */
 210 static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){
 211   DbdataTable *pTab = (DbdataTable*)tab;
 212   int i;
 213   int iSchema = -1;
 214   int iPgno = -1;
 215   int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA);
 216
 217   for(i=0; i<pIdx->nConstraint; i++){
 218     struct sqlite3_index_constraint *p = &pIdx->aConstraint[i];
 219     if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
 220       if( p->iColumn==colSchema ){
 221         if( p->usable==0 ) return SQLITE_CONSTRAINT;
 222         iSchema = i;
 223       }
 224       if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){
 225         iPgno = i;
 226       }
 227     }
 228   }
 229
 230   if( iSchema>=0 ){
 231     pIdx->aConstraintUsage[iSchema].argvIndex = 1;
 232     pIdx->aConstraintUsage[iSchema].omit = 1;
 233   }
 234   if( iPgno>=0 ){
 235     pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0);
 236     pIdx->aConstraintUsage[iPgno].omit = 1;
 237     pIdx->estimatedCost = 100;
 238     pIdx->estimatedRows =  50;
 239
 240     if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){
 241       int iCol = pIdx->aOrderBy[0].iColumn;
 242       if( pIdx->nOrderBy==1 ){
 243         pIdx->orderByConsumed = (iCol==0 || iCol==1);
 244       }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){
 245         pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1);
 246       }
 247     }
 248
 249   }else{
 250     pIdx->estimatedCost = 100000000;
 251     pIdx->estimatedRows = 1000000000;
 252   }
 253   pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00);
 254   return SQLITE_OK;
 255 }
 256
 257 /*
 258 ** Open a new sqlite_dbdata or sqlite_dbptr cursor.
 259 */
 260 static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
 261   DbdataCursor *pCsr;
 262
 263   pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor));
 264   if( pCsr==0 ){
 265     return SQLITE_NOMEM;
 266   }else{
 267     memset(pCsr, 0, sizeof(DbdataCursor));
 268     pCsr->base.pVtab = pVTab;
 269   }
 270
 271   *ppCursor = (sqlite3_vtab_cursor *)pCsr;
 272   return SQLITE_OK;
 273 }
 274
 275 /*
 276 ** Restore a cursor object to the state it was in when first allocated
 277 ** by dbdataOpen().
 278 */
 279 static void dbdataResetCursor(DbdataCursor *pCsr){
 280   DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab);
 281   if( pTab->pStmt==0 ){
 282     pTab->pStmt = pCsr->pStmt;
 283   }else{
 284     sqlite3_finalize(pCsr->pStmt);
 285   }
 286   pCsr->pStmt = 0;
 287   pCsr->iPgno = 1;
 288   pCsr->iCell = 0;
 289   pCsr->iField = 0;
 290   pCsr->bOnePage = 0;
 291   sqlite3_free(pCsr->aPage);
 292   sqlite3_free(pCsr->pRec);
 293   pCsr->pRec = 0;
 294   pCsr->aPage = 0;
 295 }
 296
 297 /*
 298 ** Close an sqlite_dbdata or sqlite_dbptr cursor.
 299 */
 300 static int dbdataClose(sqlite3_vtab_cursor *pCursor){
 301   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 302   dbdataResetCursor(pCsr);
 303   sqlite3_free(pCsr);
 304   return SQLITE_OK;
 305 }
 306
 307 /*
 308 ** Utility methods to decode 16 and 32-bit big-endian unsigned integers.
 309 */
 310 static u32 get_uint16(unsigned char *a){
 311   return (a[0]<<8)|a[1];
 312 }
 313 static u32 get_uint32(unsigned char *a){
 314   return ((u32)a[0]<<24)
 315        | ((u32)a[1]<<16)
 316        | ((u32)a[2]<<8)
 317        | ((u32)a[3]);
 318 }
 319
 320 /*
 321 ** Load page pgno from the database via the sqlite_dbpage virtual table.
 322 ** If successful, set (*ppPage) to point to a buffer containing the page
 323 ** data, (*pnPage) to the size of that buffer in bytes and return
 324 ** SQLITE_OK. In this case it is the responsibility of the caller to
 325 ** eventually free the buffer using sqlite3_free().
 326 **
 327 ** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and
 328 ** return an SQLite error code.
 329 */
 330 static int dbdataLoadPage(
 331   DbdataCursor *pCsr,             /* Cursor object */
 332   u32 pgno,                       /* Page number of page to load */
 333   u8 **ppPage,                    /* OUT: pointer to page buffer */
 334   int *pnPage                     /* OUT: Size of (*ppPage) in bytes */
 335 ){
 336   int rc2;
 337   int rc = SQLITE_OK;
 338   sqlite3_stmt *pStmt = pCsr->pStmt;
 339
 340   *ppPage = 0;
 341   *pnPage = 0;
 342   if( pgno>0 ){
 343     sqlite3_bind_int64(pStmt, 2, pgno);
 344     if( SQLITE_ROW==sqlite3_step(pStmt) ){
 345       int nCopy = sqlite3_column_bytes(pStmt, 0);
 346       if( nCopy>0 ){
 347         u8 *pPage;
 348         pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
 349         if( pPage==0 ){
 350           rc = SQLITE_NOMEM;
 351         }else{
 352           const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
 353           memcpy(pPage, pCopy, nCopy);
 354           memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
 355         }
 356         *ppPage = pPage;
 357         *pnPage = nCopy;
 358       }
 359     }
 360     rc2 = sqlite3_reset(pStmt);
 361     if( rc==SQLITE_OK ) rc = rc2;
 362   }
 363
 364   return rc;
 365 }
 366
 367 /*
 368 ** Read a varint.  Put the value in *pVal and return the number of bytes.
 369 */
 370 static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
 371   sqlite3_uint64 u = 0;
 372   int i;
 373   for(i=0; i<8; i++){
 374     u = (u<<7) + (z[i]&0x7f);
 375     if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
 376   }
 377   u = (u<<8) + (z[i]&0xff);
 378   *pVal = (sqlite3_int64)u;
 379   return 9;
 380 }
 381
 382 /*
 383 ** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
 384 ** or greater than 0xFFFFFFFF. This can be used for all varints in an
 385 ** SQLite database except for key values in intkey tables.
 386 */
 387 static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
 388   sqlite3_int64 val;
 389   int nRet = dbdataGetVarint(z, &val);
 390   if( val<0 || val>0xFFFFFFFF ) val = 0;
 391   *pVal = val;
 392   return nRet;
 393 }
 394
 395 /*
 396 ** Return the number of bytes of space used by an SQLite value of type
 397 ** eType.
 398 */
 399 static int dbdataValueBytes(int eType){
 400   switch( eType ){
 401     case 0: case 8: case 9:
 402     case 10: case 11:
 403       return 0;
 404     case 1:
 405       return 1;
 406     case 2:
 407       return 2;
 408     case 3:
 409       return 3;
 410     case 4:
 411       return 4;
 412     case 5:
 413       return 6;
 414     case 6:
 415     case 7:
 416       return 8;
 417     default:
 418       if( eType>0 ){
 419         return ((eType-12) / 2);
 420       }
 421       return 0;
 422   }
 423 }
 424
 425 /*
 426 ** Load a value of type eType from buffer pData and use it to set the
 427 ** result of context object pCtx.
 428 */
 429 static void dbdataValue(
 430   sqlite3_context *pCtx,
 431   u32 enc,
 432   int eType,
 433   u8 *pData,
 434   sqlite3_int64 nData
 435 ){
 436   if( eType>=0 && dbdataValueBytes(eType)<=nData ){
 437     switch( eType ){
 438       case 0:
 439       case 10:
 440       case 11:
 441         sqlite3_result_null(pCtx);
 442         break;
 443
 444       case 8:
 445         sqlite3_result_int(pCtx, 0);
 446         break;
 447       case 9:
 448         sqlite3_result_int(pCtx, 1);
 449         break;
 450
 451       case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
 452         sqlite3_uint64 v = (signed char)pData[0];
 453         pData++;
 454         switch( eType ){
 455           case 7:
 456           case 6:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
 457           case 5:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
 458           case 4:  v = (v<<8) + pData[0];  pData++;
 459           case 3:  v = (v<<8) + pData[0];  pData++;
 460           case 2:  v = (v<<8) + pData[0];  pData++;
 461         }
 462
 463         if( eType==7 ){
 464           double r;
 465           memcpy(&r, &v, sizeof(r));
 466           sqlite3_result_double(pCtx, r);
 467         }else{
 468           sqlite3_result_int64(pCtx, (sqlite3_int64)v);
 469         }
 470         break;
 471       }
 472
 473       default: {
 474         int n = ((eType-12) / 2);
 475         if( eType % 2 ){
 476           switch( enc ){
 477 #ifndef SQLITE_OMIT_UTF16
 478             case SQLITE_UTF16BE:
 479               sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
 480               break;
 481             case SQLITE_UTF16LE:
 482               sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
 483               break;
 484 #endif
 485             default:
 486               sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
 487               break;
 488           }
 489         }else{
 490           sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
 491         }
 492       }
 493     }
 494   }
 495 }
 496
 497 /*
 498 ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry.
 499 */
 500 static int dbdataNext(sqlite3_vtab_cursor *pCursor){
 501   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 502   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 503
 504   pCsr->iRowid++;
 505   while( 1 ){
 506     int rc;
 507     int iOff = (pCsr->iPgno==1 ? 100 : 0);
 508     int bNextPage = 0;
 509
 510     if( pCsr->aPage==0 ){
 511       while( 1 ){
 512         if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK;
 513         rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
 514         if( rc!=SQLITE_OK ) return rc;
 515         if( pCsr->aPage && pCsr->nPage>=256 ) break;
 516         sqlite3_free(pCsr->aPage);
 517         pCsr->aPage = 0;
 518         if( pCsr->bOnePage ) return SQLITE_OK;
 519         pCsr->iPgno++;
 520       }
 521
 522       assert( iOff+3+2<=pCsr->nPage );
 523       pCsr->iCell = pTab->bPtr ? -2 : 0;
 524       pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]);
 525     }
 526
 527     if( pTab->bPtr ){
 528       if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){
 529         pCsr->iCell = pCsr->nCell;
 530       }
 531       pCsr->iCell++;
 532       if( pCsr->iCell>=pCsr->nCell ){
 533         sqlite3_free(pCsr->aPage);
 534         pCsr->aPage = 0;
 535         if( pCsr->bOnePage ) return SQLITE_OK;
 536         pCsr->iPgno++;
 537       }else{
 538         return SQLITE_OK;
 539       }
 540     }else{
 541       /* If there is no record loaded, load it now. */
 542       if( pCsr->pRec==0 ){
 543         int bHasRowid = 0;
 544         int nPointer = 0;
 545         sqlite3_int64 nPayload = 0;
 546         sqlite3_int64 nHdr = 0;
 547         int iHdr;
 548         int U, X;
 549         int nLocal;
 550
 551         switch( pCsr->aPage[iOff] ){
 552           case 0x02:
 553             nPointer = 4;
 554             break;
 555           case 0x0a:
 556             break;
 557           case 0x0d:
 558             bHasRowid = 1;
 559             break;
 560           default:
 561             /* This is not a b-tree page with records on it. Continue. */
 562             pCsr->iCell = pCsr->nCell;
 563             break;
 564         }
 565
 566         if( pCsr->iCell>=pCsr->nCell ){
 567           bNextPage = 1;
 568         }else{
 569
 570           iOff += 8 + nPointer + pCsr->iCell*2;
 571           if( iOff>pCsr->nPage ){
 572             bNextPage = 1;
 573           }else{
 574             iOff = get_uint16(&pCsr->aPage[iOff]);
 575           }
 576
 577           /* For an interior node cell, skip past the child-page number */
 578           iOff += nPointer;
 579
 580           /* Load the "byte of payload including overflow" field */
 581           if( bNextPage || iOff>pCsr->nPage ){
 582             bNextPage = 1;
 583           }else{
 584             iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
 585             if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
 586           }
 587
 588           /* If this is a leaf intkey cell, load the rowid */
 589           if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){
 590             iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey);
 591           }
 592
 593           /* Figure out how much data to read from the local page */
 594           U = pCsr->nPage;
 595           if( bHasRowid ){
 596             X = U-35;
 597           }else{
 598             X = ((U-12)*64/255)-23;
 599           }
 600           if( nPayload<=X ){
 601             nLocal = nPayload;
 602           }else{
 603             int M, K;
 604             M = ((U-12)*32/255)-23;
 605             K = M+((nPayload-M)%(U-4));
 606             if( K<=X ){
 607               nLocal = K;
 608             }else{
 609               nLocal = M;
 610             }
 611           }
 612
 613           if( bNextPage || nLocal+iOff>pCsr->nPage ){
 614             bNextPage = 1;
 615           }else{
 616
 617             /* Allocate space for payload. And a bit more to catch small buffer
 618             ** overruns caused by attempting to read a varint or similar from
 619             ** near the end of a corrupt record.  */
 620             pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES);
 621             if( pCsr->pRec==0 ) return SQLITE_NOMEM;
 622             memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES);
 623             pCsr->nRec = nPayload;
 624
 625             /* Load the nLocal bytes of payload */
 626             memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal);
 627             iOff += nLocal;
 628
 629             /* Load content from overflow pages */
 630             if( nPayload>nLocal ){
 631               sqlite3_int64 nRem = nPayload - nLocal;
 632               u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
 633               while( nRem>0 ){
 634                 u8 *aOvfl = 0;
 635                 int nOvfl = 0;
 636                 int nCopy;
 637                 rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl);
 638                 assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage );
 639                 if( rc!=SQLITE_OK ) return rc;
 640                 if( aOvfl==0 ) break;
 641
 642                 nCopy = U-4;
 643                 if( nCopy>nRem ) nCopy = nRem;
 644                 memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
 645                 nRem -= nCopy;
 646
 647                 pgnoOvfl = get_uint32(aOvfl);
 648                 sqlite3_free(aOvfl);
 649               }
 650             }
 651
 652             iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
 653             if( nHdr>nPayload ) nHdr = 0;
 654             pCsr->nHdr = nHdr;
 655             pCsr->pHdrPtr = &pCsr->pRec[iHdr];
 656             pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
 657             pCsr->iField = (bHasRowid ? -1 : 0);
 658           }
 659         }
 660       }else{
 661         pCsr->iField++;
 662         if( pCsr->iField>0 ){
 663           sqlite3_int64 iType;
 664           if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
 665             bNextPage = 1;
 666           }else{
 667             int szField = 0;
 668             pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
 669             szField = dbdataValueBytes(iType);
 670             if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))<szField ){
 671               pCsr->pPtr = &pCsr->pRec[pCsr->nRec];
 672             }else{
 673               pCsr->pPtr += szField;
 674             }
 675           }
 676         }
 677       }
 678
 679       if( bNextPage ){
 680         sqlite3_free(pCsr->aPage);
 681         sqlite3_free(pCsr->pRec);
 682         pCsr->aPage = 0;
 683         pCsr->pRec = 0;
 684         if( pCsr->bOnePage ) return SQLITE_OK;
 685         pCsr->iPgno++;
 686       }else{
 687         if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){
 688           return SQLITE_OK;
 689         }
 690
 691         /* Advance to the next cell. The next iteration of the loop will load
 692         ** the record and so on. */
 693         sqlite3_free(pCsr->pRec);
 694         pCsr->pRec = 0;
 695         pCsr->iCell++;
 696       }
 697     }
 698   }
 699
 700   assert( !"can't get here" );
 701   return SQLITE_OK;
 702 }
 703
 704 /*
 705 ** Return true if the cursor is at EOF.
 706 */
 707 static int dbdataEof(sqlite3_vtab_cursor *pCursor){
 708   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 709   return pCsr->aPage==0;
 710 }
 711
 712 /*
 713 ** Return true if nul-terminated string zSchema ends in "()". Or false
 714 ** otherwise.
 715 */
 716 static int dbdataIsFunction(const char *zSchema){
 717   size_t n = strlen(zSchema);
 718   if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
 719     return (int)n-2;
 720   }
 721   return 0;
 722 }
 723
 724 /*
 725 ** Determine the size in pages of database zSchema (where zSchema is
 726 ** "main", "temp" or the name of an attached database) and set
 727 ** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise,
 728 ** an SQLite error code.
 729 */
 730 static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
 731   DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
 732   char *zSql = 0;
 733   int rc, rc2;
 734   int nFunc = 0;
 735   sqlite3_stmt *pStmt = 0;
 736
 737   if( (nFunc = dbdataIsFunction(zSchema))>0 ){
 738     zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
 739   }else{
 740     zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
 741   }
 742   if( zSql==0 ) return SQLITE_NOMEM;
 743
 744   rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
 745   sqlite3_free(zSql);
 746   if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
 747     pCsr->szDb = sqlite3_column_int(pStmt, 0);
 748   }
 749   rc2 = sqlite3_finalize(pStmt);
 750   if( rc==SQLITE_OK ) rc = rc2;
 751   return rc;
 752 }
 753
 754 /*
 755 ** Attempt to figure out the encoding of the database by retrieving page 1
 756 ** and inspecting the header field. If successful, set the pCsr->enc variable
 757 ** and return SQLITE_OK. Otherwise, return an SQLite error code.
 758 */
 759 static int dbdataGetEncoding(DbdataCursor *pCsr){
 760   int rc = SQLITE_OK;
 761   int nPg1 = 0;
 762   u8 *aPg1 = 0;
 763   rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
 764   if( rc==SQLITE_OK && nPg1>=(56+4) ){
 765     pCsr->enc = get_uint32(&aPg1[56]);
 766   }
 767   sqlite3_free(aPg1);
 768   return rc;
 769 }
 770
 771
 772 /*
 773 ** xFilter method for sqlite_dbdata and sqlite_dbptr.
 774 */
 775 static int dbdataFilter(
 776   sqlite3_vtab_cursor *pCursor,
 777   int idxNum, const char *idxStr,
 778   int argc, sqlite3_value **argv
 779 ){
 780   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 781   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 782   int rc = SQLITE_OK;
 783   const char *zSchema = "main";
 784   (void)idxStr;
 785   (void)argc;
 786
 787   dbdataResetCursor(pCsr);
 788   assert( pCsr->iPgno==1 );
 789   if( idxNum & 0x01 ){
 790     zSchema = (const char*)sqlite3_value_text(argv[0]);
 791     if( zSchema==0 ) zSchema = "";
 792   }
 793   if( idxNum & 0x02 ){
 794     pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
 795     pCsr->bOnePage = 1;
 796   }else{
 797     rc = dbdataDbsize(pCsr, zSchema);
 798   }
 799
 800   if( rc==SQLITE_OK ){
 801     int nFunc = 0;
 802     if( pTab->pStmt ){
 803       pCsr->pStmt = pTab->pStmt;
 804       pTab->pStmt = 0;
 805     }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
 806       char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
 807       if( zSql==0 ){
 808         rc = SQLITE_NOMEM;
 809       }else{
 810         rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
 811         sqlite3_free(zSql);
 812       }
 813     }else{
 814       rc = sqlite3_prepare_v2(pTab->db,
 815           "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
 816           &pCsr->pStmt, 0
 817       );
 818     }
 819   }
 820   if( rc==SQLITE_OK ){
 821     rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
 822   }
 823
 824   /* Try to determine the encoding of the db by inspecting the header
 825   ** field on page 1. */
 826   if( rc==SQLITE_OK ){
 827     rc = dbdataGetEncoding(pCsr);
 828   }
 829
 830   if( rc!=SQLITE_OK ){
 831     pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
 832   }
 833
 834   if( rc==SQLITE_OK ){
 835     rc = dbdataNext(pCursor);
 836   }
 837   return rc;
 838 }
 839
 840 /*
 841 ** Return a column for the sqlite_dbdata or sqlite_dbptr table.
 842 */
 843 static int dbdataColumn(
 844   sqlite3_vtab_cursor *pCursor,
 845   sqlite3_context *ctx,
 846   int i
 847 ){
 848   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 849   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 850   if( pTab->bPtr ){
 851     switch( i ){
 852       case DBPTR_COLUMN_PGNO:
 853         sqlite3_result_int64(ctx, pCsr->iPgno);
 854         break;
 855       case DBPTR_COLUMN_CHILD: {
 856         int iOff = pCsr->iPgno==1 ? 100 : 0;
 857         if( pCsr->iCell<0 ){
 858           iOff += 8;
 859         }else{
 860           iOff += 12 + pCsr->iCell*2;
 861           if( iOff>pCsr->nPage ) return SQLITE_OK;
 862           iOff = get_uint16(&pCsr->aPage[iOff]);
 863         }
 864         if( iOff<=pCsr->nPage ){
 865           sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff]));
 866         }
 867         break;
 868       }
 869     }
 870   }else{
 871     switch( i ){
 872       case DBDATA_COLUMN_PGNO:
 873         sqlite3_result_int64(ctx, pCsr->iPgno);
 874         break;
 875       case DBDATA_COLUMN_CELL:
 876         sqlite3_result_int(ctx, pCsr->iCell);
 877         break;
 878       case DBDATA_COLUMN_FIELD:
 879         sqlite3_result_int(ctx, pCsr->iField);
 880         break;
 881       case DBDATA_COLUMN_VALUE: {
 882         if( pCsr->iField<0 ){
 883           sqlite3_result_int64(ctx, pCsr->iIntkey);
 884         }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
 885           sqlite3_int64 iType;
 886           dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
 887           dbdataValue(
 888               ctx, pCsr->enc, iType, pCsr->pPtr,
 889               &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
 890           );
 891         }
 892         break;
 893       }
 894     }
 895   }
 896   return SQLITE_OK;
 897 }
 898
 899 /*
 900 ** Return the rowid for an sqlite_dbdata or sqlite_dptr table.
 901 */
 902 static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
 903   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 904   *pRowid = pCsr->iRowid;
 905   return SQLITE_OK;
 906 }
 907
 908
 909 /*
 910 ** Invoke this routine to register the "sqlite_dbdata" virtual table module
 911 */
 912 static int sqlite3DbdataRegister(sqlite3 *db){
 913   static sqlite3_module dbdata_module = {
 914     0,                            /* iVersion */
 915     0,                            /* xCreate */
 916     dbdataConnect,                /* xConnect */
 917     dbdataBestIndex,              /* xBestIndex */
 918     dbdataDisconnect,             /* xDisconnect */
 919     0,                            /* xDestroy */
 920     dbdataOpen,                   /* xOpen - open a cursor */
 921     dbdataClose,                  /* xClose - close a cursor */
 922     dbdataFilter,                 /* xFilter - configure scan constraints */
 923     dbdataNext,                   /* xNext - advance a cursor */
 924     dbdataEof,                    /* xEof - check for end of scan */
 925     dbdataColumn,                 /* xColumn - read data */
 926     dbdataRowid,                  /* xRowid - read data */
 927     0,                            /* xUpdate */
 928     0,                            /* xBegin */
 929     0,                            /* xSync */
 930     0,                            /* xCommit */
 931     0,                            /* xRollback */
 932     0,                            /* xFindMethod */
 933     0,                            /* xRename */
 934     0,                            /* xSavepoint */
 935     0,                            /* xRelease */
 936     0,                            /* xRollbackTo */
 937     0,                            /* xShadowName */
 938     0                             /* xIntegrity */
 939   };
 940
 941   int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0);
 942   if( rc==SQLITE_OK ){
 943     rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1);
 944   }
 945   return rc;
 946 }
 947
 948 int sqlite3_dbdata_init(
 949   sqlite3 *db,
 950   char **pzErrMsg,
 951   const sqlite3_api_routines *pApi
 952 ){
 953   (void)pzErrMsg;
 954   return sqlite3DbdataRegister(db);
 955 }
 956
 957 #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */