ext/fts5/fts5_storage.c

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14
  15
  16
  17 #include "fts5Int.h"
  18
  19 struct Fts5Storage {
  20   Fts5Config *pConfig;
  21   Fts5Index *pIndex;
  22   int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  23   i64 nTotalRow;                  /* Total number of rows in FTS table */
  24   i64 *aTotalSize;                /* Total sizes of each column */
  25   sqlite3_stmt *aStmt[11];
  26 };
  27
  28
  29 #if FTS5_STMT_SCAN_ASC!=0
  30 # error "FTS5_STMT_SCAN_ASC mismatch"
  31 #endif
  32 #if FTS5_STMT_SCAN_DESC!=1
  33 # error "FTS5_STMT_SCAN_DESC mismatch"
  34 #endif
  35 #if FTS5_STMT_LOOKUP!=2
  36 # error "FTS5_STMT_LOOKUP mismatch"
  37 #endif
  38
  39 #define FTS5_STMT_INSERT_CONTENT  3
  40 #define FTS5_STMT_REPLACE_CONTENT 4
  41 #define FTS5_STMT_DELETE_CONTENT  5
  42 #define FTS5_STMT_REPLACE_DOCSIZE  6
  43 #define FTS5_STMT_DELETE_DOCSIZE  7
  44 #define FTS5_STMT_LOOKUP_DOCSIZE  8
  45 #define FTS5_STMT_REPLACE_CONFIG 9
  46 #define FTS5_STMT_SCAN 10
  47
  48 /*
  49 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
  50 ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
  51 ** Return SQLITE_OK if successful, or an SQLite error code if an error
  52 ** occurs.
  53 */
  54 static int fts5StorageGetStmt(
  55   Fts5Storage *p,                 /* Storage handle */
  56   int eStmt,                      /* FTS5_STMT_XXX constant */
  57   sqlite3_stmt **ppStmt,          /* OUT: Prepared statement handle */
  58   char **pzErrMsg                 /* OUT: Error message (if any) */
  59 ){
  60   int rc = SQLITE_OK;
  61
  62   /* If there is no %_docsize table, there should be no requests for
  63   ** statements to operate on it.  */
  64   assert( p->pConfig->bColumnsize || (
  65         eStmt!=FTS5_STMT_REPLACE_DOCSIZE
  66      && eStmt!=FTS5_STMT_DELETE_DOCSIZE
  67      && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE
  68   ));
  69
  70   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
  71   if( p->aStmt[eStmt]==0 ){
  72     const char *azStmt[] = {
  73       "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
  74       "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
  75       "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
  76
  77       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
  78       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
  79       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
  80       "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
  81       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
  82
  83       "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
  84
  85       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
  86       "SELECT %s FROM %s AS T",                         /* SCAN */
  87     };
  88     Fts5Config *pC = p->pConfig;
  89     char *zSql = 0;
  90
  91     switch( eStmt ){
  92       case FTS5_STMT_SCAN:
  93         zSql = sqlite3_mprintf(azStmt[eStmt],
  94             pC->zContentExprlist, pC->zContent
  95         );
  96         break;
  97
  98       case FTS5_STMT_SCAN_ASC:
  99       case FTS5_STMT_SCAN_DESC:
 100         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist,
 101             pC->zContent, pC->zContentRowid, pC->zContentRowid,
 102             pC->zContentRowid
 103         );
 104         break;
 105
 106       case FTS5_STMT_LOOKUP:
 107         zSql = sqlite3_mprintf(azStmt[eStmt],
 108             pC->zContentExprlist, pC->zContent, pC->zContentRowid
 109         );
 110         break;
 111
 112       case FTS5_STMT_INSERT_CONTENT:
 113       case FTS5_STMT_REPLACE_CONTENT: {
 114         int nCol = pC->nCol + 1;
 115         char *zBind;
 116         int i;
 117
 118         zBind = sqlite3_malloc(1 + nCol*2);
 119         if( zBind ){
 120           for(i=0; i<nCol; i++){
 121             zBind[i*2] = '?';
 122             zBind[i*2 + 1] = ',';
 123           }
 124           zBind[i*2-1] = '\0';
 125           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
 126           sqlite3_free(zBind);
 127         }
 128         break;
 129       }
 130
 131       default:
 132         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
 133         break;
 134     }
 135
 136     if( zSql==0 ){
 137       rc = SQLITE_NOMEM;
 138     }else{
 139       rc = sqlite3_prepare_v3(pC->db, zSql, -1,
 140                               SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
 141       sqlite3_free(zSql);
 142       if( rc!=SQLITE_OK && pzErrMsg ){
 143         *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
 144       }
 145     }
 146   }
 147
 148   *ppStmt = p->aStmt[eStmt];
 149   sqlite3_reset(*ppStmt);
 150   return rc;
 151 }
 152
 153
 154 static int fts5ExecPrintf(
 155   sqlite3 *db,
 156   char **pzErr,
 157   const char *zFormat,
 158   ...
 159 ){
 160   int rc;
 161   va_list ap;                     /* ... printf arguments */
 162   char *zSql;
 163
 164   va_start(ap, zFormat);
 165   zSql = sqlite3_vmprintf(zFormat, ap);
 166
 167   if( zSql==0 ){
 168     rc = SQLITE_NOMEM;
 169   }else{
 170     rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
 171     sqlite3_free(zSql);
 172   }
 173
 174   va_end(ap);
 175   return rc;
 176 }
 177
 178 /*
 179 ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
 180 ** code otherwise.
 181 */
 182 int sqlite3Fts5DropAll(Fts5Config *pConfig){
 183   int rc = fts5ExecPrintf(pConfig->db, 0,
 184       "DROP TABLE IF EXISTS %Q.'%q_data';"
 185       "DROP TABLE IF EXISTS %Q.'%q_idx';"
 186       "DROP TABLE IF EXISTS %Q.'%q_config';",
 187       pConfig->zDb, pConfig->zName,
 188       pConfig->zDb, pConfig->zName,
 189       pConfig->zDb, pConfig->zName
 190   );
 191   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 192     rc = fts5ExecPrintf(pConfig->db, 0,
 193         "DROP TABLE IF EXISTS %Q.'%q_docsize';",
 194         pConfig->zDb, pConfig->zName
 195     );
 196   }
 197   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
 198     rc = fts5ExecPrintf(pConfig->db, 0,
 199         "DROP TABLE IF EXISTS %Q.'%q_content';",
 200         pConfig->zDb, pConfig->zName
 201     );
 202   }
 203   return rc;
 204 }
 205
 206 static void fts5StorageRenameOne(
 207   Fts5Config *pConfig,            /* Current FTS5 configuration */
 208   int *pRc,                       /* IN/OUT: Error code */
 209   const char *zTail,              /* Tail of table name e.g. "data", "config" */
 210   const char *zName               /* New name of FTS5 table */
 211 ){
 212   if( *pRc==SQLITE_OK ){
 213     *pRc = fts5ExecPrintf(pConfig->db, 0,
 214         "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
 215         pConfig->zDb, pConfig->zName, zTail, zName, zTail
 216     );
 217   }
 218 }
 219
 220 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
 221   Fts5Config *pConfig = pStorage->pConfig;
 222   int rc = sqlite3Fts5StorageSync(pStorage);
 223
 224   fts5StorageRenameOne(pConfig, &rc, "data", zName);
 225   fts5StorageRenameOne(pConfig, &rc, "idx", zName);
 226   fts5StorageRenameOne(pConfig, &rc, "config", zName);
 227   if( pConfig->bColumnsize ){
 228     fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
 229   }
 230   if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 231     fts5StorageRenameOne(pConfig, &rc, "content", zName);
 232   }
 233   return rc;
 234 }
 235
 236 /*
 237 ** Create the shadow table named zPost, with definition zDefn. Return
 238 ** SQLITE_OK if successful, or an SQLite error code otherwise.
 239 */
 240 int sqlite3Fts5CreateTable(
 241   Fts5Config *pConfig,            /* FTS5 configuration */
 242   const char *zPost,              /* Shadow table to create (e.g. "content") */
 243   const char *zDefn,              /* Columns etc. for shadow table */
 244   int bWithout,                   /* True for without rowid */
 245   char **pzErr                    /* OUT: Error message */
 246 ){
 247   int rc;
 248   char *zErr = 0;
 249
 250   rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
 251       pConfig->zDb, pConfig->zName, zPost, zDefn,
 252 #ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
 253       bWithout?" WITHOUT ROWID":
 254 #endif
 255       ""
 256   );
 257   if( zErr ){
 258     *pzErr = sqlite3_mprintf(
 259         "fts5: error creating shadow table %q_%s: %s",
 260         pConfig->zName, zPost, zErr
 261     );
 262     sqlite3_free(zErr);
 263   }
 264
 265   return rc;
 266 }
 267
 268 /*
 269 ** Open a new Fts5Index handle. If the bCreate argument is true, create
 270 ** and initialize the underlying tables
 271 **
 272 ** If successful, set *pp to point to the new object and return SQLITE_OK.
 273 ** Otherwise, set *pp to NULL and return an SQLite error code.
 274 */
 275 int sqlite3Fts5StorageOpen(
 276   Fts5Config *pConfig,
 277   Fts5Index *pIndex,
 278   int bCreate,
 279   Fts5Storage **pp,
 280   char **pzErr                    /* OUT: Error message */
 281 ){
 282   int rc = SQLITE_OK;
 283   Fts5Storage *p;                 /* New object */
 284   int nByte;                      /* Bytes of space to allocate */
 285
 286   nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
 287         + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
 288   *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
 289   if( !p ) return SQLITE_NOMEM;
 290
 291   memset(p, 0, nByte);
 292   p->aTotalSize = (i64*)&p[1];
 293   p->pConfig = pConfig;
 294   p->pIndex = pIndex;
 295
 296   if( bCreate ){
 297     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 298       int nDefn = 32 + pConfig->nCol*10;
 299       char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
 300       if( zDefn==0 ){
 301         rc = SQLITE_NOMEM;
 302       }else{
 303         int i;
 304         int iOff;
 305         sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
 306         iOff = (int)strlen(zDefn);
 307         for(i=0; i<pConfig->nCol; i++){
 308           sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
 309           iOff += (int)strlen(&zDefn[iOff]);
 310         }
 311         rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
 312       }
 313       sqlite3_free(zDefn);
 314     }
 315
 316     if( rc==SQLITE_OK && pConfig->bColumnsize ){
 317       rc = sqlite3Fts5CreateTable(
 318           pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
 319       );
 320     }
 321     if( rc==SQLITE_OK ){
 322       rc = sqlite3Fts5CreateTable(
 323           pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
 324       );
 325     }
 326     if( rc==SQLITE_OK ){
 327       rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
 328     }
 329   }
 330
 331   if( rc ){
 332     sqlite3Fts5StorageClose(p);
 333     *pp = 0;
 334   }
 335   return rc;
 336 }
 337
 338 /*
 339 ** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
 340 */
 341 int sqlite3Fts5StorageClose(Fts5Storage *p){
 342   int rc = SQLITE_OK;
 343   if( p ){
 344     int i;
 345
 346     /* Finalize all SQL statements */
 347     for(i=0; i<ArraySize(p->aStmt); i++){
 348       sqlite3_finalize(p->aStmt[i]);
 349     }
 350
 351     sqlite3_free(p);
 352   }
 353   return rc;
 354 }
 355
 356 typedef struct Fts5InsertCtx Fts5InsertCtx;
 357 struct Fts5InsertCtx {
 358   Fts5Storage *pStorage;
 359   int iCol;
 360   int szCol;                      /* Size of column value in tokens */
 361 };
 362
 363 /*
 364 ** Tokenization callback used when inserting tokens into the FTS index.
 365 */
 366 static int fts5StorageInsertCallback(
 367   void *pContext,                 /* Pointer to Fts5InsertCtx object */
 368   int tflags,
 369   const char *pToken,             /* Buffer containing token */
 370   int nToken,                     /* Size of token in bytes */
 371   int iUnused1,                   /* Start offset of token */
 372   int iUnused2                    /* End offset of token */
 373 ){
 374   Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
 375   Fts5Index *pIdx = pCtx->pStorage->pIndex;
 376   UNUSED_PARAM2(iUnused1, iUnused2);
 377   if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
 378   if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
 379     pCtx->szCol++;
 380   }
 381   return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 382 }
 383
 384 /*
 385 ** If a row with rowid iDel is present in the %_content table, add the
 386 ** delete-markers to the FTS index necessary to delete it. Do not actually
 387 ** remove the %_content row at this time though.
 388 */
 389 static int fts5StorageDeleteFromIndex(
 390   Fts5Storage *p,
 391   i64 iDel,
 392   sqlite3_value **apVal
 393 ){
 394   Fts5Config *pConfig = p->pConfig;
 395   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
 396   int rc;                         /* Return code */
 397   int rc2;                        /* sqlite3_reset() return code */
 398   int iCol;
 399   Fts5InsertCtx ctx;
 400
 401   if( apVal==0 ){
 402     rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
 403     if( rc!=SQLITE_OK ) return rc;
 404     sqlite3_bind_int64(pSeek, 1, iDel);
 405     if( sqlite3_step(pSeek)!=SQLITE_ROW ){
 406       return sqlite3_reset(pSeek);
 407     }
 408   }
 409
 410   ctx.pStorage = p;
 411   ctx.iCol = -1;
 412   rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
 413   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
 414     if( pConfig->abUnindexed[iCol-1]==0 ){
 415       const char *zText;
 416       int nText;
 417       if( pSeek ){
 418         zText = (const char*)sqlite3_column_text(pSeek, iCol);
 419         nText = sqlite3_column_bytes(pSeek, iCol);
 420       }else{
 421         zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
 422         nText = sqlite3_value_bytes(apVal[iCol-1]);
 423       }
 424       ctx.szCol = 0;
 425       rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
 426           zText, nText, (void*)&ctx, fts5StorageInsertCallback
 427       );
 428       p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
 429     }
 430   }
 431   p->nTotalRow--;
 432
 433   rc2 = sqlite3_reset(pSeek);
 434   if( rc==SQLITE_OK ) rc = rc2;
 435   return rc;
 436 }
 437
 438
 439 /*
 440 ** Insert a record into the %_docsize table. Specifically, do:
 441 **
 442 **   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
 443 **
 444 ** If there is no %_docsize table (as happens if the columnsize=0 option
 445 ** is specified when the FTS5 table is created), this function is a no-op.
 446 */
 447 static int fts5StorageInsertDocsize(
 448   Fts5Storage *p,                 /* Storage module to write to */
 449   i64 iRowid,                     /* id value */
 450   Fts5Buffer *pBuf                /* sz value */
 451 ){
 452   int rc = SQLITE_OK;
 453   if( p->pConfig->bColumnsize ){
 454     sqlite3_stmt *pReplace = 0;
 455     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
 456     if( rc==SQLITE_OK ){
 457       sqlite3_bind_int64(pReplace, 1, iRowid);
 458       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
 459       sqlite3_step(pReplace);
 460       rc = sqlite3_reset(pReplace);
 461     }
 462   }
 463   return rc;
 464 }
 465
 466 /*
 467 ** Load the contents of the "averages" record from disk into the
 468 ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
 469 ** argument bCache is true, set the p->bTotalsValid flag to indicate
 470 ** that the contents of aTotalSize[] and nTotalRow are valid until
 471 ** further notice.
 472 **
 473 ** Return SQLITE_OK if successful, or an SQLite error code if an error
 474 ** occurs.
 475 */
 476 static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
 477   int rc = SQLITE_OK;
 478   if( p->bTotalsValid==0 ){
 479     rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
 480     p->bTotalsValid = bCache;
 481   }
 482   return rc;
 483 }
 484
 485 /*
 486 ** Store the current contents of the p->nTotalRow and p->aTotalSize[]
 487 ** variables in the "averages" record on disk.
 488 **
 489 ** Return SQLITE_OK if successful, or an SQLite error code if an error
 490 ** occurs.
 491 */
 492 static int fts5StorageSaveTotals(Fts5Storage *p){
 493   int nCol = p->pConfig->nCol;
 494   int i;
 495   Fts5Buffer buf;
 496   int rc = SQLITE_OK;
 497   memset(&buf, 0, sizeof(buf));
 498
 499   sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
 500   for(i=0; i<nCol; i++){
 501     sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
 502   }
 503   if( rc==SQLITE_OK ){
 504     rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
 505   }
 506   sqlite3_free(buf.p);
 507
 508   return rc;
 509 }
 510
 511 /*
 512 ** Remove a row from the FTS table.
 513 */
 514 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 515   Fts5Config *pConfig = p->pConfig;
 516   int rc;
 517   sqlite3_stmt *pDel = 0;
 518
 519   assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
 520   rc = fts5StorageLoadTotals(p, 1);
 521
 522   /* Delete the index records */
 523   if( rc==SQLITE_OK ){
 524     rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
 525   }
 526
 527   /* Delete the %_docsize record */
 528   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 529     rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
 530     if( rc==SQLITE_OK ){
 531       sqlite3_bind_int64(pDel, 1, iDel);
 532       sqlite3_step(pDel);
 533       rc = sqlite3_reset(pDel);
 534     }
 535   }
 536
 537   /* Delete the %_content record */
 538   if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 539     if( rc==SQLITE_OK ){
 540       rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
 541     }
 542     if( rc==SQLITE_OK ){
 543       sqlite3_bind_int64(pDel, 1, iDel);
 544       sqlite3_step(pDel);
 545       rc = sqlite3_reset(pDel);
 546     }
 547   }
 548
 549   return rc;
 550 }
 551
 552 /*
 553 ** Delete all entries in the FTS5 index.
 554 */
 555 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
 556   Fts5Config *pConfig = p->pConfig;
 557   int rc;
 558
 559   /* Delete the contents of the %_data and %_docsize tables. */
 560   rc = fts5ExecPrintf(pConfig->db, 0,
 561       "DELETE FROM %Q.'%q_data';"
 562       "DELETE FROM %Q.'%q_idx';",
 563       pConfig->zDb, pConfig->zName,
 564       pConfig->zDb, pConfig->zName
 565   );
 566   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 567     rc = fts5ExecPrintf(pConfig->db, 0,
 568         "DELETE FROM %Q.'%q_docsize';",
 569         pConfig->zDb, pConfig->zName
 570     );
 571   }
 572
 573   /* Reinitialize the %_data table. This call creates the initial structure
 574   ** and averages records.  */
 575   if( rc==SQLITE_OK ){
 576     rc = sqlite3Fts5IndexReinit(p->pIndex);
 577   }
 578   if( rc==SQLITE_OK ){
 579     rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
 580   }
 581   return rc;
 582 }
 583
 584 int sqlite3Fts5StorageRebuild(Fts5Storage *p){
 585   Fts5Buffer buf = {0,0,0};
 586   Fts5Config *pConfig = p->pConfig;
 587   sqlite3_stmt *pScan = 0;
 588   Fts5InsertCtx ctx;
 589   int rc;
 590
 591   memset(&ctx, 0, sizeof(Fts5InsertCtx));
 592   ctx.pStorage = p;
 593   rc = sqlite3Fts5StorageDeleteAll(p);
 594   if( rc==SQLITE_OK ){
 595     rc = fts5StorageLoadTotals(p, 1);
 596   }
 597
 598   if( rc==SQLITE_OK ){
 599     rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
 600   }
 601
 602   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
 603     i64 iRowid = sqlite3_column_int64(pScan, 0);
 604
 605     sqlite3Fts5BufferZero(&buf);
 606     rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
 607     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
 608       ctx.szCol = 0;
 609       if( pConfig->abUnindexed[ctx.iCol]==0 ){
 610         rc = sqlite3Fts5Tokenize(pConfig,
 611             FTS5_TOKENIZE_DOCUMENT,
 612             (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
 613             sqlite3_column_bytes(pScan, ctx.iCol+1),
 614             (void*)&ctx,
 615             fts5StorageInsertCallback
 616         );
 617       }
 618       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
 619       p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
 620     }
 621     p->nTotalRow++;
 622
 623     if( rc==SQLITE_OK ){
 624       rc = fts5StorageInsertDocsize(p, iRowid, &buf);
 625     }
 626   }
 627   sqlite3_free(buf.p);
 628
 629   /* Write the averages record */
 630   if( rc==SQLITE_OK ){
 631     rc = fts5StorageSaveTotals(p);
 632   }
 633   return rc;
 634 }
 635
 636 int sqlite3Fts5StorageOptimize(Fts5Storage *p){
 637   return sqlite3Fts5IndexOptimize(p->pIndex);
 638 }
 639
 640 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
 641   return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
 642 }
 643
 644 int sqlite3Fts5StorageReset(Fts5Storage *p){
 645   return sqlite3Fts5IndexReset(p->pIndex);
 646 }
 647
 648 /*
 649 ** Allocate a new rowid. This is used for "external content" tables when
 650 ** a NULL value is inserted into the rowid column. The new rowid is allocated
 651 ** by inserting a dummy row into the %_docsize table. The dummy will be
 652 ** overwritten later.
 653 **
 654 ** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
 655 ** this case the user is required to provide a rowid explicitly.
 656 */
 657 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
 658   int rc = SQLITE_MISMATCH;
 659   if( p->pConfig->bColumnsize ){
 660     sqlite3_stmt *pReplace = 0;
 661     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
 662     if( rc==SQLITE_OK ){
 663       sqlite3_bind_null(pReplace, 1);
 664       sqlite3_bind_null(pReplace, 2);
 665       sqlite3_step(pReplace);
 666       rc = sqlite3_reset(pReplace);
 667     }
 668     if( rc==SQLITE_OK ){
 669       *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
 670     }
 671   }
 672   return rc;
 673 }
 674
 675 /*
 676 ** Insert a new row into the FTS content table.
 677 */
 678 int sqlite3Fts5StorageContentInsert(
 679   Fts5Storage *p,
 680   sqlite3_value **apVal,
 681   i64 *piRowid
 682 ){
 683   Fts5Config *pConfig = p->pConfig;
 684   int rc = SQLITE_OK;
 685
 686   /* Insert the new row into the %_content table. */
 687   if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
 688     if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
 689       *piRowid = sqlite3_value_int64(apVal[1]);
 690     }else{
 691       rc = fts5StorageNewRowid(p, piRowid);
 692     }
 693   }else{
 694     sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
 695     int i;                        /* Counter variable */
 696     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
 697     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
 698       rc = sqlite3_bind_value(pInsert, i, apVal[i]);
 699     }
 700     if( rc==SQLITE_OK ){
 701       sqlite3_step(pInsert);
 702       rc = sqlite3_reset(pInsert);
 703     }
 704     *piRowid = sqlite3_last_insert_rowid(pConfig->db);
 705   }
 706
 707   return rc;
 708 }
 709
 710 /*
 711 ** Insert new entries into the FTS index and %_docsize table.
 712 */
 713 int sqlite3Fts5StorageIndexInsert(
 714   Fts5Storage *p,
 715   sqlite3_value **apVal,
 716   i64 iRowid
 717 ){
 718   Fts5Config *pConfig = p->pConfig;
 719   int rc = SQLITE_OK;             /* Return code */
 720   Fts5InsertCtx ctx;              /* Tokenization callback context object */
 721   Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
 722
 723   memset(&buf, 0, sizeof(Fts5Buffer));
 724   ctx.pStorage = p;
 725   rc = fts5StorageLoadTotals(p, 1);
 726
 727   if( rc==SQLITE_OK ){
 728     rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
 729   }
 730   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
 731     ctx.szCol = 0;
 732     if( pConfig->abUnindexed[ctx.iCol]==0 ){
 733       rc = sqlite3Fts5Tokenize(pConfig,
 734           FTS5_TOKENIZE_DOCUMENT,
 735           (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
 736           sqlite3_value_bytes(apVal[ctx.iCol+2]),
 737           (void*)&ctx,
 738           fts5StorageInsertCallback
 739       );
 740     }
 741     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
 742     p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
 743   }
 744   p->nTotalRow++;
 745
 746   /* Write the %_docsize record */
 747   if( rc==SQLITE_OK ){
 748     rc = fts5StorageInsertDocsize(p, iRowid, &buf);
 749   }
 750   sqlite3_free(buf.p);
 751
 752   return rc;
 753 }
 754
 755 static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
 756   Fts5Config *pConfig = p->pConfig;
 757   char *zSql;
 758   int rc;
 759
 760   zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'",
 761       pConfig->zDb, pConfig->zName, zSuffix
 762   );
 763   if( zSql==0 ){
 764     rc = SQLITE_NOMEM;
 765   }else{
 766     sqlite3_stmt *pCnt = 0;
 767     rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
 768     if( rc==SQLITE_OK ){
 769       if( SQLITE_ROW==sqlite3_step(pCnt) ){
 770         *pnRow = sqlite3_column_int64(pCnt, 0);
 771       }
 772       rc = sqlite3_finalize(pCnt);
 773     }
 774   }
 775
 776   sqlite3_free(zSql);
 777   return rc;
 778 }
 779
 780 /*
 781 ** Context object used by sqlite3Fts5StorageIntegrity().
 782 */
 783 typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
 784 struct Fts5IntegrityCtx {
 785   i64 iRowid;
 786   int iCol;
 787   int szCol;
 788   u64 cksum;
 789   Fts5Termset *pTermset;
 790   Fts5Config *pConfig;
 791 };
 792
 793
 794 /*
 795 ** Tokenization callback used by integrity check.
 796 */
 797 static int fts5StorageIntegrityCallback(
 798   void *pContext,                 /* Pointer to Fts5IntegrityCtx object */
 799   int tflags,
 800   const char *pToken,             /* Buffer containing token */
 801   int nToken,                     /* Size of token in bytes */
 802   int iUnused1,                   /* Start offset of token */
 803   int iUnused2                    /* End offset of token */
 804 ){
 805   Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
 806   Fts5Termset *pTermset = pCtx->pTermset;
 807   int bPresent;
 808   int ii;
 809   int rc = SQLITE_OK;
 810   int iPos;
 811   int iCol;
 812
 813   UNUSED_PARAM2(iUnused1, iUnused2);
 814   if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
 815
 816   if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
 817     pCtx->szCol++;
 818   }
 819
 820   switch( pCtx->pConfig->eDetail ){
 821     case FTS5_DETAIL_FULL:
 822       iPos = pCtx->szCol-1;
 823       iCol = pCtx->iCol;
 824       break;
 825
 826     case FTS5_DETAIL_COLUMNS:
 827       iPos = pCtx->iCol;
 828       iCol = 0;
 829       break;
 830
 831     default:
 832       assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
 833       iPos = 0;
 834       iCol = 0;
 835       break;
 836   }
 837
 838   rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
 839   if( rc==SQLITE_OK && bPresent==0 ){
 840     pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
 841         pCtx->iRowid, iCol, iPos, 0, pToken, nToken
 842     );
 843   }
 844
 845   for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
 846     const int nChar = pCtx->pConfig->aPrefix[ii];
 847     int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
 848     if( nByte ){
 849       rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
 850       if( bPresent==0 ){
 851         pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
 852             pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
 853         );
 854       }
 855     }
 856   }
 857
 858   return rc;
 859 }
 860
 861 /*
 862 ** Check that the contents of the FTS index match that of the %_content
 863 ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
 864 ** some other SQLite error code if an error occurs while attempting to
 865 ** determine this.
 866 */
 867 int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
 868   Fts5Config *pConfig = p->pConfig;
 869   int rc;                         /* Return code */
 870   int *aColSize;                  /* Array of size pConfig->nCol */
 871   i64 *aTotalSize;                /* Array of size pConfig->nCol */
 872   Fts5IntegrityCtx ctx;
 873   sqlite3_stmt *pScan;
 874
 875   memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
 876   ctx.pConfig = p->pConfig;
 877   aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
 878   if( !aTotalSize ) return SQLITE_NOMEM;
 879   aColSize = (int*)&aTotalSize[pConfig->nCol];
 880   memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
 881
 882   /* Generate the expected index checksum based on the contents of the
 883   ** %_content table. This block stores the checksum in ctx.cksum. */
 884   rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
 885   if( rc==SQLITE_OK ){
 886     int rc2;
 887     while( SQLITE_ROW==sqlite3_step(pScan) ){
 888       int i;
 889       ctx.iRowid = sqlite3_column_int64(pScan, 0);
 890       ctx.szCol = 0;
 891       if( pConfig->bColumnsize ){
 892         rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
 893       }
 894       if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
 895         rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
 896       }
 897       for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
 898         if( pConfig->abUnindexed[i] ) continue;
 899         ctx.iCol = i;
 900         ctx.szCol = 0;
 901         if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
 902           rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
 903         }
 904         if( rc==SQLITE_OK ){
 905           rc = sqlite3Fts5Tokenize(pConfig,
 906               FTS5_TOKENIZE_DOCUMENT,
 907               (const char*)sqlite3_column_text(pScan, i+1),
 908               sqlite3_column_bytes(pScan, i+1),
 909               (void*)&ctx,
 910               fts5StorageIntegrityCallback
 911           );
 912         }
 913         if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
 914           rc = FTS5_CORRUPT;
 915         }
 916         aTotalSize[i] += ctx.szCol;
 917         if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
 918           sqlite3Fts5TermsetFree(ctx.pTermset);
 919           ctx.pTermset = 0;
 920         }
 921       }
 922       sqlite3Fts5TermsetFree(ctx.pTermset);
 923       ctx.pTermset = 0;
 924
 925       if( rc!=SQLITE_OK ) break;
 926     }
 927     rc2 = sqlite3_reset(pScan);
 928     if( rc==SQLITE_OK ) rc = rc2;
 929   }
 930
 931   /* Test that the "totals" (sometimes called "averages") record looks Ok */
 932   if( rc==SQLITE_OK ){
 933     int i;
 934     rc = fts5StorageLoadTotals(p, 0);
 935     for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
 936       if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT;
 937     }
 938   }
 939
 940   /* Check that the %_docsize and %_content tables contain the expected
 941   ** number of rows.  */
 942   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
 943     i64 nRow = 0;
 944     rc = fts5StorageCount(p, "content", &nRow);
 945     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
 946   }
 947   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 948     i64 nRow = 0;
 949     rc = fts5StorageCount(p, "docsize", &nRow);
 950     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
 951   }
 952
 953   /* Pass the expected checksum down to the FTS index module. It will
 954   ** verify, amongst other things, that it matches the checksum generated by
 955   ** inspecting the index itself.  */
 956   if( rc==SQLITE_OK ){
 957     rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
 958   }
 959
 960   sqlite3_free(aTotalSize);
 961   return rc;
 962 }
 963
 964 /*
 965 ** Obtain an SQLite statement handle that may be used to read data from the
 966 ** %_content table.
 967 */
 968 int sqlite3Fts5StorageStmt(
 969   Fts5Storage *p,
 970   int eStmt,
 971   sqlite3_stmt **pp,
 972   char **pzErrMsg
 973 ){
 974   int rc;
 975   assert( eStmt==FTS5_STMT_SCAN_ASC
 976        || eStmt==FTS5_STMT_SCAN_DESC
 977        || eStmt==FTS5_STMT_LOOKUP
 978   );
 979   rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
 980   if( rc==SQLITE_OK ){
 981     assert( p->aStmt[eStmt]==*pp );
 982     p->aStmt[eStmt] = 0;
 983   }
 984   return rc;
 985 }
 986
 987 /*
 988 ** Release an SQLite statement handle obtained via an earlier call to
 989 ** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
 990 ** must match that passed to the sqlite3Fts5StorageStmt() call.
 991 */
 992 void sqlite3Fts5StorageStmtRelease(
 993   Fts5Storage *p,
 994   int eStmt,
 995   sqlite3_stmt *pStmt
 996 ){
 997   assert( eStmt==FTS5_STMT_SCAN_ASC
 998        || eStmt==FTS5_STMT_SCAN_DESC
 999        || eStmt==FTS5_STMT_LOOKUP
1000   );
1001   if( p->aStmt[eStmt]==0 ){
1002     sqlite3_reset(pStmt);
1003     p->aStmt[eStmt] = pStmt;
1004   }else{
1005     sqlite3_finalize(pStmt);
1006   }
1007 }
1008
1009 static int fts5StorageDecodeSizeArray(
1010   int *aCol, int nCol,            /* Array to populate */
1011   const u8 *aBlob, int nBlob      /* Record to read varints from */
1012 ){
1013   int i;
1014   int iOff = 0;
1015   for(i=0; i<nCol; i++){
1016     if( iOff>=nBlob ) return 1;
1017     iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]);
1018   }
1019   return (iOff!=nBlob);
1020 }
1021
1022 /*
1023 ** Argument aCol points to an array of integers containing one entry for
1024 ** each table column. This function reads the %_docsize record for the
1025 ** specified rowid and populates aCol[] with the results.
1026 **
1027 ** An SQLite error code is returned if an error occurs, or SQLITE_OK
1028 ** otherwise.
1029 */
1030 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
1031   int nCol = p->pConfig->nCol;    /* Number of user columns in table */
1032   sqlite3_stmt *pLookup = 0;      /* Statement to query %_docsize */
1033   int rc;                         /* Return Code */
1034
1035   assert( p->pConfig->bColumnsize );
1036   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
1037   if( rc==SQLITE_OK ){
1038     int bCorrupt = 1;
1039     sqlite3_bind_int64(pLookup, 1, iRowid);
1040     if( SQLITE_ROW==sqlite3_step(pLookup) ){
1041       const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
1042       int nBlob = sqlite3_column_bytes(pLookup, 0);
1043       if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
1044         bCorrupt = 0;
1045       }
1046     }
1047     rc = sqlite3_reset(pLookup);
1048     if( bCorrupt && rc==SQLITE_OK ){
1049       rc = FTS5_CORRUPT;
1050     }
1051   }
1052
1053   return rc;
1054 }
1055
1056 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
1057   int rc = fts5StorageLoadTotals(p, 0);
1058   if( rc==SQLITE_OK ){
1059     *pnToken = 0;
1060     if( iCol<0 ){
1061       int i;
1062       for(i=0; i<p->pConfig->nCol; i++){
1063         *pnToken += p->aTotalSize[i];
1064       }
1065     }else if( iCol<p->pConfig->nCol ){
1066       *pnToken = p->aTotalSize[iCol];
1067     }else{
1068       rc = SQLITE_RANGE;
1069     }
1070   }
1071   return rc;
1072 }
1073
1074 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
1075   int rc = fts5StorageLoadTotals(p, 0);
1076   if( rc==SQLITE_OK ){
1077     *pnRow = p->nTotalRow;
1078   }
1079   return rc;
1080 }
1081
1082 /*
1083 ** Flush any data currently held in-memory to disk.
1084 */
1085 int sqlite3Fts5StorageSync(Fts5Storage *p){
1086   int rc = SQLITE_OK;
1087   i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
1088   if( p->bTotalsValid ){
1089     rc = fts5StorageSaveTotals(p);
1090     p->bTotalsValid = 0;
1091   }
1092   if( rc==SQLITE_OK ){
1093     rc = sqlite3Fts5IndexSync(p->pIndex);
1094   }
1095   sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
1096   return rc;
1097 }
1098
1099 int sqlite3Fts5StorageRollback(Fts5Storage *p){
1100   p->bTotalsValid = 0;
1101   return sqlite3Fts5IndexRollback(p->pIndex);
1102 }
1103
1104 int sqlite3Fts5StorageConfigValue(
1105   Fts5Storage *p,
1106   const char *z,
1107   sqlite3_value *pVal,
1108   int iVal
1109 ){
1110   sqlite3_stmt *pReplace = 0;
1111   int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
1112   if( rc==SQLITE_OK ){
1113     sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
1114     if( pVal ){
1115       sqlite3_bind_value(pReplace, 2, pVal);
1116     }else{
1117       sqlite3_bind_int(pReplace, 2, iVal);
1118     }
1119     sqlite3_step(pReplace);
1120     rc = sqlite3_reset(pReplace);
1121   }
1122   if( rc==SQLITE_OK && pVal ){
1123     int iNew = p->pConfig->iCookie + 1;
1124     rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
1125     if( rc==SQLITE_OK ){
1126       p->pConfig->iCookie = iNew;
1127     }
1128   }
1129   return rc;
1130 }