ext/fts5/fts5_vocab.c

   1 /*
   2 ** 2015 May 08
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This is an SQLite virtual table module implementing direct access to an
  14 ** existing FTS5 index. The module may create several different types of
  15 ** tables:
  16 **
  17 ** col:
  18 **     CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
  19 **
  20 **   One row for each term/column combination. The value of $doc is set to
  21 **   the number of fts5 rows that contain at least one instance of term
  22 **   $term within column $col. Field $cnt is set to the total number of
  23 **   instances of term $term in column $col (in any row of the fts5 table).
  24 **
  25 ** row:
  26 **     CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
  27 **
  28 **   One row for each term in the database. The value of $doc is set to
  29 **   the number of fts5 rows that contain at least one instance of term
  30 **   $term. Field $cnt is set to the total number of instances of term
  31 **   $term in the database.
  32 **
  33 ** instance:
  34 **     CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
  35 **
  36 **   One row for each term instance in the database.
  37 */
  38
  39
  40 #include "fts5Int.h"
  41
  42
  43 typedef struct Fts5VocabTable Fts5VocabTable;
  44 typedef struct Fts5VocabCursor Fts5VocabCursor;
  45
  46 struct Fts5VocabTable {
  47   sqlite3_vtab base;
  48   char *zFts5Tbl;                 /* Name of fts5 table */
  49   char *zFts5Db;                  /* Db containing fts5 table */
  50   sqlite3 *db;                    /* Database handle */
  51   Fts5Global *pGlobal;            /* FTS5 global object for this database */
  52   int eType;                      /* FTS5_VOCAB_COL, ROW or INSTANCE */
  53 };
  54
  55 struct Fts5VocabCursor {
  56   sqlite3_vtab_cursor base;
  57   sqlite3_stmt *pStmt;            /* Statement holding lock on pIndex */
  58   Fts5Index *pIndex;              /* Associated FTS5 index */
  59
  60   int bEof;                       /* True if this cursor is at EOF */
  61   Fts5IndexIter *pIter;           /* Term/rowid iterator object */
  62
  63   int nLeTerm;                    /* Size of zLeTerm in bytes */
  64   char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
  65
  66   /* These are used by 'col' tables only */
  67   Fts5Config *pConfig;            /* Fts5 table configuration */
  68   int iCol;
  69   i64 *aCnt;
  70   i64 *aDoc;
  71
  72   /* Output values used by all tables. */
  73   i64 rowid;                      /* This table's current rowid value */
  74   Fts5Buffer term;                /* Current value of 'term' column */
  75
  76   /* Output values Used by 'instance' tables only */
  77   i64 iInstPos;
  78   int iInstOff;
  79 };
  80
  81 #define FTS5_VOCAB_COL      0
  82 #define FTS5_VOCAB_ROW      1
  83 #define FTS5_VOCAB_INSTANCE 2
  84
  85 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
  86 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
  87 #define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"
  88
  89 /*
  90 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
  91 */
  92 #define FTS5_VOCAB_TERM_EQ 0x01
  93 #define FTS5_VOCAB_TERM_GE 0x02
  94 #define FTS5_VOCAB_TERM_LE 0x04
  95
  96
  97 /*
  98 ** Translate a string containing an fts5vocab table type to an
  99 ** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
 100 ** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
 101 ** and return SQLITE_ERROR.
 102 */
 103 static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
 104   int rc = SQLITE_OK;
 105   char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
 106   if( rc==SQLITE_OK ){
 107     sqlite3Fts5Dequote(zCopy);
 108     if( sqlite3_stricmp(zCopy, "col")==0 ){
 109       *peType = FTS5_VOCAB_COL;
 110     }else
 111
 112     if( sqlite3_stricmp(zCopy, "row")==0 ){
 113       *peType = FTS5_VOCAB_ROW;
 114     }else
 115     if( sqlite3_stricmp(zCopy, "instance")==0 ){
 116       *peType = FTS5_VOCAB_INSTANCE;
 117     }else
 118     {
 119       *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
 120       rc = SQLITE_ERROR;
 121     }
 122     sqlite3_free(zCopy);
 123   }
 124
 125   return rc;
 126 }
 127
 128
 129 /*
 130 ** The xDisconnect() virtual table method.
 131 */
 132 static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
 133   Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
 134   sqlite3_free(pTab);
 135   return SQLITE_OK;
 136 }
 137
 138 /*
 139 ** The xDestroy() virtual table method.
 140 */
 141 static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
 142   Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
 143   sqlite3_free(pTab);
 144   return SQLITE_OK;
 145 }
 146
 147 /*
 148 ** This function is the implementation of both the xConnect and xCreate
 149 ** methods of the FTS3 virtual table.
 150 **
 151 ** The argv[] array contains the following:
 152 **
 153 **   argv[0]   -> module name  ("fts5vocab")
 154 **   argv[1]   -> database name
 155 **   argv[2]   -> table name
 156 **
 157 ** then:
 158 **
 159 **   argv[3]   -> name of fts5 table
 160 **   argv[4]   -> type of fts5vocab table
 161 **
 162 ** or, for tables in the TEMP schema only.
 163 **
 164 **   argv[3]   -> name of fts5 tables database
 165 **   argv[4]   -> name of fts5 table
 166 **   argv[5]   -> type of fts5vocab table
 167 */
 168 static int fts5VocabInitVtab(
 169   sqlite3 *db,                    /* The SQLite database connection */
 170   void *pAux,                     /* Pointer to Fts5Global object */
 171   int argc,                       /* Number of elements in argv array */
 172   const char * const *argv,       /* xCreate/xConnect argument array */
 173   sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
 174   char **pzErr                    /* Write any error message here */
 175 ){
 176   const char *azSchema[] = {
 177     "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")",
 178     "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")",
 179     "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
 180   };
 181
 182   Fts5VocabTable *pRet = 0;
 183   int rc = SQLITE_OK;             /* Return code */
 184   int bDb;
 185
 186   bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
 187
 188   if( argc!=5 && bDb==0 ){
 189     *pzErr = sqlite3_mprintf("wrong number of vtable arguments");
 190     rc = SQLITE_ERROR;
 191   }else{
 192     int nByte;                      /* Bytes of space to allocate */
 193     const char *zDb = bDb ? argv[3] : argv[1];
 194     const char *zTab = bDb ? argv[4] : argv[3];
 195     const char *zType = bDb ? argv[5] : argv[4];
 196     int nDb = (int)strlen(zDb)+1;
 197     int nTab = (int)strlen(zTab)+1;
 198     int eType = 0;
 199
 200     rc = fts5VocabTableType(zType, pzErr, &eType);
 201     if( rc==SQLITE_OK ){
 202       assert( eType>=0 && eType<ArraySize(azSchema) );
 203       rc = sqlite3_declare_vtab(db, azSchema[eType]);
 204     }
 205
 206     nByte = sizeof(Fts5VocabTable) + nDb + nTab;
 207     pRet = sqlite3Fts5MallocZero(&rc, nByte);
 208     if( pRet ){
 209       pRet->pGlobal = (Fts5Global*)pAux;
 210       pRet->eType = eType;
 211       pRet->db = db;
 212       pRet->zFts5Tbl = (char*)&pRet[1];
 213       pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
 214       memcpy(pRet->zFts5Tbl, zTab, nTab);
 215       memcpy(pRet->zFts5Db, zDb, nDb);
 216       sqlite3Fts5Dequote(pRet->zFts5Tbl);
 217       sqlite3Fts5Dequote(pRet->zFts5Db);
 218     }
 219   }
 220
 221   *ppVTab = (sqlite3_vtab*)pRet;
 222   return rc;
 223 }
 224
 225
 226 /*
 227 ** The xConnect() and xCreate() methods for the virtual table. All the
 228 ** work is done in function fts5VocabInitVtab().
 229 */
 230 static int fts5VocabConnectMethod(
 231   sqlite3 *db,                    /* Database connection */
 232   void *pAux,                     /* Pointer to tokenizer hash table */
 233   int argc,                       /* Number of elements in argv array */
 234   const char * const *argv,       /* xCreate/xConnect argument array */
 235   sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
 236   char **pzErr                    /* OUT: sqlite3_malloc'd error message */
 237 ){
 238   return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
 239 }
 240 static int fts5VocabCreateMethod(
 241   sqlite3 *db,                    /* Database connection */
 242   void *pAux,                     /* Pointer to tokenizer hash table */
 243   int argc,                       /* Number of elements in argv array */
 244   const char * const *argv,       /* xCreate/xConnect argument array */
 245   sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
 246   char **pzErr                    /* OUT: sqlite3_malloc'd error message */
 247 ){
 248   return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
 249 }
 250
 251 /*
 252 ** Implementation of the xBestIndex method.
 253 **
 254 ** Only constraints of the form:
 255 **
 256 **     term <= ?
 257 **     term == ?
 258 **     term >= ?
 259 **
 260 ** are interpreted. Less-than and less-than-or-equal are treated
 261 ** identically, as are greater-than and greater-than-or-equal.
 262 */
 263 static int fts5VocabBestIndexMethod(
 264   sqlite3_vtab *pUnused,
 265   sqlite3_index_info *pInfo
 266 ){
 267   int i;
 268   int iTermEq = -1;
 269   int iTermGe = -1;
 270   int iTermLe = -1;
 271   int idxNum = 0;
 272   int nArg = 0;
 273
 274   UNUSED_PARAM(pUnused);
 275
 276   for(i=0; i<pInfo->nConstraint; i++){
 277     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
 278     if( p->usable==0 ) continue;
 279     if( p->iColumn==0 ){          /* term column */
 280       if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
 281       if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
 282       if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
 283       if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
 284       if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
 285     }
 286   }
 287
 288   if( iTermEq>=0 ){
 289     idxNum |= FTS5_VOCAB_TERM_EQ;
 290     pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
 291     pInfo->estimatedCost = 100;
 292   }else{
 293     pInfo->estimatedCost = 1000000;
 294     if( iTermGe>=0 ){
 295       idxNum |= FTS5_VOCAB_TERM_GE;
 296       pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
 297       pInfo->estimatedCost = pInfo->estimatedCost / 2;
 298     }
 299     if( iTermLe>=0 ){
 300       idxNum |= FTS5_VOCAB_TERM_LE;
 301       pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
 302       pInfo->estimatedCost = pInfo->estimatedCost / 2;
 303     }
 304   }
 305
 306   /* This virtual table always delivers results in ascending order of
 307   ** the "term" column (column 0). So if the user has requested this
 308   ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
 309   ** sqlite3_index_info.orderByConsumed flag to tell the core the results
 310   ** are already in sorted order.  */
 311   if( pInfo->nOrderBy==1
 312    && pInfo->aOrderBy[0].iColumn==0
 313    && pInfo->aOrderBy[0].desc==0
 314   ){
 315     pInfo->orderByConsumed = 1;
 316   }
 317
 318   pInfo->idxNum = idxNum;
 319   return SQLITE_OK;
 320 }
 321
 322 /*
 323 ** Implementation of xOpen method.
 324 */
 325 static int fts5VocabOpenMethod(
 326   sqlite3_vtab *pVTab,
 327   sqlite3_vtab_cursor **ppCsr
 328 ){
 329   Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
 330   Fts5Index *pIndex = 0;
 331   Fts5Config *pConfig = 0;
 332   Fts5VocabCursor *pCsr = 0;
 333   int rc = SQLITE_OK;
 334   sqlite3_stmt *pStmt = 0;
 335   char *zSql = 0;
 336
 337   zSql = sqlite3Fts5Mprintf(&rc,
 338       "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
 339       pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
 340   );
 341   if( zSql ){
 342     rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
 343   }
 344   sqlite3_free(zSql);
 345   assert( rc==SQLITE_OK || pStmt==0 );
 346   if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
 347
 348   if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
 349     i64 iId = sqlite3_column_int64(pStmt, 0);
 350     pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
 351   }
 352
 353   if( rc==SQLITE_OK && pIndex==0 ){
 354     rc = sqlite3_finalize(pStmt);
 355     pStmt = 0;
 356     if( rc==SQLITE_OK ){
 357       pVTab->zErrMsg = sqlite3_mprintf(
 358           "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
 359       );
 360       rc = SQLITE_ERROR;
 361     }
 362   }
 363
 364   if( rc==SQLITE_OK ){
 365     int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
 366     pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
 367   }
 368
 369   if( pCsr ){
 370     pCsr->pIndex = pIndex;
 371     pCsr->pStmt = pStmt;
 372     pCsr->pConfig = pConfig;
 373     pCsr->aCnt = (i64*)&pCsr[1];
 374     pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
 375   }else{
 376     sqlite3_finalize(pStmt);
 377   }
 378
 379   *ppCsr = (sqlite3_vtab_cursor*)pCsr;
 380   return rc;
 381 }
 382
 383 static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
 384   pCsr->rowid = 0;
 385   sqlite3Fts5IterClose(pCsr->pIter);
 386   pCsr->pIter = 0;
 387   sqlite3_free(pCsr->zLeTerm);
 388   pCsr->nLeTerm = -1;
 389   pCsr->zLeTerm = 0;
 390 }
 391
 392 /*
 393 ** Close the cursor.  For additional information see the documentation
 394 ** on the xClose method of the virtual table interface.
 395 */
 396 static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
 397   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 398   fts5VocabResetCursor(pCsr);
 399   sqlite3Fts5BufferFree(&pCsr->term);
 400   sqlite3_finalize(pCsr->pStmt);
 401   sqlite3_free(pCsr);
 402   return SQLITE_OK;
 403 }
 404
 405 static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
 406   int rc = SQLITE_OK;
 407
 408   if( sqlite3Fts5IterEof(pCsr->pIter) ){
 409     pCsr->bEof = 1;
 410   }else{
 411     const char *zTerm;
 412     int nTerm;
 413     zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
 414     if( pCsr->nLeTerm>=0 ){
 415       int nCmp = MIN(nTerm, pCsr->nLeTerm);
 416       int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
 417       if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
 418         pCsr->bEof = 1;
 419       }
 420     }
 421
 422     sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
 423   }
 424   return rc;
 425 }
 426
 427 static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
 428   int eDetail = pCsr->pConfig->eDetail;
 429   int rc = SQLITE_OK;
 430   Fts5IndexIter *pIter = pCsr->pIter;
 431   i64 *pp = &pCsr->iInstPos;
 432   int *po = &pCsr->iInstOff;
 433
 434   while( eDetail==FTS5_DETAIL_NONE
 435       || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp)
 436   ){
 437     pCsr->iInstPos = 0;
 438     pCsr->iInstOff = 0;
 439
 440     rc = sqlite3Fts5IterNextScan(pCsr->pIter);
 441     if( rc==SQLITE_OK ){
 442       rc = fts5VocabInstanceNewTerm(pCsr);
 443       if( eDetail==FTS5_DETAIL_NONE ) break;
 444     }
 445     if( rc ){
 446       pCsr->bEof = 1;
 447       break;
 448     }
 449   }
 450
 451   return rc;
 452 }
 453
 454 /*
 455 ** Advance the cursor to the next row in the table.
 456 */
 457 static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 458   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 459   Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
 460   int rc = SQLITE_OK;
 461   int nCol = pCsr->pConfig->nCol;
 462
 463   pCsr->rowid++;
 464
 465   if( pTab->eType==FTS5_VOCAB_INSTANCE ){
 466     return fts5VocabInstanceNext(pCsr);
 467   }
 468
 469   if( pTab->eType==FTS5_VOCAB_COL ){
 470     for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
 471       if( pCsr->aDoc[pCsr->iCol] ) break;
 472     }
 473   }
 474
 475   if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
 476     if( sqlite3Fts5IterEof(pCsr->pIter) ){
 477       pCsr->bEof = 1;
 478     }else{
 479       const char *zTerm;
 480       int nTerm;
 481
 482       zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
 483       if( pCsr->nLeTerm>=0 ){
 484         int nCmp = MIN(nTerm, pCsr->nLeTerm);
 485         int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
 486         if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
 487           pCsr->bEof = 1;
 488           return SQLITE_OK;
 489         }
 490       }
 491
 492       sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
 493       memset(pCsr->aCnt, 0, nCol * sizeof(i64));
 494       memset(pCsr->aDoc, 0, nCol * sizeof(i64));
 495       pCsr->iCol = 0;
 496
 497       assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
 498       while( rc==SQLITE_OK ){
 499         int eDetail = pCsr->pConfig->eDetail;
 500         const u8 *pPos; int nPos;   /* Position list */
 501         i64 iPos = 0;               /* 64-bit position read from poslist */
 502         int iOff = 0;               /* Current offset within position list */
 503
 504         pPos = pCsr->pIter->pData;
 505         nPos = pCsr->pIter->nData;
 506
 507         switch( pTab->eType ){
 508           case FTS5_VOCAB_ROW:
 509             if( eDetail==FTS5_DETAIL_FULL ){
 510               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
 511                 pCsr->aCnt[0]++;
 512               }
 513             }
 514             pCsr->aDoc[0]++;
 515             break;
 516
 517           case FTS5_VOCAB_COL:
 518             if( eDetail==FTS5_DETAIL_FULL ){
 519               int iCol = -1;
 520               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
 521                 int ii = FTS5_POS2COLUMN(iPos);
 522                 pCsr->aCnt[ii]++;
 523                 if( iCol!=ii ){
 524                   if( ii>=nCol ){
 525                     rc = FTS5_CORRUPT;
 526                     break;
 527                   }
 528                   pCsr->aDoc[ii]++;
 529                   iCol = ii;
 530                 }
 531               }
 532             }else if( eDetail==FTS5_DETAIL_COLUMNS ){
 533               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
 534                 assert_nc( iPos>=0 && iPos<nCol );
 535                 if( iPos>=nCol ){
 536                   rc = FTS5_CORRUPT;
 537                   break;
 538                 }
 539                 pCsr->aDoc[iPos]++;
 540               }
 541             }else{
 542               assert( eDetail==FTS5_DETAIL_NONE );
 543               pCsr->aDoc[0]++;
 544             }
 545             break;
 546
 547           default:
 548             assert( pTab->eType==FTS5_VOCAB_INSTANCE );
 549             break;
 550         }
 551
 552         if( rc==SQLITE_OK ){
 553           rc = sqlite3Fts5IterNextScan(pCsr->pIter);
 554         }
 555         if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;
 556
 557         if( rc==SQLITE_OK ){
 558           zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
 559           if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
 560             break;
 561           }
 562           if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
 563         }
 564       }
 565     }
 566   }
 567
 568   if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
 569     while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
 570     assert( pCsr->iCol<pCsr->pConfig->nCol );
 571   }
 572   return rc;
 573 }
 574
 575 /*
 576 ** This is the xFilter implementation for the virtual table.
 577 */
 578 static int fts5VocabFilterMethod(
 579   sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
 580   int idxNum,                     /* Strategy index */
 581   const char *zUnused,            /* Unused */
 582   int nUnused,                    /* Number of elements in apVal */
 583   sqlite3_value **apVal           /* Arguments for the indexing scheme */
 584 ){
 585   Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
 586   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 587   int eType = pTab->eType;
 588   int rc = SQLITE_OK;
 589
 590   int iVal = 0;
 591   int f = FTS5INDEX_QUERY_SCAN;
 592   const char *zTerm = 0;
 593   int nTerm = 0;
 594
 595   sqlite3_value *pEq = 0;
 596   sqlite3_value *pGe = 0;
 597   sqlite3_value *pLe = 0;
 598
 599   UNUSED_PARAM2(zUnused, nUnused);
 600
 601   fts5VocabResetCursor(pCsr);
 602   if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
 603   if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
 604   if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
 605
 606   if( pEq ){
 607     zTerm = (const char *)sqlite3_value_text(pEq);
 608     nTerm = sqlite3_value_bytes(pEq);
 609     f = 0;
 610   }else{
 611     if( pGe ){
 612       zTerm = (const char *)sqlite3_value_text(pGe);
 613       nTerm = sqlite3_value_bytes(pGe);
 614     }
 615     if( pLe ){
 616       const char *zCopy = (const char *)sqlite3_value_text(pLe);
 617       pCsr->nLeTerm = sqlite3_value_bytes(pLe);
 618       pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
 619       if( pCsr->zLeTerm==0 ){
 620         rc = SQLITE_NOMEM;
 621       }else{
 622         memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
 623       }
 624     }
 625   }
 626
 627   if( rc==SQLITE_OK ){
 628     rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
 629   }
 630   if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
 631     rc = fts5VocabInstanceNewTerm(pCsr);
 632   }
 633   if( rc==SQLITE_OK
 634    && !pCsr->bEof
 635    && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE)
 636   ){
 637     rc = fts5VocabNextMethod(pCursor);
 638   }
 639
 640   return rc;
 641 }
 642
 643 /*
 644 ** This is the xEof method of the virtual table. SQLite calls this
 645 ** routine to find out if it has reached the end of a result set.
 646 */
 647 static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
 648   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 649   return pCsr->bEof;
 650 }
 651
 652 static int fts5VocabColumnMethod(
 653   sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
 654   sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
 655   int iCol                        /* Index of column to read value from */
 656 ){
 657   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 658   int eDetail = pCsr->pConfig->eDetail;
 659   int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
 660   i64 iVal = 0;
 661
 662   if( iCol==0 ){
 663     sqlite3_result_text(
 664         pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
 665     );
 666   }else if( eType==FTS5_VOCAB_COL ){
 667     assert( iCol==1 || iCol==2 || iCol==3 );
 668     if( iCol==1 ){
 669       if( eDetail!=FTS5_DETAIL_NONE ){
 670         const char *z = pCsr->pConfig->azCol[pCsr->iCol];
 671         sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
 672       }
 673     }else if( iCol==2 ){
 674       iVal = pCsr->aDoc[pCsr->iCol];
 675     }else{
 676       iVal = pCsr->aCnt[pCsr->iCol];
 677     }
 678   }else if( eType==FTS5_VOCAB_ROW ){
 679     assert( iCol==1 || iCol==2 );
 680     if( iCol==1 ){
 681       iVal = pCsr->aDoc[0];
 682     }else{
 683       iVal = pCsr->aCnt[0];
 684     }
 685   }else{
 686     int eDetail = pCsr->pConfig->eDetail;
 687     assert( eType==FTS5_VOCAB_INSTANCE );
 688     switch( iCol ){
 689       case 1:
 690         sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
 691         break;
 692       case 2: {
 693         int ii = -1;
 694         if( eDetail==FTS5_DETAIL_FULL ){
 695           ii = FTS5_POS2COLUMN(pCsr->iInstPos);
 696         }else if( eDetail==FTS5_DETAIL_COLUMNS ){
 697           ii = pCsr->iInstPos;
 698         }
 699         if( ii>=0 && ii<pCsr->pConfig->nCol ){
 700           const char *z = pCsr->pConfig->azCol[ii];
 701           sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
 702         }
 703         break;
 704       }
 705       default: {
 706         assert( iCol==3 );
 707         if( eDetail==FTS5_DETAIL_FULL ){
 708           int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
 709           sqlite3_result_int(pCtx, ii);
 710         }
 711         break;
 712       }
 713     }
 714   }
 715
 716   if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
 717   return SQLITE_OK;
 718 }
 719
 720 /*
 721 ** This is the xRowid method. The SQLite core calls this routine to
 722 ** retrieve the rowid for the current row of the result set. The
 723 ** rowid should be written to *pRowid.
 724 */
 725 static int fts5VocabRowidMethod(
 726   sqlite3_vtab_cursor *pCursor,
 727   sqlite_int64 *pRowid
 728 ){
 729   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
 730   *pRowid = pCsr->rowid;
 731   return SQLITE_OK;
 732 }
 733
 734 int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
 735   static const sqlite3_module fts5Vocab = {
 736     /* iVersion      */ 2,
 737     /* xCreate       */ fts5VocabCreateMethod,
 738     /* xConnect      */ fts5VocabConnectMethod,
 739     /* xBestIndex    */ fts5VocabBestIndexMethod,
 740     /* xDisconnect   */ fts5VocabDisconnectMethod,
 741     /* xDestroy      */ fts5VocabDestroyMethod,
 742     /* xOpen         */ fts5VocabOpenMethod,
 743     /* xClose        */ fts5VocabCloseMethod,
 744     /* xFilter       */ fts5VocabFilterMethod,
 745     /* xNext         */ fts5VocabNextMethod,
 746     /* xEof          */ fts5VocabEofMethod,
 747     /* xColumn       */ fts5VocabColumnMethod,
 748     /* xRowid        */ fts5VocabRowidMethod,
 749     /* xUpdate       */ 0,
 750     /* xBegin        */ 0,
 751     /* xSync         */ 0,
 752     /* xCommit       */ 0,
 753     /* xRollback     */ 0,
 754     /* xFindFunction */ 0,
 755     /* xRename       */ 0,
 756     /* xSavepoint    */ 0,
 757     /* xRelease      */ 0,
 758     /* xRollbackTo   */ 0,
 759   };
 760   void *p = (void*)pGlobal;
 761
 762   return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
 763 }
 764
 765