ext/fts3/fts3Int.h

   1 /*
   2 ** 2009 Nov 12
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTSINT_H
  15 #define _FTSINT_H
  16
  17 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
  18 # define NDEBUG 1
  19 #endif
  20
  21 /* FTS3/FTS4 require virtual tables */
  22 #ifdef SQLITE_OMIT_VIRTUALTABLE
  23 # undef SQLITE_ENABLE_FTS3
  24 # undef SQLITE_ENABLE_FTS4
  25 #endif
  26
  27 /*
  28 ** FTS4 is really an extension for FTS3.  It is enabled using the
  29 ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
  30 ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
  31 */
  32 #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
  33 # define SQLITE_ENABLE_FTS3
  34 #endif
  35
  36 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
  37
  38 /* If not building as part of the core, include sqlite3ext.h. */
  39 #ifndef SQLITE_CORE
  40 # include "sqlite3ext.h"
  41 SQLITE_EXTENSION_INIT3
  42 #endif
  43
  44 #include "sqlite3.h"
  45 #include "fts3_tokenizer.h"
  46 #include "fts3_hash.h"
  47
  48 /*
  49 ** This constant determines the maximum depth of an FTS expression tree
  50 ** that the library will create and use. FTS uses recursion to perform
  51 ** various operations on the query tree, so the disadvantage of a large
  52 ** limit is that it may allow very large queries to use large amounts
  53 ** of stack space (perhaps causing a stack overflow).
  54 */
  55 #ifndef SQLITE_FTS3_MAX_EXPR_DEPTH
  56 # define SQLITE_FTS3_MAX_EXPR_DEPTH 12
  57 #endif
  58
  59
  60 /*
  61 ** This constant controls how often segments are merged. Once there are
  62 ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
  63 ** segment of level N+1.
  64 */
  65 #define FTS3_MERGE_COUNT 16
  66
  67 /*
  68 ** This is the maximum amount of data (in bytes) to store in the
  69 ** Fts3Table.pendingTerms hash table. Normally, the hash table is
  70 ** populated as documents are inserted/updated/deleted in a transaction
  71 ** and used to create a new segment when the transaction is committed.
  72 ** However if this limit is reached midway through a transaction, a new
  73 ** segment is created and the hash table cleared immediately.
  74 */
  75 #define FTS3_MAX_PENDING_DATA (1*1024*1024)
  76
  77 /*
  78 ** Macro to return the number of elements in an array. SQLite has a
  79 ** similar macro called ArraySize(). Use a different name to avoid
  80 ** a collision when building an amalgamation with built-in FTS3.
  81 */
  82 #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
  83
  84
  85 #ifndef MIN
  86 # define MIN(x,y) ((x)<(y)?(x):(y))
  87 #endif
  88 #ifndef MAX
  89 # define MAX(x,y) ((x)>(y)?(x):(y))
  90 #endif
  91
  92 /*
  93 ** Maximum length of a varint encoded integer. The varint format is different
  94 ** from that used by SQLite, so the maximum length is 10, not 9.
  95 */
  96 #define FTS3_VARINT_MAX 10
  97
  98 /*
  99 ** FTS4 virtual tables may maintain multiple indexes - one index of all terms
 100 ** in the document set and zero or more prefix indexes. All indexes are stored
 101 ** as one or more b+-trees in the %_segments and %_segdir tables.
 102 **
 103 ** It is possible to determine which index a b+-tree belongs to based on the
 104 ** value stored in the "%_segdir.level" column. Given this value L, the index
 105 ** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
 106 ** level values between 0 and 1023 (inclusive) belong to index 0, all levels
 107 ** between 1024 and 2047 to index 1, and so on.
 108 **
 109 ** It is considered impossible for an index to use more than 1024 levels. In
 110 ** theory though this may happen, but only after at least
 111 ** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
 112 */
 113 #define FTS3_SEGDIR_MAXLEVEL      1024
 114 #define FTS3_SEGDIR_MAXLEVEL_STR "1024"
 115
 116 /*
 117 ** The testcase() macro is only used by the amalgamation.  If undefined,
 118 ** make it a no-op.
 119 */
 120 #ifndef testcase
 121 # define testcase(X)
 122 #endif
 123
 124 /*
 125 ** Terminator values for position-lists and column-lists.
 126 */
 127 #define POS_COLUMN  (1)     /* Column-list terminator */
 128 #define POS_END     (0)     /* Position-list terminator */
 129
 130 /*
 131 ** This section provides definitions to allow the
 132 ** FTS3 extension to be compiled outside of the
 133 ** amalgamation.
 134 */
 135 #ifndef SQLITE_AMALGAMATION
 136 /*
 137 ** Macros indicating that conditional expressions are always true or
 138 ** false.
 139 */
 140 #ifdef SQLITE_COVERAGE_TEST
 141 # define ALWAYS(x) (1)
 142 # define NEVER(X)  (0)
 143 #elif defined(SQLITE_DEBUG)
 144 # define ALWAYS(x) sqlite3Fts3Always((x)!=0)
 145 # define NEVER(x) sqlite3Fts3Never((x)!=0)
 146 int sqlite3Fts3Always(int b);
 147 int sqlite3Fts3Never(int b);
 148 #else
 149 # define ALWAYS(x) (x)
 150 # define NEVER(x)  (x)
 151 #endif
 152
 153 /*
 154 ** Internal types used by SQLite.
 155 */
 156 typedef unsigned char u8;         /* 1-byte (or larger) unsigned integer */
 157 typedef short int i16;            /* 2-byte (or larger) signed integer */
 158 typedef unsigned int u32;         /* 4-byte unsigned integer */
 159 typedef sqlite3_uint64 u64;       /* 8-byte unsigned integer */
 160 typedef sqlite3_int64 i64;        /* 8-byte signed integer */
 161
 162 /*
 163 ** Macro used to suppress compiler warnings for unused parameters.
 164 */
 165 #define UNUSED_PARAMETER(x) (void)(x)
 166
 167 /*
 168 ** Activate assert() only if SQLITE_TEST is enabled.
 169 */
 170 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
 171 # define NDEBUG 1
 172 #endif
 173
 174 /*
 175 ** The TESTONLY macro is used to enclose variable declarations or
 176 ** other bits of code that are needed to support the arguments
 177 ** within testcase() and assert() macros.
 178 */
 179 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
 180 # define TESTONLY(X)  X
 181 #else
 182 # define TESTONLY(X)
 183 #endif
 184
 185 #endif /* SQLITE_AMALGAMATION */
 186
 187 #ifdef SQLITE_DEBUG
 188 int sqlite3Fts3Corrupt(void);
 189 # define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt()
 190 #else
 191 # define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB
 192 #endif
 193
 194 typedef struct Fts3Table Fts3Table;
 195 typedef struct Fts3Cursor Fts3Cursor;
 196 typedef struct Fts3Expr Fts3Expr;
 197 typedef struct Fts3Phrase Fts3Phrase;
 198 typedef struct Fts3PhraseToken Fts3PhraseToken;
 199
 200 typedef struct Fts3Doclist Fts3Doclist;
 201 typedef struct Fts3SegFilter Fts3SegFilter;
 202 typedef struct Fts3DeferredToken Fts3DeferredToken;
 203 typedef struct Fts3SegReader Fts3SegReader;
 204 typedef struct Fts3MultiSegReader Fts3MultiSegReader;
 205
 206 typedef struct MatchinfoBuffer MatchinfoBuffer;
 207
 208 /*
 209 ** A connection to a fulltext index is an instance of the following
 210 ** structure. The xCreate and xConnect methods create an instance
 211 ** of this structure and xDestroy and xDisconnect free that instance.
 212 ** All other methods receive a pointer to the structure as one of their
 213 ** arguments.
 214 */
 215 struct Fts3Table {
 216   sqlite3_vtab base;              /* Base class used by SQLite core */
 217   sqlite3 *db;                    /* The database connection */
 218   const char *zDb;                /* logical database name */
 219   const char *zName;              /* virtual table name */
 220   int nColumn;                    /* number of named columns in virtual table */
 221   char **azColumn;                /* column names.  malloced */
 222   u8 *abNotindexed;               /* True for 'notindexed' columns */
 223   sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
 224   char *zContentTbl;              /* content=xxx option, or NULL */
 225   char *zLanguageid;              /* languageid=xxx option, or NULL */
 226   int nAutoincrmerge;             /* Value configured by 'automerge' */
 227   u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
 228
 229   /* Precompiled statements used by the implementation. Each of these
 230   ** statements is run and reset within a single virtual table API call.
 231   */
 232   sqlite3_stmt *aStmt[40];
 233   sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */
 234
 235   char *zReadExprlist;
 236   char *zWriteExprlist;
 237
 238   int nNodeSize;                  /* Soft limit for node size */
 239   u8 bFts4;                       /* True for FTS4, false for FTS3 */
 240   u8 bHasStat;                    /* True if %_stat table exists (2==unknown) */
 241   u8 bHasDocsize;                 /* True if %_docsize table exists */
 242   u8 bDescIdx;                    /* True if doclists are in reverse order */
 243   u8 bIgnoreSavepoint;            /* True to ignore xSavepoint invocations */
 244   int nPgsz;                      /* Page size for host database */
 245   char *zSegmentsTbl;             /* Name of %_segments table */
 246   sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
 247
 248   /*
 249   ** The following array of hash tables is used to buffer pending index
 250   ** updates during transactions. All pending updates buffered at any one
 251   ** time must share a common language-id (see the FTS4 langid= feature).
 252   ** The current language id is stored in variable iPrevLangid.
 253   **
 254   ** A single FTS4 table may have multiple full-text indexes. For each index
 255   ** there is an entry in the aIndex[] array. Index 0 is an index of all the
 256   ** terms that appear in the document set. Each subsequent index in aIndex[]
 257   ** is an index of prefixes of a specific length.
 258   **
 259   ** Variable nPendingData contains an estimate the memory consumed by the
 260   ** pending data structures, including hash table overhead, but not including
 261   ** malloc overhead.  When nPendingData exceeds nMaxPendingData, all hash
 262   ** tables are flushed to disk. Variable iPrevDocid is the docid of the most
 263   ** recently inserted record.
 264   */
 265   int nIndex;                     /* Size of aIndex[] */
 266   struct Fts3Index {
 267     int nPrefix;                  /* Prefix length (0 for main terms index) */
 268     Fts3Hash hPending;            /* Pending terms table for this index */
 269   } *aIndex;
 270   int nMaxPendingData;            /* Max pending data before flush to disk */
 271   int nPendingData;               /* Current bytes of pending data */
 272   sqlite_int64 iPrevDocid;        /* Docid of most recently inserted document */
 273   int iPrevLangid;                /* Langid of recently inserted document */
 274   int bPrevDelete;                /* True if last operation was a delete */
 275
 276 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
 277   /* State variables used for validating that the transaction control
 278   ** methods of the virtual table are called at appropriate times.  These
 279   ** values do not contribute to FTS functionality; they are used for
 280   ** verifying the operation of the SQLite core.
 281   */
 282   int inTransaction;     /* True after xBegin but before xCommit/xRollback */
 283   int mxSavepoint;       /* Largest valid xSavepoint integer */
 284 #endif
 285
 286 #ifdef SQLITE_TEST
 287   /* True to disable the incremental doclist optimization. This is controled
 288   ** by special insert command 'test-no-incr-doclist'.  */
 289   int bNoIncrDoclist;
 290 #endif
 291 };
 292
 293 /*
 294 ** When the core wants to read from the virtual table, it creates a
 295 ** virtual table cursor (an instance of the following structure) using
 296 ** the xOpen method. Cursors are destroyed using the xClose method.
 297 */
 298 struct Fts3Cursor {
 299   sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
 300   i16 eSearch;                    /* Search strategy (see below) */
 301   u8 isEof;                       /* True if at End Of Results */
 302   u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
 303   u8 bSeekStmt;                   /* True if pStmt is a seek */
 304   sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
 305   Fts3Expr *pExpr;                /* Parsed MATCH query string */
 306   int iLangid;                    /* Language being queried for */
 307   int nPhrase;                    /* Number of matchable phrases in query */
 308   Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
 309   sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
 310   char *pNextId;                  /* Pointer into the body of aDoclist */
 311   char *aDoclist;                 /* List of docids for full-text queries */
 312   int nDoclist;                   /* Size of buffer at aDoclist */
 313   u8 bDesc;                       /* True to sort in descending order */
 314   int eEvalmode;                  /* An FTS3_EVAL_XX constant */
 315   int nRowAvg;                    /* Average size of database rows, in pages */
 316   sqlite3_int64 nDoc;             /* Documents in table */
 317   i64 iMinDocid;                  /* Minimum docid to return */
 318   i64 iMaxDocid;                  /* Maximum docid to return */
 319   int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
 320   MatchinfoBuffer *pMIBuffer;     /* Buffer for matchinfo data */
 321 };
 322
 323 #define FTS3_EVAL_FILTER    0
 324 #define FTS3_EVAL_NEXT      1
 325 #define FTS3_EVAL_MATCHINFO 2
 326
 327 /*
 328 ** The Fts3Cursor.eSearch member is always set to one of the following.
 329 ** Actualy, Fts3Cursor.eSearch can be greater than or equal to
 330 ** FTS3_FULLTEXT_SEARCH.  If so, then Fts3Cursor.eSearch - 2 is the index
 331 ** of the column to be searched.  For example, in
 332 **
 333 **     CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
 334 **     SELECT docid FROM ex1 WHERE b MATCH 'one two three';
 335 **
 336 ** Because the LHS of the MATCH operator is 2nd column "b",
 337 ** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1.  (+0 for a,
 338 ** +1 for b, +2 for c, +3 for d.)  If the LHS of MATCH were "ex1"
 339 ** indicating that all columns should be searched,
 340 ** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
 341 */
 342 #define FTS3_FULLSCAN_SEARCH 0    /* Linear scan of %_content table */
 343 #define FTS3_DOCID_SEARCH    1    /* Lookup by rowid on %_content table */
 344 #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
 345
 346 /*
 347 ** The lower 16-bits of the sqlite3_index_info.idxNum value set by
 348 ** the xBestIndex() method contains the Fts3Cursor.eSearch value described
 349 ** above. The upper 16-bits contain a combination of the following
 350 ** bits, used to describe extra constraints on full-text searches.
 351 */
 352 #define FTS3_HAVE_LANGID    0x00010000      /* languageid=? */
 353 #define FTS3_HAVE_DOCID_GE  0x00020000      /* docid>=? */
 354 #define FTS3_HAVE_DOCID_LE  0x00040000      /* docid<=? */
 355
 356 struct Fts3Doclist {
 357   char *aAll;                    /* Array containing doclist (or NULL) */
 358   int nAll;                      /* Size of a[] in bytes */
 359   char *pNextDocid;              /* Pointer to next docid */
 360
 361   sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
 362   int bFreeList;                 /* True if pList should be sqlite3_free()d */
 363   char *pList;                   /* Pointer to position list following iDocid */
 364   int nList;                     /* Length of position list */
 365 };
 366
 367 /*
 368 ** A "phrase" is a sequence of one or more tokens that must match in
 369 ** sequence.  A single token is the base case and the most common case.
 370 ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
 371 ** nToken will be the number of tokens in the string.
 372 */
 373 struct Fts3PhraseToken {
 374   char *z;                        /* Text of the token */
 375   int n;                          /* Number of bytes in buffer z */
 376   int isPrefix;                   /* True if token ends with a "*" character */
 377   int bFirst;                     /* True if token must appear at position 0 */
 378
 379   /* Variables above this point are populated when the expression is
 380   ** parsed (by code in fts3_expr.c). Below this point the variables are
 381   ** used when evaluating the expression. */
 382   Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
 383   Fts3MultiSegReader *pSegcsr;    /* Segment-reader for this token */
 384 };
 385
 386 struct Fts3Phrase {
 387   /* Cache of doclist for this phrase. */
 388   Fts3Doclist doclist;
 389   int bIncr;                 /* True if doclist is loaded incrementally */
 390   int iDoclistToken;
 391
 392   /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an
 393   ** OR condition.  */
 394   char *pOrPoslist;
 395   i64 iOrDocid;
 396
 397   /* Variables below this point are populated by fts3_expr.c when parsing
 398   ** a MATCH expression. Everything above is part of the evaluation phase.
 399   */
 400   int nToken;                /* Number of tokens in the phrase */
 401   int iColumn;               /* Index of column this phrase must match */
 402   Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
 403 };
 404
 405 /*
 406 ** A tree of these objects forms the RHS of a MATCH operator.
 407 **
 408 ** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
 409 ** points to a malloced buffer, size nDoclist bytes, containing the results
 410 ** of this phrase query in FTS3 doclist format. As usual, the initial
 411 ** "Length" field found in doclists stored on disk is omitted from this
 412 ** buffer.
 413 **
 414 ** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
 415 ** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
 416 ** where nCol is the number of columns in the queried FTS table. The array
 417 ** is populated as follows:
 418 **
 419 **   aMI[iCol*3 + 0] = Undefined
 420 **   aMI[iCol*3 + 1] = Number of occurrences
 421 **   aMI[iCol*3 + 2] = Number of rows containing at least one instance
 422 **
 423 ** The aMI array is allocated using sqlite3_malloc(). It should be freed
 424 ** when the expression node is.
 425 */
 426 struct Fts3Expr {
 427   int eType;                 /* One of the FTSQUERY_XXX values defined below */
 428   int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
 429   Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
 430   Fts3Expr *pLeft;           /* Left operand */
 431   Fts3Expr *pRight;          /* Right operand */
 432   Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
 433
 434   /* The following are used by the fts3_eval.c module. */
 435   sqlite3_int64 iDocid;      /* Current docid */
 436   u8 bEof;                   /* True this expression is at EOF already */
 437   u8 bStart;                 /* True if iDocid is valid */
 438   u8 bDeferred;              /* True if this expression is entirely deferred */
 439
 440   /* The following are used by the fts3_snippet.c module. */
 441   int iPhrase;               /* Index of this phrase in matchinfo() results */
 442   u32 *aMI;                  /* See above */
 443 };
 444
 445 /*
 446 ** Candidate values for Fts3Query.eType. Note that the order of the first
 447 ** four values is in order of precedence when parsing expressions. For
 448 ** example, the following:
 449 **
 450 **   "a OR b AND c NOT d NEAR e"
 451 **
 452 ** is equivalent to:
 453 **
 454 **   "a OR (b AND (c NOT (d NEAR e)))"
 455 */
 456 #define FTSQUERY_NEAR   1
 457 #define FTSQUERY_NOT    2
 458 #define FTSQUERY_AND    3
 459 #define FTSQUERY_OR     4
 460 #define FTSQUERY_PHRASE 5
 461
 462
 463 /* fts3_write.c */
 464 int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
 465 int sqlite3Fts3PendingTermsFlush(Fts3Table *);
 466 void sqlite3Fts3PendingTermsClear(Fts3Table *);
 467 int sqlite3Fts3Optimize(Fts3Table *);
 468 int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
 469   sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
 470 int sqlite3Fts3SegReaderPending(
 471   Fts3Table*,int,const char*,int,int,Fts3SegReader**);
 472 void sqlite3Fts3SegReaderFree(Fts3SegReader *);
 473 int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **);
 474 int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
 475
 476 int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
 477 int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
 478
 479 #ifndef SQLITE_DISABLE_FTS4_DEFERRED
 480 void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
 481 int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
 482 int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
 483 void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
 484 int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
 485 #else
 486 # define sqlite3Fts3FreeDeferredTokens(x)
 487 # define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK
 488 # define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK
 489 # define sqlite3Fts3FreeDeferredDoclists(x)
 490 # define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK
 491 #endif
 492
 493 void sqlite3Fts3SegmentsClose(Fts3Table *);
 494 int sqlite3Fts3MaxLevel(Fts3Table *, int *);
 495
 496 /* Special values interpreted by sqlite3SegReaderCursor() */
 497 #define FTS3_SEGCURSOR_PENDING        -1
 498 #define FTS3_SEGCURSOR_ALL            -2
 499
 500 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
 501 int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
 502 void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
 503
 504 int sqlite3Fts3SegReaderCursor(Fts3Table *,
 505     int, int, int, const char *, int, int, int, Fts3MultiSegReader *);
 506
 507 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
 508 #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
 509 #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
 510 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
 511 #define FTS3_SEGMENT_PREFIX        0x00000008
 512 #define FTS3_SEGMENT_SCAN          0x00000010
 513 #define FTS3_SEGMENT_FIRST         0x00000020
 514
 515 /* Type passed as 4th argument to SegmentReaderIterate() */
 516 struct Fts3SegFilter {
 517   const char *zTerm;
 518   int nTerm;
 519   int iCol;
 520   int flags;
 521 };
 522
 523 struct Fts3MultiSegReader {
 524   /* Used internally by sqlite3Fts3SegReaderXXX() calls */
 525   Fts3SegReader **apSegment;      /* Array of Fts3SegReader objects */
 526   int nSegment;                   /* Size of apSegment array */
 527   int nAdvance;                   /* How many seg-readers to advance */
 528   Fts3SegFilter *pFilter;         /* Pointer to filter object */
 529   char *aBuffer;                  /* Buffer to merge doclists in */
 530   int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
 531
 532   int iColFilter;                 /* If >=0, filter for this column */
 533   int bRestart;
 534
 535   /* Used by fts3.c only. */
 536   int nCost;                      /* Cost of running iterator */
 537   int bLookup;                    /* True if a lookup of a single entry. */
 538
 539   /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
 540   char *zTerm;                    /* Pointer to term buffer */
 541   int nTerm;                      /* Size of zTerm in bytes */
 542   char *aDoclist;                 /* Pointer to doclist buffer */
 543   int nDoclist;                   /* Size of aDoclist[] in bytes */
 544 };
 545
 546 int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
 547
 548 #define fts3GetVarint32(p, piVal) (                                           \
 549   (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \
 550 )
 551
 552 /* fts3.c */
 553 void sqlite3Fts3ErrMsg(char**,const char*,...);
 554 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 555 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
 556 int sqlite3Fts3GetVarint32(const char *, int *);
 557 int sqlite3Fts3VarintLen(sqlite3_uint64);
 558 void sqlite3Fts3Dequote(char *);
 559 void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
 560 int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
 561 int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
 562 void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
 563 int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
 564
 565 /* fts3_tokenizer.c */
 566 const char *sqlite3Fts3NextToken(const char *, int *);
 567 int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
 568 int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
 569     sqlite3_tokenizer **, char **
 570 );
 571 int sqlite3Fts3IsIdChar(char);
 572
 573 /* fts3_snippet.c */
 574 void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
 575 void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
 576   const char *, const char *, int, int
 577 );
 578 void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
 579 void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
 580
 581 /* fts3_expr.c */
 582 int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 583   char **, int, int, int, const char *, int, Fts3Expr **, char **
 584 );
 585 void sqlite3Fts3ExprFree(Fts3Expr *);
 586 #ifdef SQLITE_TEST
 587 int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
 588 int sqlite3Fts3InitTerm(sqlite3 *db);
 589 #endif
 590
 591 int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
 592   sqlite3_tokenizer_cursor **
 593 );
 594
 595 /* fts3_aux.c */
 596 int sqlite3Fts3InitAux(sqlite3 *db);
 597
 598 void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
 599
 600 int sqlite3Fts3MsrIncrStart(
 601     Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
 602 int sqlite3Fts3MsrIncrNext(
 603     Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
 604 int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
 605 int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
 606 int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
 607
 608 /* fts3_tokenize_vtab.c */
 609 int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
 610
 611 /* fts3_unicode2.c (functions generated by parsing unicode text files) */
 612 #ifndef SQLITE_DISABLE_FTS3_UNICODE
 613 int sqlite3FtsUnicodeFold(int, int);
 614 int sqlite3FtsUnicodeIsalnum(int);
 615 int sqlite3FtsUnicodeIsdiacritic(int);
 616 #endif
 617
 618 #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
 619 #endif /* _FTSINT_H */