ext/fts5/fts5Int.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTS5INT_H
  15 #define _FTS5INT_H
  16
  17 #include "fts5.h"
  18 #include "sqlite3ext.h"
  19 SQLITE_EXTENSION_INIT1
  20
  21 #include <string.h>
  22 #include <assert.h>
  23
  24 #ifndef SQLITE_AMALGAMATION
  25
  26 typedef unsigned char  u8;
  27 typedef unsigned int   u32;
  28 typedef unsigned short u16;
  29 typedef short i16;
  30 typedef sqlite3_int64 i64;
  31 typedef sqlite3_uint64 u64;
  32
  33 #ifndef ArraySize
  34 # define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
  35 #endif
  36
  37 #define testcase(x)
  38
  39 #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
  40 # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
  41 #endif
  42 #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
  43 # define ALWAYS(X)      (1)
  44 # define NEVER(X)       (0)
  45 #elif !defined(NDEBUG)
  46 # define ALWAYS(X)      ((X)?1:(assert(0),0))
  47 # define NEVER(X)       ((X)?(assert(0),1):0)
  48 #else
  49 # define ALWAYS(X)      (X)
  50 # define NEVER(X)       (X)
  51 #endif
  52
  53 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
  54 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
  55
  56 /*
  57 ** Constants for the largest and smallest possible 64-bit signed integers.
  58 */
  59 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
  60 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
  61
  62 #endif
  63
  64 /* Truncate very long tokens to this many bytes. Hard limit is
  65 ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
  66 ** field that occurs at the start of each leaf page (see fts5_index.c). */
  67 #define FTS5_MAX_TOKEN_SIZE 32768
  68
  69 /*
  70 ** Maximum number of prefix indexes on single FTS5 table. This must be
  71 ** less than 32. If it is set to anything large than that, an #error
  72 ** directive in fts5_index.c will cause the build to fail.
  73 */
  74 #define FTS5_MAX_PREFIX_INDEXES 31
  75
  76 /*
  77 ** Maximum segments permitted in a single index
  78 */
  79 #define FTS5_MAX_SEGMENT 2000
  80
  81 #define FTS5_DEFAULT_NEARDIST 10
  82 #define FTS5_DEFAULT_RANK     "bm25"
  83
  84 /* Name of rank and rowid columns */
  85 #define FTS5_RANK_NAME "rank"
  86 #define FTS5_ROWID_NAME "rowid"
  87
  88 #ifdef SQLITE_DEBUG
  89 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
  90 int sqlite3Fts5Corrupt(void);
  91 #else
  92 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
  93 #endif
  94
  95 /*
  96 ** The assert_nc() macro is similar to the assert() macro, except that it
  97 ** is used for assert() conditions that are true only if it can be
  98 ** guranteed that the database is not corrupt.
  99 */
 100 #ifdef SQLITE_DEBUG
 101 extern int sqlite3_fts5_may_be_corrupt;
 102 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
 103 #else
 104 # define assert_nc(x) assert(x)
 105 #endif
 106
 107 /*
 108 ** A version of memcmp() that does not cause asan errors if one of the pointer
 109 ** parameters is NULL and the number of bytes to compare is zero.
 110 */
 111 #define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
 112
 113 /* Mark a function parameter as unused, to suppress nuisance compiler
 114 ** warnings. */
 115 #ifndef UNUSED_PARAM
 116 # define UNUSED_PARAM(X)  (void)(X)
 117 #endif
 118
 119 #ifndef UNUSED_PARAM2
 120 # define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
 121 #endif
 122
 123 typedef struct Fts5Global Fts5Global;
 124 typedef struct Fts5Colset Fts5Colset;
 125
 126 /* If a NEAR() clump or phrase may only match a specific set of columns,
 127 ** then an object of the following type is used to record the set of columns.
 128 ** Each entry in the aiCol[] array is a column that may be matched.
 129 **
 130 ** This object is used by fts5_expr.c and fts5_index.c.
 131 */
 132 struct Fts5Colset {
 133   int nCol;
 134   int aiCol[1];
 135 };
 136
 137
 138
 139 /**************************************************************************
 140 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 141 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 142 */
 143
 144 typedef struct Fts5Config Fts5Config;
 145
 146 /*
 147 ** An instance of the following structure encodes all information that can
 148 ** be gleaned from the CREATE VIRTUAL TABLE statement.
 149 **
 150 ** And all information loaded from the %_config table.
 151 **
 152 ** nAutomerge:
 153 **   The minimum number of segments that an auto-merge operation should
 154 **   attempt to merge together. A value of 1 sets the object to use the
 155 **   compile time default. Zero disables auto-merge altogether.
 156 **
 157 ** bContentlessDelete:
 158 **   True if the contentless_delete option was present in the CREATE
 159 **   VIRTUAL TABLE statement.
 160 **
 161 ** zContent:
 162 **
 163 ** zContentRowid:
 164 **   The value of the content_rowid= option, if one was specified. Or
 165 **   the string "rowid" otherwise. This text is not quoted - if it is
 166 **   used as part of an SQL statement it needs to be quoted appropriately.
 167 **
 168 ** zContentExprlist:
 169 **
 170 ** pzErrmsg:
 171 **   This exists in order to allow the fts5_index.c module to return a
 172 **   decent error message if it encounters a file-format version it does
 173 **   not understand.
 174 **
 175 ** bColumnsize:
 176 **   True if the %_docsize table is created.
 177 **
 178 ** bPrefixIndex:
 179 **   This is only used for debugging. If set to false, any prefix indexes
 180 **   are ignored. This value is configured using:
 181 **
 182 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 183 **
 184 */
 185 struct Fts5Config {
 186   sqlite3 *db;                    /* Database handle */
 187   char *zDb;                      /* Database holding FTS index (e.g. "main") */
 188   char *zName;                    /* Name of FTS index */
 189   int nCol;                       /* Number of columns */
 190   char **azCol;                   /* Column names */
 191   u8 *abUnindexed;                /* True for unindexed columns */
 192   int nPrefix;                    /* Number of prefix indexes */
 193   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
 194   int eContent;                   /* An FTS5_CONTENT value */
 195   int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
 196   char *zContent;                 /* content table */
 197   char *zContentRowid;            /* "content_rowid=" option value */
 198   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
 199   int bTokendata;                 /* "tokendata=" option value (dflt==0) */
 200   int eDetail;                    /* FTS5_DETAIL_XXX value */
 201   char *zContentExprlist;
 202   Fts5Tokenizer *pTok;
 203   fts5_tokenizer *pTokApi;
 204   int bLock;                      /* True when table is preparing statement */
 205   int ePattern;                   /* FTS_PATTERN_XXX constant */
 206
 207   /* Values loaded from the %_config table */
 208   int iVersion;                   /* fts5 file format 'version' */
 209   int iCookie;                    /* Incremented when %_config is modified */
 210   int pgsz;                       /* Approximate page size used in %_data */
 211   int nAutomerge;                 /* 'automerge' setting */
 212   int nCrisisMerge;               /* Maximum allowed segments per level */
 213   int nUsermerge;                 /* 'usermerge' setting */
 214   int nHashSize;                  /* Bytes of memory for in-memory hash */
 215   char *zRank;                    /* Name of rank function */
 216   char *zRankArgs;                /* Arguments to rank function */
 217   int bSecureDelete;              /* 'secure-delete' */
 218   int nDeleteMerge;           /* 'deletemerge' */
 219
 220   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
 221   char **pzErrmsg;
 222
 223 #ifdef SQLITE_DEBUG
 224   int bPrefixIndex;               /* True to use prefix-indexes */
 225 #endif
 226 };
 227
 228 /* Current expected value of %_config table 'version' field. And
 229 ** the expected version if the 'secure-delete' option has ever been
 230 ** set on the table.  */
 231 #define FTS5_CURRENT_VERSION               4
 232 #define FTS5_CURRENT_VERSION_SECUREDELETE  5
 233
 234 #define FTS5_CONTENT_NORMAL   0
 235 #define FTS5_CONTENT_NONE     1
 236 #define FTS5_CONTENT_EXTERNAL 2
 237
 238 #define FTS5_DETAIL_FULL      0
 239 #define FTS5_DETAIL_NONE      1
 240 #define FTS5_DETAIL_COLUMNS   2
 241
 242 #define FTS5_PATTERN_NONE     0
 243 #define FTS5_PATTERN_LIKE     65  /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
 244 #define FTS5_PATTERN_GLOB     66  /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
 245
 246 int sqlite3Fts5ConfigParse(
 247     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 248 );
 249 void sqlite3Fts5ConfigFree(Fts5Config*);
 250
 251 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 252
 253 int sqlite3Fts5Tokenize(
 254   Fts5Config *pConfig,            /* FTS5 Configuration object */
 255   int flags,                      /* FTS5_TOKENIZE_* flags */
 256   const char *pText, int nText,   /* Text to tokenize */
 257   void *pCtx,                     /* Context passed to xToken() */
 258   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 259 );
 260
 261 void sqlite3Fts5Dequote(char *z);
 262
 263 /* Load the contents of the %_config table */
 264 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
 265
 266 /* Set the value of a single config attribute */
 267 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 268
 269 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 270
 271 /*
 272 ** End of interface to code in fts5_config.c.
 273 **************************************************************************/
 274
 275 /**************************************************************************
 276 ** Interface to code in fts5_buffer.c.
 277 */
 278
 279 /*
 280 ** Buffer object for the incremental building of string data.
 281 */
 282 typedef struct Fts5Buffer Fts5Buffer;
 283 struct Fts5Buffer {
 284   u8 *p;
 285   int n;
 286   int nSpace;
 287 };
 288
 289 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 290 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 291 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 292 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 293 void sqlite3Fts5BufferFree(Fts5Buffer*);
 294 void sqlite3Fts5BufferZero(Fts5Buffer*);
 295 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 296 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 297
 298 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 299
 300 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 301 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
 302 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 303 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 304 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 305
 306 #define fts5BufferGrow(pRc,pBuf,nn) ( \
 307   (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
 308     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 309 )
 310
 311 /* Write and decode big-endian 32-bit integer values */
 312 void sqlite3Fts5Put32(u8*, int);
 313 int sqlite3Fts5Get32(const u8*);
 314
 315 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 316 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
 317
 318 typedef struct Fts5PoslistReader Fts5PoslistReader;
 319 struct Fts5PoslistReader {
 320   /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
 321   const u8 *a;                    /* Position list to iterate through */
 322   int n;                          /* Size of buffer at a[] in bytes */
 323   int i;                          /* Current offset in a[] */
 324
 325   u8 bFlag;                       /* For client use (any custom purpose) */
 326
 327   /* Output variables */
 328   u8 bEof;                        /* Set to true at EOF */
 329   i64 iPos;                       /* (iCol<<32) + iPos */
 330 };
 331 int sqlite3Fts5PoslistReaderInit(
 332   const u8 *a, int n,             /* Poslist buffer to iterate through */
 333   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 334 );
 335 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 336
 337 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 338 struct Fts5PoslistWriter {
 339   i64 iPrev;
 340 };
 341 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
 342 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 343
 344 int sqlite3Fts5PoslistNext64(
 345   const u8 *a, int n,             /* Buffer containing poslist */
 346   int *pi,                        /* IN/OUT: Offset within a[] */
 347   i64 *piOff                      /* IN/OUT: Current offset */
 348 );
 349
 350 /* Malloc utility */
 351 void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
 352 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 353
 354 /* Character set tests (like isspace(), isalpha() etc.) */
 355 int sqlite3Fts5IsBareword(char t);
 356
 357
 358 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
 359 typedef struct Fts5Termset Fts5Termset;
 360 int sqlite3Fts5TermsetNew(Fts5Termset**);
 361 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
 362 void sqlite3Fts5TermsetFree(Fts5Termset*);
 363
 364 /*
 365 ** End of interface to code in fts5_buffer.c.
 366 **************************************************************************/
 367
 368 /**************************************************************************
 369 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 370 ** to access the data stored in the %_data table.
 371 */
 372
 373 typedef struct Fts5Index Fts5Index;
 374 typedef struct Fts5IndexIter Fts5IndexIter;
 375
 376 struct Fts5IndexIter {
 377   i64 iRowid;
 378   const u8 *pData;
 379   int nData;
 380   u8 bEof;
 381 };
 382
 383 #define sqlite3Fts5IterEof(x) ((x)->bEof)
 384
 385 /*
 386 ** Values used as part of the flags argument passed to IndexQuery().
 387 */
 388 #define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
 389 #define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
 390 #define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
 391 #define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */
 392
 393 /* The following are used internally by the fts5_index.c module. They are
 394 ** defined here only to make it easier to avoid clashes with the flags
 395 ** above. */
 396 #define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
 397 #define FTS5INDEX_QUERY_NOOUTPUT    0x0020
 398 #define FTS5INDEX_QUERY_SKIPHASH    0x0040
 399 #define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
 400 #define FTS5INDEX_QUERY_SCANONETERM 0x0100
 401
 402 /*
 403 ** Create/destroy an Fts5Index object.
 404 */
 405 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 406 int sqlite3Fts5IndexClose(Fts5Index *p);
 407
 408 /*
 409 ** Return a simple checksum value based on the arguments.
 410 */
 411 u64 sqlite3Fts5IndexEntryCksum(
 412   i64 iRowid,
 413   int iCol,
 414   int iPos,
 415   int iIdx,
 416   const char *pTerm,
 417   int nTerm
 418 );
 419
 420 /*
 421 ** Argument p points to a buffer containing utf-8 text that is n bytes in
 422 ** size. Return the number of bytes in the nChar character prefix of the
 423 ** buffer, or 0 if there are less than nChar characters in total.
 424 */
 425 int sqlite3Fts5IndexCharlenToBytelen(
 426   const char *p,
 427   int nByte,
 428   int nChar
 429 );
 430
 431 /*
 432 ** Open a new iterator to iterate though all rowids that match the
 433 ** specified token or token prefix.
 434 */
 435 int sqlite3Fts5IndexQuery(
 436   Fts5Index *p,                   /* FTS index to query */
 437   const char *pToken, int nToken, /* Token (or prefix) to query for */
 438   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
 439   Fts5Colset *pColset,            /* Match these columns only */
 440   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 441 );
 442
 443 /*
 444 ** The various operations on open token or token prefix iterators opened
 445 ** using sqlite3Fts5IndexQuery().
 446 */
 447 int sqlite3Fts5IterNext(Fts5IndexIter*);
 448 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 449
 450 /*
 451 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 452 */
 453 void sqlite3Fts5IterClose(Fts5IndexIter*);
 454
 455 /*
 456 ** Close the reader blob handle, if it is open.
 457 */
 458 void sqlite3Fts5IndexCloseReader(Fts5Index*);
 459
 460 /*
 461 ** This interface is used by the fts5vocab module.
 462 */
 463 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 464 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 465 void *sqlite3Fts5StructureRef(Fts5Index*);
 466 void sqlite3Fts5StructureRelease(void*);
 467 int sqlite3Fts5StructureTest(Fts5Index*, void*);
 468
 469 /*
 470 ** Used by xInstToken():
 471 */
 472 int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
 473
 474 /*
 475 ** Insert or remove data to or from the index. Each time a document is
 476 ** added to or removed from the index, this function is called one or more
 477 ** times.
 478 **
 479 ** For an insert, it must be called once for each token in the new document.
 480 ** If the operation is a delete, it must be called (at least) once for each
 481 ** unique token in the document with an iCol value less than zero. The iPos
 482 ** argument is ignored for a delete.
 483 */
 484 int sqlite3Fts5IndexWrite(
 485   Fts5Index *p,                   /* Index to write to */
 486   int iCol,                       /* Column token appears in (-ve -> delete) */
 487   int iPos,                       /* Position of token within column */
 488   const char *pToken, int nToken  /* Token to add or remove to or from index */
 489 );
 490
 491 /*
 492 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 493 ** document iDocid.
 494 */
 495 int sqlite3Fts5IndexBeginWrite(
 496   Fts5Index *p,                   /* Index to write to */
 497   int bDelete,                    /* True if current operation is a delete */
 498   i64 iDocid                      /* Docid to add or remove data from */
 499 );
 500
 501 /*
 502 ** Flush any data stored in the in-memory hash tables to the database.
 503 ** Also close any open blob handles.
 504 */
 505 int sqlite3Fts5IndexSync(Fts5Index *p);
 506
 507 /*
 508 ** Discard any data stored in the in-memory hash tables. Do not write it
 509 ** to the database. Additionally, assume that the contents of the %_data
 510 ** table may have changed on disk. So any in-memory caches of %_data
 511 ** records must be invalidated.
 512 */
 513 int sqlite3Fts5IndexRollback(Fts5Index *p);
 514
 515 /*
 516 ** Get or set the "averages" values.
 517 */
 518 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 519 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 520
 521 /*
 522 ** Functions called by the storage module as part of integrity-check.
 523 */
 524 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
 525
 526 /*
 527 ** Called during virtual module initialization to register UDF
 528 ** fts5_decode() with SQLite
 529 */
 530 int sqlite3Fts5IndexInit(sqlite3*);
 531
 532 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 533
 534 /*
 535 ** Return the total number of entries read from the %_data table by
 536 ** this connection since it was created.
 537 */
 538 int sqlite3Fts5IndexReads(Fts5Index *p);
 539
 540 int sqlite3Fts5IndexReinit(Fts5Index *p);
 541 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 542 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 543 int sqlite3Fts5IndexReset(Fts5Index *p);
 544
 545 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 546
 547 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 548 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
 549
 550 void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
 551
 552 /* Used to populate hash tables for xInstToken in detail=none/column mode. */
 553 int sqlite3Fts5IndexIterWriteTokendata(
 554     Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
 555 );
 556
 557 /*
 558 ** End of interface to code in fts5_index.c.
 559 **************************************************************************/
 560
 561 /**************************************************************************
 562 ** Interface to code in fts5_varint.c.
 563 */
 564 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 565 int sqlite3Fts5GetVarintLen(u32 iVal);
 566 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 567 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 568
 569 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
 570 #define fts5GetVarint    sqlite3Fts5GetVarint
 571
 572 #define fts5FastGetVarint32(a, iOff, nVal) {      \
 573   nVal = (a)[iOff++];                             \
 574   if( nVal & 0x80 ){                              \
 575     iOff--;                                       \
 576     iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
 577   }                                               \
 578 }
 579
 580
 581 /*
 582 ** End of interface to code in fts5_varint.c.
 583 **************************************************************************/
 584
 585
 586 /**************************************************************************
 587 ** Interface to code in fts5_main.c.
 588 */
 589
 590 /*
 591 ** Virtual-table object.
 592 */
 593 typedef struct Fts5Table Fts5Table;
 594 struct Fts5Table {
 595   sqlite3_vtab base;              /* Base class used by SQLite core */
 596   Fts5Config *pConfig;            /* Virtual table configuration */
 597   Fts5Index *pIndex;              /* Full-text index */
 598 };
 599
 600 int sqlite3Fts5GetTokenizer(
 601   Fts5Global*,
 602   const char **azArg,
 603   int nArg,
 604   Fts5Config*,
 605   char **pzErr
 606 );
 607
 608 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 609
 610 int sqlite3Fts5FlushToDisk(Fts5Table*);
 611
 612 /*
 613 ** End of interface to code in fts5.c.
 614 **************************************************************************/
 615
 616 /**************************************************************************
 617 ** Interface to code in fts5_hash.c.
 618 */
 619 typedef struct Fts5Hash Fts5Hash;
 620
 621 /*
 622 ** Create a hash table, free a hash table.
 623 */
 624 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 625 void sqlite3Fts5HashFree(Fts5Hash*);
 626
 627 int sqlite3Fts5HashWrite(
 628   Fts5Hash*,
 629   i64 iRowid,                     /* Rowid for this entry */
 630   int iCol,                       /* Column token appears in (-ve -> delete) */
 631   int iPos,                       /* Position of token within column */
 632   char bByte,
 633   const char *pToken, int nToken  /* Token to add or remove to or from index */
 634 );
 635
 636 /*
 637 ** Empty (but do not delete) a hash table.
 638 */
 639 void sqlite3Fts5HashClear(Fts5Hash*);
 640
 641 /*
 642 ** Return true if the hash is empty, false otherwise.
 643 */
 644 int sqlite3Fts5HashIsEmpty(Fts5Hash*);
 645
 646 int sqlite3Fts5HashQuery(
 647   Fts5Hash*,                      /* Hash table to query */
 648   int nPre,
 649   const char *pTerm, int nTerm,   /* Query term */
 650   void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
 651   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 652 );
 653
 654 int sqlite3Fts5HashScanInit(
 655   Fts5Hash*,                      /* Hash table to query */
 656   const char *pTerm, int nTerm    /* Query prefix */
 657 );
 658 void sqlite3Fts5HashScanNext(Fts5Hash*);
 659 int sqlite3Fts5HashScanEof(Fts5Hash*);
 660 void sqlite3Fts5HashScanEntry(Fts5Hash *,
 661   const char **pzTerm,            /* OUT: term (nul-terminated) */
 662   int *pnTerm,                    /* OUT: Size of term in bytes */
 663   const u8 **ppDoclist,           /* OUT: pointer to doclist */
 664   int *pnDoclist                  /* OUT: size of doclist in bytes */
 665 );
 666
 667
 668
 669 /*
 670 ** End of interface to code in fts5_hash.c.
 671 **************************************************************************/
 672
 673 /**************************************************************************
 674 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
 675 ** code to access the data stored in the %_content and %_docsize tables.
 676 */
 677
 678 #define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
 679 #define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
 680 #define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
 681
 682 typedef struct Fts5Storage Fts5Storage;
 683
 684 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 685 int sqlite3Fts5StorageClose(Fts5Storage *p);
 686 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 687
 688 int sqlite3Fts5DropAll(Fts5Config*);
 689 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 690
 691 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 692 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 693 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 694
 695 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
 696
 697 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 698 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 699
 700 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 701 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 702 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 703
 704 int sqlite3Fts5StorageSync(Fts5Storage *p);
 705 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 706
 707 int sqlite3Fts5StorageConfigValue(
 708     Fts5Storage *p, const char*, sqlite3_value*, int
 709 );
 710
 711 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 712 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 713 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 714 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 715 int sqlite3Fts5StorageReset(Fts5Storage *p);
 716
 717 /*
 718 ** End of interface to code in fts5_storage.c.
 719 **************************************************************************/
 720
 721
 722 /**************************************************************************
 723 ** Interface to code in fts5_expr.c.
 724 */
 725 typedef struct Fts5Expr Fts5Expr;
 726 typedef struct Fts5ExprNode Fts5ExprNode;
 727 typedef struct Fts5Parse Fts5Parse;
 728 typedef struct Fts5Token Fts5Token;
 729 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
 730 typedef struct Fts5ExprNearset Fts5ExprNearset;
 731
 732 struct Fts5Token {
 733   const char *p;                  /* Token text (not NULL terminated) */
 734   int n;                          /* Size of buffer p in bytes */
 735 };
 736
 737 /* Parse a MATCH expression. */
 738 int sqlite3Fts5ExprNew(
 739   Fts5Config *pConfig,
 740   int bPhraseToAnd,
 741   int iCol,                       /* Column on LHS of MATCH operator */
 742   const char *zExpr,
 743   Fts5Expr **ppNew,
 744   char **pzErr
 745 );
 746 int sqlite3Fts5ExprPattern(
 747   Fts5Config *pConfig,
 748   int bGlob,
 749   int iCol,
 750   const char *zText,
 751   Fts5Expr **pp
 752 );
 753
 754 /*
 755 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
 756 **     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
 757 **     rc = sqlite3Fts5ExprNext(pExpr)
 758 ** ){
 759 **   // The document with rowid iRowid matches the expression!
 760 **   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
 761 ** }
 762 */
 763 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
 764 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
 765 int sqlite3Fts5ExprEof(Fts5Expr*);
 766 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 767
 768 void sqlite3Fts5ExprFree(Fts5Expr*);
 769 int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
 770
 771 /* Called during startup to register a UDF with SQLite */
 772 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 773
 774 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 775 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 776 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 777
 778 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
 779 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 780 int sqlite3Fts5ExprPopulatePoslists(
 781     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 782 );
 783 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 784
 785 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 786
 787 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 788
 789 int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
 790 int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
 791 void sqlite3Fts5ExprClearTokens(Fts5Expr*);
 792
 793 /*******************************************
 794 ** The fts5_expr.c API above this point is used by the other hand-written
 795 ** C code in this module. The interfaces below this point are called by
 796 ** the parser code in fts5parse.y.  */
 797
 798 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 799
 800 Fts5ExprNode *sqlite3Fts5ParseNode(
 801   Fts5Parse *pParse,
 802   int eType,
 803   Fts5ExprNode *pLeft,
 804   Fts5ExprNode *pRight,
 805   Fts5ExprNearset *pNear
 806 );
 807
 808 Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
 809   Fts5Parse *pParse,
 810   Fts5ExprNode *pLeft,
 811   Fts5ExprNode *pRight
 812 );
 813
 814 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 815   Fts5Parse *pParse,
 816   Fts5ExprPhrase *pPhrase,
 817   Fts5Token *pToken,
 818   int bPrefix
 819 );
 820
 821 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
 822
 823 Fts5ExprNearset *sqlite3Fts5ParseNearset(
 824   Fts5Parse*,
 825   Fts5ExprNearset*,
 826   Fts5ExprPhrase*
 827 );
 828
 829 Fts5Colset *sqlite3Fts5ParseColset(
 830   Fts5Parse*,
 831   Fts5Colset*,
 832   Fts5Token *
 833 );
 834
 835 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 836 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 837 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 838
 839 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 840 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 841 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 842 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 843 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 844
 845 /*
 846 ** End of interface to code in fts5_expr.c.
 847 **************************************************************************/
 848
 849
 850
 851 /**************************************************************************
 852 ** Interface to code in fts5_aux.c.
 853 */
 854
 855 int sqlite3Fts5AuxInit(fts5_api*);
 856 /*
 857 ** End of interface to code in fts5_aux.c.
 858 **************************************************************************/
 859
 860 /**************************************************************************
 861 ** Interface to code in fts5_tokenizer.c.
 862 */
 863
 864 int sqlite3Fts5TokenizerInit(fts5_api*);
 865 int sqlite3Fts5TokenizerPattern(
 866     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
 867     Fts5Tokenizer *pTok
 868 );
 869 /*
 870 ** End of interface to code in fts5_tokenizer.c.
 871 **************************************************************************/
 872
 873 /**************************************************************************
 874 ** Interface to code in fts5_vocab.c.
 875 */
 876
 877 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 878
 879 /*
 880 ** End of interface to code in fts5_vocab.c.
 881 **************************************************************************/
 882
 883
 884 /**************************************************************************
 885 ** Interface to automatically generated code in fts5_unicode2.c.
 886 */
 887 int sqlite3Fts5UnicodeIsdiacritic(int c);
 888 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 889
 890 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
 891 int sqlite3Fts5UnicodeCategory(u32 iCode);
 892 void sqlite3Fts5UnicodeAscii(u8*, u8*);
 893 /*
 894 ** End of interface to code in fts5_unicode2.c.
 895 **************************************************************************/
 896
 897 #endif