ext/fts5/fts5Int.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTS5INT_H
  15 #define _FTS5INT_H
  16
  17 #include "fts5.h"
  18 #include "sqlite3ext.h"
  19 SQLITE_EXTENSION_INIT1
  20
  21 #include <string.h>
  22 #include <assert.h>
  23
  24 #ifndef SQLITE_AMALGAMATION
  25
  26 typedef unsigned char  u8;
  27 typedef unsigned int   u32;
  28 typedef unsigned short u16;
  29 typedef short i16;
  30 typedef sqlite3_int64 i64;
  31 typedef sqlite3_uint64 u64;
  32
  33 #ifndef ArraySize
  34 # define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
  35 #endif
  36
  37 #define testcase(x)
  38 #define ALWAYS(x) 1
  39 #define NEVER(x) 0
  40
  41 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
  42 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
  43
  44 /*
  45 ** Constants for the largest and smallest possible 64-bit signed integers.
  46 */
  47 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
  48 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
  49
  50 #endif
  51
  52 /* Truncate very long tokens to this many bytes. Hard limit is
  53 ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
  54 ** field that occurs at the start of each leaf page (see fts5_index.c). */
  55 #define FTS5_MAX_TOKEN_SIZE 32768
  56
  57 /*
  58 ** Maximum number of prefix indexes on single FTS5 table. This must be
  59 ** less than 32. If it is set to anything large than that, an #error
  60 ** directive in fts5_index.c will cause the build to fail.
  61 */
  62 #define FTS5_MAX_PREFIX_INDEXES 31
  63
  64 #define FTS5_DEFAULT_NEARDIST 10
  65 #define FTS5_DEFAULT_RANK     "bm25"
  66
  67 /* Name of rank and rowid columns */
  68 #define FTS5_RANK_NAME "rank"
  69 #define FTS5_ROWID_NAME "rowid"
  70
  71 #ifdef SQLITE_DEBUG
  72 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
  73 int sqlite3Fts5Corrupt(void);
  74 #else
  75 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
  76 #endif
  77
  78 /*
  79 ** The assert_nc() macro is similar to the assert() macro, except that it
  80 ** is used for assert() conditions that are true only if it can be
  81 ** guranteed that the database is not corrupt.
  82 */
  83 #ifdef SQLITE_DEBUG
  84 extern int sqlite3_fts5_may_be_corrupt;
  85 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
  86 #else
  87 # define assert_nc(x) assert(x)
  88 #endif
  89
  90 /* Mark a function parameter as unused, to suppress nuisance compiler
  91 ** warnings. */
  92 #ifndef UNUSED_PARAM
  93 # define UNUSED_PARAM(X)  (void)(X)
  94 #endif
  95
  96 #ifndef UNUSED_PARAM2
  97 # define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
  98 #endif
  99
 100 typedef struct Fts5Global Fts5Global;
 101 typedef struct Fts5Colset Fts5Colset;
 102
 103 /* If a NEAR() clump or phrase may only match a specific set of columns,
 104 ** then an object of the following type is used to record the set of columns.
 105 ** Each entry in the aiCol[] array is a column that may be matched.
 106 **
 107 ** This object is used by fts5_expr.c and fts5_index.c.
 108 */
 109 struct Fts5Colset {
 110   int nCol;
 111   int aiCol[1];
 112 };
 113
 114
 115
 116 /**************************************************************************
 117 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 118 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 119 */
 120
 121 typedef struct Fts5Config Fts5Config;
 122
 123 /*
 124 ** An instance of the following structure encodes all information that can
 125 ** be gleaned from the CREATE VIRTUAL TABLE statement.
 126 **
 127 ** And all information loaded from the %_config table.
 128 **
 129 ** nAutomerge:
 130 **   The minimum number of segments that an auto-merge operation should
 131 **   attempt to merge together. A value of 1 sets the object to use the
 132 **   compile time default. Zero disables auto-merge altogether.
 133 **
 134 ** zContent:
 135 **
 136 ** zContentRowid:
 137 **   The value of the content_rowid= option, if one was specified. Or
 138 **   the string "rowid" otherwise. This text is not quoted - if it is
 139 **   used as part of an SQL statement it needs to be quoted appropriately.
 140 **
 141 ** zContentExprlist:
 142 **
 143 ** pzErrmsg:
 144 **   This exists in order to allow the fts5_index.c module to return a
 145 **   decent error message if it encounters a file-format version it does
 146 **   not understand.
 147 **
 148 ** bColumnsize:
 149 **   True if the %_docsize table is created.
 150 **
 151 ** bPrefixIndex:
 152 **   This is only used for debugging. If set to false, any prefix indexes
 153 **   are ignored. This value is configured using:
 154 **
 155 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 156 **
 157 */
 158 struct Fts5Config {
 159   sqlite3 *db;                    /* Database handle */
 160   char *zDb;                      /* Database holding FTS index (e.g. "main") */
 161   char *zName;                    /* Name of FTS index */
 162   int nCol;                       /* Number of columns */
 163   char **azCol;                   /* Column names */
 164   u8 *abUnindexed;                /* True for unindexed columns */
 165   int nPrefix;                    /* Number of prefix indexes */
 166   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
 167   int eContent;                   /* An FTS5_CONTENT value */
 168   char *zContent;                 /* content table */
 169   char *zContentRowid;            /* "content_rowid=" option value */
 170   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
 171   int eDetail;                    /* FTS5_DETAIL_XXX value */
 172   char *zContentExprlist;
 173   Fts5Tokenizer *pTok;
 174   fts5_tokenizer *pTokApi;
 175
 176   /* Values loaded from the %_config table */
 177   int iCookie;                    /* Incremented when %_config is modified */
 178   int pgsz;                       /* Approximate page size used in %_data */
 179   int nAutomerge;                 /* 'automerge' setting */
 180   int nCrisisMerge;               /* Maximum allowed segments per level */
 181   int nUsermerge;                 /* 'usermerge' setting */
 182   int nHashSize;                  /* Bytes of memory for in-memory hash */
 183   char *zRank;                    /* Name of rank function */
 184   char *zRankArgs;                /* Arguments to rank function */
 185
 186   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
 187   char **pzErrmsg;
 188
 189 #ifdef SQLITE_DEBUG
 190   int bPrefixIndex;               /* True to use prefix-indexes */
 191 #endif
 192 };
 193
 194 /* Current expected value of %_config table 'version' field */
 195 #define FTS5_CURRENT_VERSION 4
 196
 197 #define FTS5_CONTENT_NORMAL   0
 198 #define FTS5_CONTENT_NONE     1
 199 #define FTS5_CONTENT_EXTERNAL 2
 200
 201 #define FTS5_DETAIL_FULL    0
 202 #define FTS5_DETAIL_NONE    1
 203 #define FTS5_DETAIL_COLUMNS 2
 204
 205
 206
 207 int sqlite3Fts5ConfigParse(
 208     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 209 );
 210 void sqlite3Fts5ConfigFree(Fts5Config*);
 211
 212 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 213
 214 int sqlite3Fts5Tokenize(
 215   Fts5Config *pConfig,            /* FTS5 Configuration object */
 216   int flags,                      /* FTS5_TOKENIZE_* flags */
 217   const char *pText, int nText,   /* Text to tokenize */
 218   void *pCtx,                     /* Context passed to xToken() */
 219   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 220 );
 221
 222 void sqlite3Fts5Dequote(char *z);
 223
 224 /* Load the contents of the %_config table */
 225 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
 226
 227 /* Set the value of a single config attribute */
 228 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 229
 230 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 231
 232 /*
 233 ** End of interface to code in fts5_config.c.
 234 **************************************************************************/
 235
 236 /**************************************************************************
 237 ** Interface to code in fts5_buffer.c.
 238 */
 239
 240 /*
 241 ** Buffer object for the incremental building of string data.
 242 */
 243 typedef struct Fts5Buffer Fts5Buffer;
 244 struct Fts5Buffer {
 245   u8 *p;
 246   int n;
 247   int nSpace;
 248 };
 249
 250 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 251 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 252 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 253 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 254 void sqlite3Fts5BufferFree(Fts5Buffer*);
 255 void sqlite3Fts5BufferZero(Fts5Buffer*);
 256 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 257 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 258
 259 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 260
 261 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 262 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
 263 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 264 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 265 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 266
 267 #define fts5BufferGrow(pRc,pBuf,nn) ( \
 268   (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
 269     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 270 )
 271
 272 /* Write and decode big-endian 32-bit integer values */
 273 void sqlite3Fts5Put32(u8*, int);
 274 int sqlite3Fts5Get32(const u8*);
 275
 276 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 277 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
 278
 279 typedef struct Fts5PoslistReader Fts5PoslistReader;
 280 struct Fts5PoslistReader {
 281   /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
 282   const u8 *a;                    /* Position list to iterate through */
 283   int n;                          /* Size of buffer at a[] in bytes */
 284   int i;                          /* Current offset in a[] */
 285
 286   u8 bFlag;                       /* For client use (any custom purpose) */
 287
 288   /* Output variables */
 289   u8 bEof;                        /* Set to true at EOF */
 290   i64 iPos;                       /* (iCol<<32) + iPos */
 291 };
 292 int sqlite3Fts5PoslistReaderInit(
 293   const u8 *a, int n,             /* Poslist buffer to iterate through */
 294   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 295 );
 296 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 297
 298 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 299 struct Fts5PoslistWriter {
 300   i64 iPrev;
 301 };
 302 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
 303 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 304
 305 int sqlite3Fts5PoslistNext64(
 306   const u8 *a, int n,             /* Buffer containing poslist */
 307   int *pi,                        /* IN/OUT: Offset within a[] */
 308   i64 *piOff                      /* IN/OUT: Current offset */
 309 );
 310
 311 /* Malloc utility */
 312 void *sqlite3Fts5MallocZero(int *pRc, int nByte);
 313 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 314
 315 /* Character set tests (like isspace(), isalpha() etc.) */
 316 int sqlite3Fts5IsBareword(char t);
 317
 318
 319 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
 320 typedef struct Fts5Termset Fts5Termset;
 321 int sqlite3Fts5TermsetNew(Fts5Termset**);
 322 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
 323 void sqlite3Fts5TermsetFree(Fts5Termset*);
 324
 325 /*
 326 ** End of interface to code in fts5_buffer.c.
 327 **************************************************************************/
 328
 329 /**************************************************************************
 330 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 331 ** to access the data stored in the %_data table.
 332 */
 333
 334 typedef struct Fts5Index Fts5Index;
 335 typedef struct Fts5IndexIter Fts5IndexIter;
 336
 337 struct Fts5IndexIter {
 338   i64 iRowid;
 339   const u8 *pData;
 340   int nData;
 341   u8 bEof;
 342 };
 343
 344 #define sqlite3Fts5IterEof(x) ((x)->bEof)
 345
 346 /*
 347 ** Values used as part of the flags argument passed to IndexQuery().
 348 */
 349 #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
 350 #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
 351 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 352 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 353
 354 /* The following are used internally by the fts5_index.c module. They are
 355 ** defined here only to make it easier to avoid clashes with the flags
 356 ** above. */
 357 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 358 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
 359
 360 /*
 361 ** Create/destroy an Fts5Index object.
 362 */
 363 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 364 int sqlite3Fts5IndexClose(Fts5Index *p);
 365
 366 /*
 367 ** Return a simple checksum value based on the arguments.
 368 */
 369 u64 sqlite3Fts5IndexEntryCksum(
 370   i64 iRowid,
 371   int iCol,
 372   int iPos,
 373   int iIdx,
 374   const char *pTerm,
 375   int nTerm
 376 );
 377
 378 /*
 379 ** Argument p points to a buffer containing utf-8 text that is n bytes in
 380 ** size. Return the number of bytes in the nChar character prefix of the
 381 ** buffer, or 0 if there are less than nChar characters in total.
 382 */
 383 int sqlite3Fts5IndexCharlenToBytelen(
 384   const char *p,
 385   int nByte,
 386   int nChar
 387 );
 388
 389 /*
 390 ** Open a new iterator to iterate though all rowids that match the
 391 ** specified token or token prefix.
 392 */
 393 int sqlite3Fts5IndexQuery(
 394   Fts5Index *p,                   /* FTS index to query */
 395   const char *pToken, int nToken, /* Token (or prefix) to query for */
 396   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
 397   Fts5Colset *pColset,            /* Match these columns only */
 398   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 399 );
 400
 401 /*
 402 ** The various operations on open token or token prefix iterators opened
 403 ** using sqlite3Fts5IndexQuery().
 404 */
 405 int sqlite3Fts5IterNext(Fts5IndexIter*);
 406 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 407
 408 /*
 409 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 410 */
 411 void sqlite3Fts5IterClose(Fts5IndexIter*);
 412
 413 /*
 414 ** This interface is used by the fts5vocab module.
 415 */
 416 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 417 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 418
 419
 420 /*
 421 ** Insert or remove data to or from the index. Each time a document is
 422 ** added to or removed from the index, this function is called one or more
 423 ** times.
 424 **
 425 ** For an insert, it must be called once for each token in the new document.
 426 ** If the operation is a delete, it must be called (at least) once for each
 427 ** unique token in the document with an iCol value less than zero. The iPos
 428 ** argument is ignored for a delete.
 429 */
 430 int sqlite3Fts5IndexWrite(
 431   Fts5Index *p,                   /* Index to write to */
 432   int iCol,                       /* Column token appears in (-ve -> delete) */
 433   int iPos,                       /* Position of token within column */
 434   const char *pToken, int nToken  /* Token to add or remove to or from index */
 435 );
 436
 437 /*
 438 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 439 ** document iDocid.
 440 */
 441 int sqlite3Fts5IndexBeginWrite(
 442   Fts5Index *p,                   /* Index to write to */
 443   int bDelete,                    /* True if current operation is a delete */
 444   i64 iDocid                      /* Docid to add or remove data from */
 445 );
 446
 447 /*
 448 ** Flush any data stored in the in-memory hash tables to the database.
 449 ** Also close any open blob handles.
 450 */
 451 int sqlite3Fts5IndexSync(Fts5Index *p);
 452
 453 /*
 454 ** Discard any data stored in the in-memory hash tables. Do not write it
 455 ** to the database. Additionally, assume that the contents of the %_data
 456 ** table may have changed on disk. So any in-memory caches of %_data
 457 ** records must be invalidated.
 458 */
 459 int sqlite3Fts5IndexRollback(Fts5Index *p);
 460
 461 /*
 462 ** Get or set the "averages" values.
 463 */
 464 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 465 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 466
 467 /*
 468 ** Functions called by the storage module as part of integrity-check.
 469 */
 470 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
 471
 472 /*
 473 ** Called during virtual module initialization to register UDF
 474 ** fts5_decode() with SQLite
 475 */
 476 int sqlite3Fts5IndexInit(sqlite3*);
 477
 478 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 479
 480 /*
 481 ** Return the total number of entries read from the %_data table by
 482 ** this connection since it was created.
 483 */
 484 int sqlite3Fts5IndexReads(Fts5Index *p);
 485
 486 int sqlite3Fts5IndexReinit(Fts5Index *p);
 487 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 488 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 489 int sqlite3Fts5IndexReset(Fts5Index *p);
 490
 491 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 492
 493 /*
 494 ** End of interface to code in fts5_index.c.
 495 **************************************************************************/
 496
 497 /**************************************************************************
 498 ** Interface to code in fts5_varint.c.
 499 */
 500 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 501 int sqlite3Fts5GetVarintLen(u32 iVal);
 502 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 503 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 504
 505 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
 506 #define fts5GetVarint    sqlite3Fts5GetVarint
 507
 508 #define fts5FastGetVarint32(a, iOff, nVal) {      \
 509   nVal = (a)[iOff++];                             \
 510   if( nVal & 0x80 ){                              \
 511     iOff--;                                       \
 512     iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
 513   }                                               \
 514 }
 515
 516
 517 /*
 518 ** End of interface to code in fts5_varint.c.
 519 **************************************************************************/
 520
 521
 522 /**************************************************************************
 523 ** Interface to code in fts5.c.
 524 */
 525
 526 int sqlite3Fts5GetTokenizer(
 527   Fts5Global*,
 528   const char **azArg,
 529   int nArg,
 530   Fts5Tokenizer**,
 531   fts5_tokenizer**,
 532   char **pzErr
 533 );
 534
 535 Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
 536
 537 /*
 538 ** End of interface to code in fts5.c.
 539 **************************************************************************/
 540
 541 /**************************************************************************
 542 ** Interface to code in fts5_hash.c.
 543 */
 544 typedef struct Fts5Hash Fts5Hash;
 545
 546 /*
 547 ** Create a hash table, free a hash table.
 548 */
 549 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 550 void sqlite3Fts5HashFree(Fts5Hash*);
 551
 552 int sqlite3Fts5HashWrite(
 553   Fts5Hash*,
 554   i64 iRowid,                     /* Rowid for this entry */
 555   int iCol,                       /* Column token appears in (-ve -> delete) */
 556   int iPos,                       /* Position of token within column */
 557   char bByte,
 558   const char *pToken, int nToken  /* Token to add or remove to or from index */
 559 );
 560
 561 /*
 562 ** Empty (but do not delete) a hash table.
 563 */
 564 void sqlite3Fts5HashClear(Fts5Hash*);
 565
 566 int sqlite3Fts5HashQuery(
 567   Fts5Hash*,                      /* Hash table to query */
 568   const char *pTerm, int nTerm,   /* Query term */
 569   const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
 570   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 571 );
 572
 573 int sqlite3Fts5HashScanInit(
 574   Fts5Hash*,                      /* Hash table to query */
 575   const char *pTerm, int nTerm    /* Query prefix */
 576 );
 577 void sqlite3Fts5HashScanNext(Fts5Hash*);
 578 int sqlite3Fts5HashScanEof(Fts5Hash*);
 579 void sqlite3Fts5HashScanEntry(Fts5Hash *,
 580   const char **pzTerm,            /* OUT: term (nul-terminated) */
 581   const u8 **ppDoclist,           /* OUT: pointer to doclist */
 582   int *pnDoclist                  /* OUT: size of doclist in bytes */
 583 );
 584
 585
 586 /*
 587 ** End of interface to code in fts5_hash.c.
 588 **************************************************************************/
 589
 590 /**************************************************************************
 591 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
 592 ** code to access the data stored in the %_content and %_docsize tables.
 593 */
 594
 595 #define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
 596 #define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
 597 #define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
 598
 599 typedef struct Fts5Storage Fts5Storage;
 600
 601 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 602 int sqlite3Fts5StorageClose(Fts5Storage *p);
 603 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 604
 605 int sqlite3Fts5DropAll(Fts5Config*);
 606 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 607
 608 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 609 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 610 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 611
 612 int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
 613
 614 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 615 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 616
 617 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 618 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 619 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 620
 621 int sqlite3Fts5StorageSync(Fts5Storage *p);
 622 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 623
 624 int sqlite3Fts5StorageConfigValue(
 625     Fts5Storage *p, const char*, sqlite3_value*, int
 626 );
 627
 628 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 629 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 630 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 631 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 632 int sqlite3Fts5StorageReset(Fts5Storage *p);
 633
 634 /*
 635 ** End of interface to code in fts5_storage.c.
 636 **************************************************************************/
 637
 638
 639 /**************************************************************************
 640 ** Interface to code in fts5_expr.c.
 641 */
 642 typedef struct Fts5Expr Fts5Expr;
 643 typedef struct Fts5ExprNode Fts5ExprNode;
 644 typedef struct Fts5Parse Fts5Parse;
 645 typedef struct Fts5Token Fts5Token;
 646 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
 647 typedef struct Fts5ExprNearset Fts5ExprNearset;
 648
 649 struct Fts5Token {
 650   const char *p;                  /* Token text (not NULL terminated) */
 651   int n;                          /* Size of buffer p in bytes */
 652 };
 653
 654 /* Parse a MATCH expression. */
 655 int sqlite3Fts5ExprNew(
 656   Fts5Config *pConfig,
 657   int iCol,                       /* Column on LHS of MATCH operator */
 658   const char *zExpr,
 659   Fts5Expr **ppNew,
 660   char **pzErr
 661 );
 662
 663 /*
 664 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
 665 **     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
 666 **     rc = sqlite3Fts5ExprNext(pExpr)
 667 ** ){
 668 **   // The document with rowid iRowid matches the expression!
 669 **   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
 670 ** }
 671 */
 672 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
 673 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
 674 int sqlite3Fts5ExprEof(Fts5Expr*);
 675 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 676
 677 void sqlite3Fts5ExprFree(Fts5Expr*);
 678
 679 /* Called during startup to register a UDF with SQLite */
 680 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 681
 682 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 683 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 684 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 685
 686 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
 687 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 688 int sqlite3Fts5ExprPopulatePoslists(
 689     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 690 );
 691 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 692
 693 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 694
 695 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 696
 697 /*******************************************
 698 ** The fts5_expr.c API above this point is used by the other hand-written
 699 ** C code in this module. The interfaces below this point are called by
 700 ** the parser code in fts5parse.y.  */
 701
 702 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 703
 704 Fts5ExprNode *sqlite3Fts5ParseNode(
 705   Fts5Parse *pParse,
 706   int eType,
 707   Fts5ExprNode *pLeft,
 708   Fts5ExprNode *pRight,
 709   Fts5ExprNearset *pNear
 710 );
 711
 712 Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
 713   Fts5Parse *pParse,
 714   Fts5ExprNode *pLeft,
 715   Fts5ExprNode *pRight
 716 );
 717
 718 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 719   Fts5Parse *pParse,
 720   Fts5ExprPhrase *pPhrase,
 721   Fts5Token *pToken,
 722   int bPrefix
 723 );
 724
 725 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
 726
 727 Fts5ExprNearset *sqlite3Fts5ParseNearset(
 728   Fts5Parse*,
 729   Fts5ExprNearset*,
 730   Fts5ExprPhrase*
 731 );
 732
 733 Fts5Colset *sqlite3Fts5ParseColset(
 734   Fts5Parse*,
 735   Fts5Colset*,
 736   Fts5Token *
 737 );
 738
 739 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 740 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 741 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 742
 743 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 744 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 745 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 746 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 747 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 748
 749 /*
 750 ** End of interface to code in fts5_expr.c.
 751 **************************************************************************/
 752
 753
 754
 755 /**************************************************************************
 756 ** Interface to code in fts5_aux.c.
 757 */
 758
 759 int sqlite3Fts5AuxInit(fts5_api*);
 760 /*
 761 ** End of interface to code in fts5_aux.c.
 762 **************************************************************************/
 763
 764 /**************************************************************************
 765 ** Interface to code in fts5_tokenizer.c.
 766 */
 767
 768 int sqlite3Fts5TokenizerInit(fts5_api*);
 769 /*
 770 ** End of interface to code in fts5_tokenizer.c.
 771 **************************************************************************/
 772
 773 /**************************************************************************
 774 ** Interface to code in fts5_vocab.c.
 775 */
 776
 777 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 778
 779 /*
 780 ** End of interface to code in fts5_vocab.c.
 781 **************************************************************************/
 782
 783
 784 /**************************************************************************
 785 ** Interface to automatically generated code in fts5_unicode2.c.
 786 */
 787 int sqlite3Fts5UnicodeIsalnum(int c);
 788 int sqlite3Fts5UnicodeIsdiacritic(int c);
 789 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 790 /*
 791 ** End of interface to code in fts5_unicode2.c.
 792 **************************************************************************/
 793
 794 #endif