ext/fts5/fts5Int.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTS5INT_H
  15 #define _FTS5INT_H
  16
  17 #include "fts5.h"
  18 #include "sqlite3ext.h"
  19 SQLITE_EXTENSION_INIT1
  20
  21 #include <string.h>
  22 #include <assert.h>
  23
  24 #ifndef SQLITE_AMALGAMATION
  25
  26 typedef unsigned char  u8;
  27 typedef unsigned int   u32;
  28 typedef unsigned short u16;
  29 typedef short i16;
  30 typedef sqlite3_int64 i64;
  31 typedef sqlite3_uint64 u64;
  32
  33 #define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
  34
  35 #define testcase(x)
  36 #define ALWAYS(x) 1
  37 #define NEVER(x) 0
  38
  39 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
  40 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
  41
  42 /*
  43 ** Constants for the largest and smallest possible 64-bit signed integers.
  44 */
  45 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
  46 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
  47
  48 #endif
  49
  50
  51 /*
  52 ** Maximum number of prefix indexes on single FTS5 table. This must be
  53 ** less than 32. If it is set to anything large than that, an #error
  54 ** directive in fts5_index.c will cause the build to fail.
  55 */
  56 #define FTS5_MAX_PREFIX_INDEXES 31
  57
  58 #define FTS5_DEFAULT_NEARDIST 10
  59 #define FTS5_DEFAULT_RANK     "bm25"
  60
  61 /* Name of rank and rowid columns */
  62 #define FTS5_RANK_NAME "rank"
  63 #define FTS5_ROWID_NAME "rowid"
  64
  65 #ifdef SQLITE_DEBUG
  66 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
  67 int sqlite3Fts5Corrupt(void);
  68 #else
  69 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
  70 #endif
  71
  72 /*
  73 ** The assert_nc() macro is similar to the assert() macro, except that it
  74 ** is used for assert() conditions that are true only if it can be
  75 ** guranteed that the database is not corrupt.
  76 */
  77 #ifdef SQLITE_DEBUG
  78 extern int sqlite3_fts5_may_be_corrupt;
  79 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
  80 #else
  81 # define assert_nc(x) assert(x)
  82 #endif
  83
  84 /* Mark a function parameter as unused, to suppress nuisance compiler
  85 ** warnings. */
  86 #ifndef UNUSED_PARAM
  87 # define UNUSED_PARAM(X)  (void)(X)
  88 #endif
  89
  90 #ifndef UNUSED_PARAM2
  91 # define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
  92 #endif
  93
  94 typedef struct Fts5Global Fts5Global;
  95 typedef struct Fts5Colset Fts5Colset;
  96
  97 /* If a NEAR() clump or phrase may only match a specific set of columns,
  98 ** then an object of the following type is used to record the set of columns.
  99 ** Each entry in the aiCol[] array is a column that may be matched.
 100 **
 101 ** This object is used by fts5_expr.c and fts5_index.c.
 102 */
 103 struct Fts5Colset {
 104   int nCol;
 105   int aiCol[1];
 106 };
 107
 108
 109
 110 /**************************************************************************
 111 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 112 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 113 */
 114
 115 typedef struct Fts5Config Fts5Config;
 116
 117 /*
 118 ** An instance of the following structure encodes all information that can
 119 ** be gleaned from the CREATE VIRTUAL TABLE statement.
 120 **
 121 ** And all information loaded from the %_config table.
 122 **
 123 ** nAutomerge:
 124 **   The minimum number of segments that an auto-merge operation should
 125 **   attempt to merge together. A value of 1 sets the object to use the
 126 **   compile time default. Zero disables auto-merge altogether.
 127 **
 128 ** zContent:
 129 **
 130 ** zContentRowid:
 131 **   The value of the content_rowid= option, if one was specified. Or
 132 **   the string "rowid" otherwise. This text is not quoted - if it is
 133 **   used as part of an SQL statement it needs to be quoted appropriately.
 134 **
 135 ** zContentExprlist:
 136 **
 137 ** pzErrmsg:
 138 **   This exists in order to allow the fts5_index.c module to return a
 139 **   decent error message if it encounters a file-format version it does
 140 **   not understand.
 141 **
 142 ** bColumnsize:
 143 **   True if the %_docsize table is created.
 144 **
 145 ** bPrefixIndex:
 146 **   This is only used for debugging. If set to false, any prefix indexes
 147 **   are ignored. This value is configured using:
 148 **
 149 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 150 **
 151 */
 152 struct Fts5Config {
 153   sqlite3 *db;                    /* Database handle */
 154   char *zDb;                      /* Database holding FTS index (e.g. "main") */
 155   char *zName;                    /* Name of FTS index */
 156   int nCol;                       /* Number of columns */
 157   char **azCol;                   /* Column names */
 158   u8 *abUnindexed;                /* True for unindexed columns */
 159   int nPrefix;                    /* Number of prefix indexes */
 160   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
 161   int eContent;                   /* An FTS5_CONTENT value */
 162   char *zContent;                 /* content table */
 163   char *zContentRowid;            /* "content_rowid=" option value */
 164   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
 165   int eDetail;                    /* FTS5_DETAIL_XXX value */
 166   char *zContentExprlist;
 167   Fts5Tokenizer *pTok;
 168   fts5_tokenizer *pTokApi;
 169
 170   /* Values loaded from the %_config table */
 171   int iCookie;                    /* Incremented when %_config is modified */
 172   int pgsz;                       /* Approximate page size used in %_data */
 173   int nAutomerge;                 /* 'automerge' setting */
 174   int nCrisisMerge;               /* Maximum allowed segments per level */
 175   int nHashSize;                  /* Bytes of memory for in-memory hash */
 176   char *zRank;                    /* Name of rank function */
 177   char *zRankArgs;                /* Arguments to rank function */
 178
 179   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
 180   char **pzErrmsg;
 181
 182 #ifdef SQLITE_DEBUG
 183   int bPrefixIndex;               /* True to use prefix-indexes */
 184 #endif
 185 };
 186
 187 /* Current expected value of %_config table 'version' field */
 188 #define FTS5_CURRENT_VERSION 4
 189
 190 #define FTS5_CONTENT_NORMAL   0
 191 #define FTS5_CONTENT_NONE     1
 192 #define FTS5_CONTENT_EXTERNAL 2
 193
 194 #define FTS5_DETAIL_FULL    0
 195 #define FTS5_DETAIL_NONE    1
 196 #define FTS5_DETAIL_COLUMNS 2
 197
 198
 199
 200 int sqlite3Fts5ConfigParse(
 201     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 202 );
 203 void sqlite3Fts5ConfigFree(Fts5Config*);
 204
 205 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 206
 207 int sqlite3Fts5Tokenize(
 208   Fts5Config *pConfig,            /* FTS5 Configuration object */
 209   int flags,                      /* FTS5_TOKENIZE_* flags */
 210   const char *pText, int nText,   /* Text to tokenize */
 211   void *pCtx,                     /* Context passed to xToken() */
 212   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 213 );
 214
 215 void sqlite3Fts5Dequote(char *z);
 216
 217 /* Load the contents of the %_config table */
 218 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
 219
 220 /* Set the value of a single config attribute */
 221 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 222
 223 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 224
 225 /*
 226 ** End of interface to code in fts5_config.c.
 227 **************************************************************************/
 228
 229 /**************************************************************************
 230 ** Interface to code in fts5_buffer.c.
 231 */
 232
 233 /*
 234 ** Buffer object for the incremental building of string data.
 235 */
 236 typedef struct Fts5Buffer Fts5Buffer;
 237 struct Fts5Buffer {
 238   u8 *p;
 239   int n;
 240   int nSpace;
 241 };
 242
 243 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 244 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 245 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 246 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 247 void sqlite3Fts5BufferFree(Fts5Buffer*);
 248 void sqlite3Fts5BufferZero(Fts5Buffer*);
 249 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 250 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 251
 252 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 253
 254 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 255 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
 256 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 257 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 258 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 259
 260 #define fts5BufferGrow(pRc,pBuf,nn) ( \
 261   (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
 262     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 263 )
 264
 265 /* Write and decode big-endian 32-bit integer values */
 266 void sqlite3Fts5Put32(u8*, int);
 267 int sqlite3Fts5Get32(const u8*);
 268
 269 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 270 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
 271
 272 typedef struct Fts5PoslistReader Fts5PoslistReader;
 273 struct Fts5PoslistReader {
 274   /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
 275   const u8 *a;                    /* Position list to iterate through */
 276   int n;                          /* Size of buffer at a[] in bytes */
 277   int i;                          /* Current offset in a[] */
 278
 279   u8 bFlag;                       /* For client use (any custom purpose) */
 280
 281   /* Output variables */
 282   u8 bEof;                        /* Set to true at EOF */
 283   i64 iPos;                       /* (iCol<<32) + iPos */
 284 };
 285 int sqlite3Fts5PoslistReaderInit(
 286   const u8 *a, int n,             /* Poslist buffer to iterate through */
 287   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 288 );
 289 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 290
 291 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 292 struct Fts5PoslistWriter {
 293   i64 iPrev;
 294 };
 295 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
 296 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 297
 298 int sqlite3Fts5PoslistNext64(
 299   const u8 *a, int n,             /* Buffer containing poslist */
 300   int *pi,                        /* IN/OUT: Offset within a[] */
 301   i64 *piOff                      /* IN/OUT: Current offset */
 302 );
 303
 304 /* Malloc utility */
 305 void *sqlite3Fts5MallocZero(int *pRc, int nByte);
 306 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 307
 308 /* Character set tests (like isspace(), isalpha() etc.) */
 309 int sqlite3Fts5IsBareword(char t);
 310
 311
 312 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
 313 typedef struct Fts5Termset Fts5Termset;
 314 int sqlite3Fts5TermsetNew(Fts5Termset**);
 315 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
 316 void sqlite3Fts5TermsetFree(Fts5Termset*);
 317
 318 /*
 319 ** End of interface to code in fts5_buffer.c.
 320 **************************************************************************/
 321
 322 /**************************************************************************
 323 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 324 ** to access the data stored in the %_data table.
 325 */
 326
 327 typedef struct Fts5Index Fts5Index;
 328 typedef struct Fts5IndexIter Fts5IndexIter;
 329
 330 struct Fts5IndexIter {
 331   i64 iRowid;
 332   const u8 *pData;
 333   int nData;
 334   u8 bEof;
 335 };
 336
 337 #define sqlite3Fts5IterEof(x) ((x)->bEof)
 338
 339 /*
 340 ** Values used as part of the flags argument passed to IndexQuery().
 341 */
 342 #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
 343 #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
 344 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 345 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 346
 347 /* The following are used internally by the fts5_index.c module. They are
 348 ** defined here only to make it easier to avoid clashes with the flags
 349 ** above. */
 350 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 351 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
 352
 353 /*
 354 ** Create/destroy an Fts5Index object.
 355 */
 356 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 357 int sqlite3Fts5IndexClose(Fts5Index *p);
 358
 359 /*
 360 ** Return a simple checksum value based on the arguments.
 361 */
 362 u64 sqlite3Fts5IndexEntryCksum(
 363   i64 iRowid,
 364   int iCol,
 365   int iPos,
 366   int iIdx,
 367   const char *pTerm,
 368   int nTerm
 369 );
 370
 371 /*
 372 ** Argument p points to a buffer containing utf-8 text that is n bytes in
 373 ** size. Return the number of bytes in the nChar character prefix of the
 374 ** buffer, or 0 if there are less than nChar characters in total.
 375 */
 376 int sqlite3Fts5IndexCharlenToBytelen(
 377   const char *p,
 378   int nByte,
 379   int nChar
 380 );
 381
 382 /*
 383 ** Open a new iterator to iterate though all rowids that match the
 384 ** specified token or token prefix.
 385 */
 386 int sqlite3Fts5IndexQuery(
 387   Fts5Index *p,                   /* FTS index to query */
 388   const char *pToken, int nToken, /* Token (or prefix) to query for */
 389   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
 390   Fts5Colset *pColset,            /* Match these columns only */
 391   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 392 );
 393
 394 /*
 395 ** The various operations on open token or token prefix iterators opened
 396 ** using sqlite3Fts5IndexQuery().
 397 */
 398 int sqlite3Fts5IterNext(Fts5IndexIter*);
 399 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 400
 401 /*
 402 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 403 */
 404 void sqlite3Fts5IterClose(Fts5IndexIter*);
 405
 406 /*
 407 ** This interface is used by the fts5vocab module.
 408 */
 409 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 410 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 411
 412
 413 /*
 414 ** Insert or remove data to or from the index. Each time a document is
 415 ** added to or removed from the index, this function is called one or more
 416 ** times.
 417 **
 418 ** For an insert, it must be called once for each token in the new document.
 419 ** If the operation is a delete, it must be called (at least) once for each
 420 ** unique token in the document with an iCol value less than zero. The iPos
 421 ** argument is ignored for a delete.
 422 */
 423 int sqlite3Fts5IndexWrite(
 424   Fts5Index *p,                   /* Index to write to */
 425   int iCol,                       /* Column token appears in (-ve -> delete) */
 426   int iPos,                       /* Position of token within column */
 427   const char *pToken, int nToken  /* Token to add or remove to or from index */
 428 );
 429
 430 /*
 431 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 432 ** document iDocid.
 433 */
 434 int sqlite3Fts5IndexBeginWrite(
 435   Fts5Index *p,                   /* Index to write to */
 436   int bDelete,                    /* True if current operation is a delete */
 437   i64 iDocid                      /* Docid to add or remove data from */
 438 );
 439
 440 /*
 441 ** Flush any data stored in the in-memory hash tables to the database.
 442 ** If the bCommit flag is true, also close any open blob handles.
 443 */
 444 int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
 445
 446 /*
 447 ** Discard any data stored in the in-memory hash tables. Do not write it
 448 ** to the database. Additionally, assume that the contents of the %_data
 449 ** table may have changed on disk. So any in-memory caches of %_data
 450 ** records must be invalidated.
 451 */
 452 int sqlite3Fts5IndexRollback(Fts5Index *p);
 453
 454 /*
 455 ** Get or set the "averages" values.
 456 */
 457 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 458 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 459
 460 /*
 461 ** Functions called by the storage module as part of integrity-check.
 462 */
 463 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
 464
 465 /*
 466 ** Called during virtual module initialization to register UDF
 467 ** fts5_decode() with SQLite
 468 */
 469 int sqlite3Fts5IndexInit(sqlite3*);
 470
 471 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 472
 473 /*
 474 ** Return the total number of entries read from the %_data table by
 475 ** this connection since it was created.
 476 */
 477 int sqlite3Fts5IndexReads(Fts5Index *p);
 478
 479 int sqlite3Fts5IndexReinit(Fts5Index *p);
 480 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 481 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 482
 483 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 484
 485 /*
 486 ** End of interface to code in fts5_index.c.
 487 **************************************************************************/
 488
 489 /**************************************************************************
 490 ** Interface to code in fts5_varint.c.
 491 */
 492 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 493 int sqlite3Fts5GetVarintLen(u32 iVal);
 494 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 495 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 496
 497 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
 498 #define fts5GetVarint    sqlite3Fts5GetVarint
 499
 500 #define fts5FastGetVarint32(a, iOff, nVal) {      \
 501   nVal = (a)[iOff++];                             \
 502   if( nVal & 0x80 ){                              \
 503     iOff--;                                       \
 504     iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
 505   }                                               \
 506 }
 507
 508
 509 /*
 510 ** End of interface to code in fts5_varint.c.
 511 **************************************************************************/
 512
 513
 514 /**************************************************************************
 515 ** Interface to code in fts5.c.
 516 */
 517
 518 int sqlite3Fts5GetTokenizer(
 519   Fts5Global*,
 520   const char **azArg,
 521   int nArg,
 522   Fts5Tokenizer**,
 523   fts5_tokenizer**,
 524   char **pzErr
 525 );
 526
 527 Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
 528
 529 /*
 530 ** End of interface to code in fts5.c.
 531 **************************************************************************/
 532
 533 /**************************************************************************
 534 ** Interface to code in fts5_hash.c.
 535 */
 536 typedef struct Fts5Hash Fts5Hash;
 537
 538 /*
 539 ** Create a hash table, free a hash table.
 540 */
 541 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 542 void sqlite3Fts5HashFree(Fts5Hash*);
 543
 544 int sqlite3Fts5HashWrite(
 545   Fts5Hash*,
 546   i64 iRowid,                     /* Rowid for this entry */
 547   int iCol,                       /* Column token appears in (-ve -> delete) */
 548   int iPos,                       /* Position of token within column */
 549   char bByte,
 550   const char *pToken, int nToken  /* Token to add or remove to or from index */
 551 );
 552
 553 /*
 554 ** Empty (but do not delete) a hash table.
 555 */
 556 void sqlite3Fts5HashClear(Fts5Hash*);
 557
 558 int sqlite3Fts5HashQuery(
 559   Fts5Hash*,                      /* Hash table to query */
 560   const char *pTerm, int nTerm,   /* Query term */
 561   const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
 562   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 563 );
 564
 565 int sqlite3Fts5HashScanInit(
 566   Fts5Hash*,                      /* Hash table to query */
 567   const char *pTerm, int nTerm    /* Query prefix */
 568 );
 569 void sqlite3Fts5HashScanNext(Fts5Hash*);
 570 int sqlite3Fts5HashScanEof(Fts5Hash*);
 571 void sqlite3Fts5HashScanEntry(Fts5Hash *,
 572   const char **pzTerm,            /* OUT: term (nul-terminated) */
 573   const u8 **ppDoclist,           /* OUT: pointer to doclist */
 574   int *pnDoclist                  /* OUT: size of doclist in bytes */
 575 );
 576
 577
 578 /*
 579 ** End of interface to code in fts5_hash.c.
 580 **************************************************************************/
 581
 582 /**************************************************************************
 583 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
 584 ** code to access the data stored in the %_content and %_docsize tables.
 585 */
 586
 587 #define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
 588 #define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
 589 #define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
 590
 591 typedef struct Fts5Storage Fts5Storage;
 592
 593 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 594 int sqlite3Fts5StorageClose(Fts5Storage *p);
 595 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 596
 597 int sqlite3Fts5DropAll(Fts5Config*);
 598 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 599
 600 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 601 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 602 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 603
 604 int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
 605
 606 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 607 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 608
 609 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 610 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 611 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 612
 613 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
 614 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 615
 616 int sqlite3Fts5StorageConfigValue(
 617     Fts5Storage *p, const char*, sqlite3_value*, int
 618 );
 619
 620 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 621 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 622 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 623 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 624
 625 /*
 626 ** End of interface to code in fts5_storage.c.
 627 **************************************************************************/
 628
 629
 630 /**************************************************************************
 631 ** Interface to code in fts5_expr.c.
 632 */
 633 typedef struct Fts5Expr Fts5Expr;
 634 typedef struct Fts5ExprNode Fts5ExprNode;
 635 typedef struct Fts5Parse Fts5Parse;
 636 typedef struct Fts5Token Fts5Token;
 637 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
 638 typedef struct Fts5ExprNearset Fts5ExprNearset;
 639
 640 struct Fts5Token {
 641   const char *p;                  /* Token text (not NULL terminated) */
 642   int n;                          /* Size of buffer p in bytes */
 643 };
 644
 645 /* Parse a MATCH expression. */
 646 int sqlite3Fts5ExprNew(
 647   Fts5Config *pConfig,
 648   const char *zExpr,
 649   Fts5Expr **ppNew,
 650   char **pzErr
 651 );
 652
 653 /*
 654 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
 655 **     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
 656 **     rc = sqlite3Fts5ExprNext(pExpr)
 657 ** ){
 658 **   // The document with rowid iRowid matches the expression!
 659 **   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
 660 ** }
 661 */
 662 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
 663 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
 664 int sqlite3Fts5ExprEof(Fts5Expr*);
 665 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 666
 667 void sqlite3Fts5ExprFree(Fts5Expr*);
 668
 669 /* Called during startup to register a UDF with SQLite */
 670 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 671
 672 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 673 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 674 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 675
 676 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
 677 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 678 int sqlite3Fts5ExprPopulatePoslists(
 679     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 680 );
 681 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 682 void sqlite3Fts5ExprClearEof(Fts5Expr*);
 683
 684 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 685
 686 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 687
 688 /*******************************************
 689 ** The fts5_expr.c API above this point is used by the other hand-written
 690 ** C code in this module. The interfaces below this point are called by
 691 ** the parser code in fts5parse.y.  */
 692
 693 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 694
 695 Fts5ExprNode *sqlite3Fts5ParseNode(
 696   Fts5Parse *pParse,
 697   int eType,
 698   Fts5ExprNode *pLeft,
 699   Fts5ExprNode *pRight,
 700   Fts5ExprNearset *pNear
 701 );
 702
 703 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 704   Fts5Parse *pParse,
 705   Fts5ExprPhrase *pPhrase,
 706   Fts5Token *pToken,
 707   int bPrefix
 708 );
 709
 710 Fts5ExprNearset *sqlite3Fts5ParseNearset(
 711   Fts5Parse*,
 712   Fts5ExprNearset*,
 713   Fts5ExprPhrase*
 714 );
 715
 716 Fts5Colset *sqlite3Fts5ParseColset(
 717   Fts5Parse*,
 718   Fts5Colset*,
 719   Fts5Token *
 720 );
 721
 722 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 723 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 724 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 725
 726 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 727 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
 728 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 729 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 730
 731 /*
 732 ** End of interface to code in fts5_expr.c.
 733 **************************************************************************/
 734
 735
 736
 737 /**************************************************************************
 738 ** Interface to code in fts5_aux.c.
 739 */
 740
 741 int sqlite3Fts5AuxInit(fts5_api*);
 742 /*
 743 ** End of interface to code in fts5_aux.c.
 744 **************************************************************************/
 745
 746 /**************************************************************************
 747 ** Interface to code in fts5_tokenizer.c.
 748 */
 749
 750 int sqlite3Fts5TokenizerInit(fts5_api*);
 751 /*
 752 ** End of interface to code in fts5_tokenizer.c.
 753 **************************************************************************/
 754
 755 /**************************************************************************
 756 ** Interface to code in fts5_vocab.c.
 757 */
 758
 759 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 760
 761 /*
 762 ** End of interface to code in fts5_vocab.c.
 763 **************************************************************************/
 764
 765
 766 /**************************************************************************
 767 ** Interface to automatically generated code in fts5_unicode2.c.
 768 */
 769 int sqlite3Fts5UnicodeIsalnum(int c);
 770 int sqlite3Fts5UnicodeIsdiacritic(int c);
 771 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 772 /*
 773 ** End of interface to code in fts5_unicode2.c.
 774 **************************************************************************/
 775
 776 #endif