ext/fts5/fts5.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** Interfaces to extend FTS5. Using the interfaces defined in this file,
  14 ** FTS5 may be extended with:
  15 **
  16 **     * custom tokenizers, and
  17 **     * custom auxiliary functions.
  18 */
  19
  20
  21 #ifndef _FTS5_H
  22 #define _FTS5_H
  23
  24 #include "sqlite3.h"
  25
  26 #ifdef __cplusplus
  27 extern "C" {
  28 #endif
  29
  30 /*************************************************************************
  31 ** CUSTOM AUXILIARY FUNCTIONS
  32 **
  33 ** Virtual table implementations may overload SQL functions by implementing
  34 ** the sqlite3_module.xFindFunction() method.
  35 */
  36
  37 typedef struct Fts5ExtensionApi Fts5ExtensionApi;
  38 typedef struct Fts5Context Fts5Context;
  39 typedef struct Fts5PhraseIter Fts5PhraseIter;
  40
  41 typedef void (*fts5_extension_function)(
  42   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  43   Fts5Context *pFts,              /* First arg to pass to pApi functions */
  44   sqlite3_context *pCtx,          /* Context for returning result/error */
  45   int nVal,                       /* Number of values in apVal[] array */
  46   sqlite3_value **apVal           /* Array of trailing arguments */
  47 );
  48
  49 struct Fts5PhraseIter {
  50   const unsigned char *a;
  51   const unsigned char *b;
  52 };
  53
  54 /*
  55 ** EXTENSION API FUNCTIONS
  56 **
  57 ** xUserData(pFts):
  58 **   Return a copy of the pUserData pointer passed to the xCreateFunction()
  59 **   API when the extension function was registered.
  60 **
  61 ** xColumnTotalSize(pFts, iCol, pnToken):
  62 **   If parameter iCol is less than zero, set output variable *pnToken
  63 **   to the total number of tokens in the FTS5 table. Or, if iCol is
  64 **   non-negative but less than the number of columns in the table, return
  65 **   the total number of tokens in column iCol, considering all rows in
  66 **   the FTS5 table.
  67 **
  68 **   If parameter iCol is greater than or equal to the number of columns
  69 **   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
  70 **   an OOM condition or IO error), an appropriate SQLite error code is
  71 **   returned.
  72 **
  73 ** xColumnCount(pFts):
  74 **   Return the number of columns in the table.
  75 **
  76 ** xColumnSize(pFts, iCol, pnToken):
  77 **   If parameter iCol is less than zero, set output variable *pnToken
  78 **   to the total number of tokens in the current row. Or, if iCol is
  79 **   non-negative but less than the number of columns in the table, set
  80 **   *pnToken to the number of tokens in column iCol of the current row.
  81 **
  82 **   If parameter iCol is greater than or equal to the number of columns
  83 **   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
  84 **   an OOM condition or IO error), an appropriate SQLite error code is
  85 **   returned.
  86 **
  87 **   This function may be quite inefficient if used with an FTS5 table
  88 **   created with the "columnsize=0" option.
  89 **
  90 ** xColumnText:
  91 **   If parameter iCol is less than zero, or greater than or equal to the
  92 **   number of columns in the table, SQLITE_RANGE is returned.
  93 **
  94 **   Otherwise, this function attempts to retrieve the text of column iCol of
  95 **   the current document. If successful, (*pz) is set to point to a buffer
  96 **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
  97 **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
  98 **   if an error occurs, an SQLite error code is returned and the final values
  99 **   of (*pz) and (*pn) are undefined.
 100 **
 101 ** xPhraseCount:
 102 **   Returns the number of phrases in the current query expression.
 103 **
 104 ** xPhraseSize:
 105 **   If parameter iCol is less than zero, or greater than or equal to the
 106 **   number of phrases in the current query, as returned by xPhraseCount,
 107 **   0 is returned. Otherwise, this function returns the number of tokens in
 108 **   phrase iPhrase of the query. Phrases are numbered starting from zero.
 109 **
 110 ** xInstCount:
 111 **   Set *pnInst to the total number of occurrences of all phrases within
 112 **   the query within the current row. Return SQLITE_OK if successful, or
 113 **   an error code (i.e. SQLITE_NOMEM) if an error occurs.
 114 **
 115 **   This API can be quite slow if used with an FTS5 table created with the
 116 **   "detail=none" or "detail=column" option. If the FTS5 table is created
 117 **   with either "detail=none" or "detail=column" and "content=" option
 118 **   (i.e. if it is a contentless table), then this API always returns 0.
 119 **
 120 ** xInst:
 121 **   Query for the details of phrase match iIdx within the current row.
 122 **   Phrase matches are numbered starting from zero, so the iIdx argument
 123 **   should be greater than or equal to zero and smaller than the value
 124 **   output by xInstCount(). If iIdx is less than zero or greater than
 125 **   or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
 126 **
 127 **   Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
 128 **   to the column in which it occurs and *piOff the token offset of the
 129 **   first token of the phrase. SQLITE_OK is returned if successful, or an
 130 **   error code (i.e. SQLITE_NOMEM) if an error occurs.
 131 **
 132 **   This API can be quite slow if used with an FTS5 table created with the
 133 **   "detail=none" or "detail=column" option.
 134 **
 135 ** xRowid:
 136 **   Returns the rowid of the current row.
 137 **
 138 ** xTokenize:
 139 **   Tokenize text using the tokenizer belonging to the FTS5 table.
 140 **
 141 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
 142 **   This API function is used to query the FTS table for phrase iPhrase
 143 **   of the current query. Specifically, a query equivalent to:
 144 **
 145 **       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
 146 **
 147 **   with $p set to a phrase equivalent to the phrase iPhrase of the
 148 **   current query is executed. Any column filter that applies to
 149 **   phrase iPhrase of the current query is included in $p. For each
 150 **   row visited, the callback function passed as the fourth argument
 151 **   is invoked. The context and API objects passed to the callback
 152 **   function may be used to access the properties of each matched row.
 153 **   Invoking Api.xUserData() returns a copy of the pointer passed as
 154 **   the third argument to pUserData.
 155 **
 156 **   If parameter iPhrase is less than zero, or greater than or equal to
 157 **   the number of phrases in the query, as returned by xPhraseCount(),
 158 **   this function returns SQLITE_RANGE.
 159 **
 160 **   If the callback function returns any value other than SQLITE_OK, the
 161 **   query is abandoned and the xQueryPhrase function returns immediately.
 162 **   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
 163 **   Otherwise, the error code is propagated upwards.
 164 **
 165 **   If the query runs to completion without incident, SQLITE_OK is returned.
 166 **   Or, if some error occurs before the query completes or is aborted by
 167 **   the callback, an SQLite error code is returned.
 168 **
 169 **
 170 ** xSetAuxdata(pFts5, pAux, xDelete)
 171 **
 172 **   Save the pointer passed as the second argument as the extension function's
 173 **   "auxiliary data". The pointer may then be retrieved by the current or any
 174 **   future invocation of the same fts5 extension function made as part of
 175 **   the same MATCH query using the xGetAuxdata() API.
 176 **
 177 **   Each extension function is allocated a single auxiliary data slot for
 178 **   each FTS query (MATCH expression). If the extension function is invoked
 179 **   more than once for a single FTS query, then all invocations share a
 180 **   single auxiliary data context.
 181 **
 182 **   If there is already an auxiliary data pointer when this function is
 183 **   invoked, then it is replaced by the new pointer. If an xDelete callback
 184 **   was specified along with the original pointer, it is invoked at this
 185 **   point.
 186 **
 187 **   The xDelete callback, if one is specified, is also invoked on the
 188 **   auxiliary data pointer after the FTS5 query has finished.
 189 **
 190 **   If an error (e.g. an OOM condition) occurs within this function,
 191 **   the auxiliary data is set to NULL and an error code returned. If the
 192 **   xDelete parameter was not NULL, it is invoked on the auxiliary data
 193 **   pointer before returning.
 194 **
 195 **
 196 ** xGetAuxdata(pFts5, bClear)
 197 **
 198 **   Returns the current auxiliary data pointer for the fts5 extension
 199 **   function. See the xSetAuxdata() method for details.
 200 **
 201 **   If the bClear argument is non-zero, then the auxiliary data is cleared
 202 **   (set to NULL) before this function returns. In this case the xDelete,
 203 **   if any, is not invoked.
 204 **
 205 **
 206 ** xRowCount(pFts5, pnRow)
 207 **
 208 **   This function is used to retrieve the total number of rows in the table.
 209 **   In other words, the same value that would be returned by:
 210 **
 211 **        SELECT count(*) FROM ftstable;
 212 **
 213 ** xPhraseFirst()
 214 **   This function is used, along with type Fts5PhraseIter and the xPhraseNext
 215 **   method, to iterate through all instances of a single query phrase within
 216 **   the current row. This is the same information as is accessible via the
 217 **   xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
 218 **   to use, this API may be faster under some circumstances. To iterate
 219 **   through instances of phrase iPhrase, use the following code:
 220 **
 221 **       Fts5PhraseIter iter;
 222 **       int iCol, iOff;
 223 **       for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
 224 **           iCol>=0;
 225 **           pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
 226 **       ){
 227 **         // An instance of phrase iPhrase at offset iOff of column iCol
 228 **       }
 229 **
 230 **   The Fts5PhraseIter structure is defined above. Applications should not
 231 **   modify this structure directly - it should only be used as shown above
 232 **   with the xPhraseFirst() and xPhraseNext() API methods (and by
 233 **   xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
 234 **
 235 **   This API can be quite slow if used with an FTS5 table created with the
 236 **   "detail=none" or "detail=column" option. If the FTS5 table is created
 237 **   with either "detail=none" or "detail=column" and "content=" option
 238 **   (i.e. if it is a contentless table), then this API always iterates
 239 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 240 **
 241 ** xPhraseNext()
 242 **   See xPhraseFirst above.
 243 **
 244 ** xPhraseFirstColumn()
 245 **   This function and xPhraseNextColumn() are similar to the xPhraseFirst()
 246 **   and xPhraseNext() APIs described above. The difference is that instead
 247 **   of iterating through all instances of a phrase in the current row, these
 248 **   APIs are used to iterate through the set of columns in the current row
 249 **   that contain one or more instances of a specified phrase. For example:
 250 **
 251 **       Fts5PhraseIter iter;
 252 **       int iCol;
 253 **       for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
 254 **           iCol>=0;
 255 **           pApi->xPhraseNextColumn(pFts, &iter, &iCol)
 256 **       ){
 257 **         // Column iCol contains at least one instance of phrase iPhrase
 258 **       }
 259 **
 260 **   This API can be quite slow if used with an FTS5 table created with the
 261 **   "detail=none" option. If the FTS5 table is created with either
 262 **   "detail=none" "content=" option (i.e. if it is a contentless table),
 263 **   then this API always iterates through an empty set (all calls to
 264 **   xPhraseFirstColumn() set iCol to -1).
 265 **
 266 **   The information accessed using this API and its companion
 267 **   xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
 268 **   (or xInst/xInstCount). The chief advantage of this API is that it is
 269 **   significantly more efficient than those alternatives when used with
 270 **   "detail=column" tables.
 271 **
 272 ** xPhraseNextColumn()
 273 **   See xPhraseFirstColumn above.
 274 **
 275 ** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
 276 **   This is used to access token iToken of phrase iPhrase of the current
 277 **   query. Before returning, output parameter *ppToken is set to point
 278 **   to a buffer containing the requested token, and *pnToken to the
 279 **   size of this buffer in bytes.
 280 **
 281 **   If iPhrase or iToken are less than zero, or if iPhrase is greater than
 282 **   or equal to the number of phrases in the query as reported by
 283 **   xPhraseCount(), or if iToken is equal to or greater than the number of
 284 **   tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
 285      are both zeroed.
 286 **
 287 **   The output text is not a copy of the query text that specified the
 288 **   token. It is the output of the tokenizer module. For tokendata=1
 289 **   tables, this includes any embedded 0x00 and trailing data.
 290 **
 291 ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
 292 **   This is used to access token iToken of phrase hit iIdx within the
 293 **   current row. If iIdx is less than zero or greater than or equal to the
 294 **   value returned by xInstCount(), SQLITE_RANGE is returned.  Otherwise,
 295 **   output variable (*ppToken) is set to point to a buffer containing the
 296 **   matching document token, and (*pnToken) to the size of that buffer in
 297 **   bytes. This API is not available if the specified token matches a
 298 **   prefix query term. In that case both output variables are always set
 299 **   to 0.
 300 **
 301 **   The output text is not a copy of the document text that was tokenized.
 302 **   It is the output of the tokenizer module. For tokendata=1 tables, this
 303 **   includes any embedded 0x00 and trailing data.
 304 **
 305 **   This API can be quite slow if used with an FTS5 table created with the
 306 **   "detail=none" or "detail=column" option.
 307 */
 308 struct Fts5ExtensionApi {
 309   int iVersion;                   /* Currently always set to 3 */
 310
 311   void *(*xUserData)(Fts5Context*);
 312
 313   int (*xColumnCount)(Fts5Context*);
 314   int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
 315   int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
 316
 317   int (*xTokenize)(Fts5Context*,
 318     const char *pText, int nText, /* Text to tokenize */
 319     void *pCtx,                   /* Context passed to xToken() */
 320     int (*xToken)(void*, int, const char*, int, int, int)       /* Callback */
 321   );
 322
 323   int (*xPhraseCount)(Fts5Context*);
 324   int (*xPhraseSize)(Fts5Context*, int iPhrase);
 325
 326   int (*xInstCount)(Fts5Context*, int *pnInst);
 327   int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
 328
 329   sqlite3_int64 (*xRowid)(Fts5Context*);
 330   int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
 331   int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
 332
 333   int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
 334     int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
 335   );
 336   int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
 337   void *(*xGetAuxdata)(Fts5Context*, int bClear);
 338
 339   int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
 340   void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
 341
 342   int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
 343   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
 344
 345   /* Below this point are iVersion>=3 only */
 346   int (*xQueryToken)(Fts5Context*,
 347       int iPhrase, int iToken,
 348       const char **ppToken, int *pnToken
 349   );
 350   int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 351 };
 352
 353 /*
 354 ** CUSTOM AUXILIARY FUNCTIONS
 355 *************************************************************************/
 356
 357 /*************************************************************************
 358 ** CUSTOM TOKENIZERS
 359 **
 360 ** Applications may also register custom tokenizer types. A tokenizer
 361 ** is registered by providing fts5 with a populated instance of the
 362 ** following structure. All structure methods must be defined, setting
 363 ** any member of the fts5_tokenizer struct to NULL leads to undefined
 364 ** behaviour. The structure methods are expected to function as follows:
 365 **
 366 ** xCreate:
 367 **   This function is used to allocate and initialize a tokenizer instance.
 368 **   A tokenizer instance is required to actually tokenize text.
 369 **
 370 **   The first argument passed to this function is a copy of the (void*)
 371 **   pointer provided by the application when the fts5_tokenizer object
 372 **   was registered with FTS5 (the third argument to xCreateTokenizer()).
 373 **   The second and third arguments are an array of nul-terminated strings
 374 **   containing the tokenizer arguments, if any, specified following the
 375 **   tokenizer name as part of the CREATE VIRTUAL TABLE statement used
 376 **   to create the FTS5 table.
 377 **
 378 **   The final argument is an output variable. If successful, (*ppOut)
 379 **   should be set to point to the new tokenizer handle and SQLITE_OK
 380 **   returned. If an error occurs, some value other than SQLITE_OK should
 381 **   be returned. In this case, fts5 assumes that the final value of *ppOut
 382 **   is undefined.
 383 **
 384 ** xDelete:
 385 **   This function is invoked to delete a tokenizer handle previously
 386 **   allocated using xCreate(). Fts5 guarantees that this function will
 387 **   be invoked exactly once for each successful call to xCreate().
 388 **
 389 ** xTokenize:
 390 **   This function is expected to tokenize the nText byte string indicated
 391 **   by argument pText. pText may or may not be nul-terminated. The first
 392 **   argument passed to this function is a pointer to an Fts5Tokenizer object
 393 **   returned by an earlier call to xCreate().
 394 **
 395 **   The second argument indicates the reason that FTS5 is requesting
 396 **   tokenization of the supplied text. This is always one of the following
 397 **   four values:
 398 **
 399 **   <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
 400 **            or removed from the FTS table. The tokenizer is being invoked to
 401 **            determine the set of tokens to add to (or delete from) the
 402 **            FTS index.
 403 **
 404 **       <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
 405 **            against the FTS index. The tokenizer is being called to tokenize
 406 **            a bareword or quoted string specified as part of the query.
 407 **
 408 **       <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
 409 **            FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
 410 **            followed by a "*" character, indicating that the last token
 411 **            returned by the tokenizer will be treated as a token prefix.
 412 **
 413 **       <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
 414 **            satisfy an fts5_api.xTokenize() request made by an auxiliary
 415 **            function. Or an fts5_api.xColumnSize() request made by the same
 416 **            on a columnsize=0 database.
 417 **   </ul>
 418 **
 419 **   For each token in the input string, the supplied callback xToken() must
 420 **   be invoked. The first argument to it should be a copy of the pointer
 421 **   passed as the second argument to xTokenize(). The third and fourth
 422 **   arguments are a pointer to a buffer containing the token text, and the
 423 **   size of the token in bytes. The 4th and 5th arguments are the byte offsets
 424 **   of the first byte of and first byte immediately following the text from
 425 **   which the token is derived within the input.
 426 **
 427 **   The second argument passed to the xToken() callback ("tflags") should
 428 **   normally be set to 0. The exception is if the tokenizer supports
 429 **   synonyms. In this case see the discussion below for details.
 430 **
 431 **   FTS5 assumes the xToken() callback is invoked for each token in the
 432 **   order that they occur within the input text.
 433 **
 434 **   If an xToken() callback returns any value other than SQLITE_OK, then
 435 **   the tokenization should be abandoned and the xTokenize() method should
 436 **   immediately return a copy of the xToken() return value. Or, if the
 437 **   input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
 438 **   if an error occurs with the xTokenize() implementation itself, it
 439 **   may abandon the tokenization and return any error code other than
 440 **   SQLITE_OK or SQLITE_DONE.
 441 **
 442 ** SYNONYM SUPPORT
 443 **
 444 **   Custom tokenizers may also support synonyms. Consider a case in which a
 445 **   user wishes to query for a phrase such as "first place". Using the
 446 **   built-in tokenizers, the FTS5 query 'first + place' will match instances
 447 **   of "first place" within the document set, but not alternative forms
 448 **   such as "1st place". In some applications, it would be better to match
 449 **   all instances of "first place" or "1st place" regardless of which form
 450 **   the user specified in the MATCH query text.
 451 **
 452 **   There are several ways to approach this in FTS5:
 453 **
 454 **   <ol><li> By mapping all synonyms to a single token. In this case, using
 455 **            the above example, this means that the tokenizer returns the
 456 **            same token for inputs "first" and "1st". Say that token is in
 457 **            fact "first", so that when the user inserts the document "I won
 458 **            1st place" entries are added to the index for tokens "i", "won",
 459 **            "first" and "place". If the user then queries for '1st + place',
 460 **            the tokenizer substitutes "first" for "1st" and the query works
 461 **            as expected.
 462 **
 463 **       <li> By querying the index for all synonyms of each query term
 464 **            separately. In this case, when tokenizing query text, the
 465 **            tokenizer may provide multiple synonyms for a single term
 466 **            within the document. FTS5 then queries the index for each
 467 **            synonym individually. For example, faced with the query:
 468 **
 469 **   <codeblock>
 470 **     ... MATCH 'first place'</codeblock>
 471 **
 472 **            the tokenizer offers both "1st" and "first" as synonyms for the
 473 **            first token in the MATCH query and FTS5 effectively runs a query
 474 **            similar to:
 475 **
 476 **   <codeblock>
 477 **     ... MATCH '(first OR 1st) place'</codeblock>
 478 **
 479 **            except that, for the purposes of auxiliary functions, the query
 480 **            still appears to contain just two phrases - "(first OR 1st)"
 481 **            being treated as a single phrase.
 482 **
 483 **       <li> By adding multiple synonyms for a single term to the FTS index.
 484 **            Using this method, when tokenizing document text, the tokenizer
 485 **            provides multiple synonyms for each token. So that when a
 486 **            document such as "I won first place" is tokenized, entries are
 487 **            added to the FTS index for "i", "won", "first", "1st" and
 488 **            "place".
 489 **
 490 **            This way, even if the tokenizer does not provide synonyms
 491 **            when tokenizing query text (it should not - to do so would be
 492 **            inefficient), it doesn't matter if the user queries for
 493 **            'first + place' or '1st + place', as there are entries in the
 494 **            FTS index corresponding to both forms of the first token.
 495 **   </ol>
 496 **
 497 **   Whether it is parsing document or query text, any call to xToken that
 498 **   specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
 499 **   is considered to supply a synonym for the previous token. For example,
 500 **   when parsing the document "I won first place", a tokenizer that supports
 501 **   synonyms would call xToken() 5 times, as follows:
 502 **
 503 **   <codeblock>
 504 **       xToken(pCtx, 0, "i",                      1,  0,  1);
 505 **       xToken(pCtx, 0, "won",                    3,  2,  5);
 506 **       xToken(pCtx, 0, "first",                  5,  6, 11);
 507 **       xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3,  6, 11);
 508 **       xToken(pCtx, 0, "place",                  5, 12, 17);
 509 **</codeblock>
 510 **
 511 **   It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
 512 **   xToken() is called. Multiple synonyms may be specified for a single token
 513 **   by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
 514 **   There is no limit to the number of synonyms that may be provided for a
 515 **   single token.
 516 **
 517 **   In many cases, method (1) above is the best approach. It does not add
 518 **   extra data to the FTS index or require FTS5 to query for multiple terms,
 519 **   so it is efficient in terms of disk space and query speed. However, it
 520 **   does not support prefix queries very well. If, as suggested above, the
 521 **   token "first" is substituted for "1st" by the tokenizer, then the query:
 522 **
 523 **   <codeblock>
 524 **     ... MATCH '1s*'</codeblock>
 525 **
 526 **   will not match documents that contain the token "1st" (as the tokenizer
 527 **   will probably not map "1s" to any prefix of "first").
 528 **
 529 **   For full prefix support, method (3) may be preferred. In this case,
 530 **   because the index contains entries for both "first" and "1st", prefix
 531 **   queries such as 'fi*' or '1s*' will match correctly. However, because
 532 **   extra entries are added to the FTS index, this method uses more space
 533 **   within the database.
 534 **
 535 **   Method (2) offers a midpoint between (1) and (3). Using this method,
 536 **   a query such as '1s*' will match documents that contain the literal
 537 **   token "1st", but not "first" (assuming the tokenizer is not able to
 538 **   provide synonyms for prefixes). However, a non-prefix query like '1st'
 539 **   will match against "1st" and "first". This method does not require
 540 **   extra disk space, as no extra entries are added to the FTS index.
 541 **   On the other hand, it may require more CPU cycles to run MATCH queries,
 542 **   as separate queries of the FTS index are required for each synonym.
 543 **
 544 **   When using methods (2) or (3), it is important that the tokenizer only
 545 **   provide synonyms when tokenizing document text (method (3)) or query
 546 **   text (method (2)), not both. Doing so will not cause any errors, but is
 547 **   inefficient.
 548 */
 549 typedef struct Fts5Tokenizer Fts5Tokenizer;
 550 typedef struct fts5_tokenizer fts5_tokenizer;
 551 struct fts5_tokenizer {
 552   int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
 553   void (*xDelete)(Fts5Tokenizer*);
 554   int (*xTokenize)(Fts5Tokenizer*,
 555       void *pCtx,
 556       int flags,            /* Mask of FTS5_TOKENIZE_* flags */
 557       const char *pText, int nText,
 558       int (*xToken)(
 559         void *pCtx,         /* Copy of 2nd argument to xTokenize() */
 560         int tflags,         /* Mask of FTS5_TOKEN_* flags */
 561         const char *pToken, /* Pointer to buffer containing token */
 562         int nToken,         /* Size of token in bytes */
 563         int iStart,         /* Byte offset of token within input text */
 564         int iEnd            /* Byte offset of end of token within input text */
 565       )
 566   );
 567 };
 568
 569 /* Flags that may be passed as the third argument to xTokenize() */
 570 #define FTS5_TOKENIZE_QUERY     0x0001
 571 #define FTS5_TOKENIZE_PREFIX    0x0002
 572 #define FTS5_TOKENIZE_DOCUMENT  0x0004
 573 #define FTS5_TOKENIZE_AUX       0x0008
 574
 575 /* Flags that may be passed by the tokenizer implementation back to FTS5
 576 ** as the third argument to the supplied xToken callback. */
 577 #define FTS5_TOKEN_COLOCATED    0x0001      /* Same position as prev. token */
 578
 579 /*
 580 ** END OF CUSTOM TOKENIZERS
 581 *************************************************************************/
 582
 583 /*************************************************************************
 584 ** FTS5 EXTENSION REGISTRATION API
 585 */
 586 typedef struct fts5_api fts5_api;
 587 struct fts5_api {
 588   int iVersion;                   /* Currently always set to 2 */
 589
 590   /* Create a new tokenizer */
 591   int (*xCreateTokenizer)(
 592     fts5_api *pApi,
 593     const char *zName,
 594     void *pUserData,
 595     fts5_tokenizer *pTokenizer,
 596     void (*xDestroy)(void*)
 597   );
 598
 599   /* Find an existing tokenizer */
 600   int (*xFindTokenizer)(
 601     fts5_api *pApi,
 602     const char *zName,
 603     void **ppUserData,
 604     fts5_tokenizer *pTokenizer
 605   );
 606
 607   /* Create a new auxiliary function */
 608   int (*xCreateFunction)(
 609     fts5_api *pApi,
 610     const char *zName,
 611     void *pUserData,
 612     fts5_extension_function xFunction,
 613     void (*xDestroy)(void*)
 614   );
 615 };
 616
 617 /*
 618 ** END OF REGISTRATION API
 619 *************************************************************************/
 620
 621 #ifdef __cplusplus
 622 }  /* end of the 'extern "C"' block */
 623 #endif
 624
 625 #endif /* _FTS5_H */