4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 ******************************************************************************
13 ** Interfaces to extend FTS5. Using the interfaces defined in this file,
14 ** FTS5 may be extended with:
16 ** * custom tokenizers, and
17 ** * custom auxiliary functions.
30 /*************************************************************************
31 ** CUSTOM AUXILIARY FUNCTIONS
33 ** Virtual table implementations may overload SQL functions by implementing
34 ** the sqlite3_module.xFindFunction() method.
37 typedef struct Fts5ExtensionApi Fts5ExtensionApi
;
38 typedef struct Fts5Context Fts5Context
;
39 typedef struct Fts5PhraseIter Fts5PhraseIter
;
41 typedef void (*fts5_extension_function
)(
42 const Fts5ExtensionApi
*pApi
, /* API offered by current FTS version */
43 Fts5Context
*pFts
, /* First arg to pass to pApi functions */
44 sqlite3_context
*pCtx
, /* Context for returning result/error */
45 int nVal
, /* Number of values in apVal[] array */
46 sqlite3_value
**apVal
/* Array of trailing arguments */
49 struct Fts5PhraseIter
{
50 const unsigned char *a
;
51 const unsigned char *b
;
55 ** EXTENSION API FUNCTIONS
58 ** Return a copy of the context pointer the extension function was
61 ** xColumnTotalSize(pFts, iCol, pnToken):
62 ** If parameter iCol is less than zero, set output variable *pnToken
63 ** to the total number of tokens in the FTS5 table. Or, if iCol is
64 ** non-negative but less than the number of columns in the table, return
65 ** the total number of tokens in column iCol, considering all rows in
68 ** If parameter iCol is greater than or equal to the number of columns
69 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
70 ** an OOM condition or IO error), an appropriate SQLite error code is
73 ** xColumnCount(pFts):
74 ** Return the number of columns in the table.
76 ** xColumnSize(pFts, iCol, pnToken):
77 ** If parameter iCol is less than zero, set output variable *pnToken
78 ** to the total number of tokens in the current row. Or, if iCol is
79 ** non-negative but less than the number of columns in the table, set
80 ** *pnToken to the number of tokens in column iCol of the current row.
82 ** If parameter iCol is greater than or equal to the number of columns
83 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
84 ** an OOM condition or IO error), an appropriate SQLite error code is
87 ** This function may be quite inefficient if used with an FTS5 table
88 ** created with the "columnsize=0" option.
91 ** If parameter iCol is less than zero, or greater than or equal to the
92 ** number of columns in the table, SQLITE_RANGE is returned.
94 ** Otherwise, this function attempts to retrieve the text of column iCol of
95 ** the current document. If successful, (*pz) is set to point to a buffer
96 ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
97 ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
98 ** if an error occurs, an SQLite error code is returned and the final values
99 ** of (*pz) and (*pn) are undefined.
102 ** Returns the number of phrases in the current query expression.
105 ** If parameter iCol is less than zero, or greater than or equal to the
106 ** number of phrases in the current query, as returned by xPhraseCount,
107 ** 0 is returned. Otherwise, this function returns the number of tokens in
108 ** phrase iPhrase of the query. Phrases are numbered starting from zero.
111 ** Set *pnInst to the total number of occurrences of all phrases within
112 ** the query within the current row. Return SQLITE_OK if successful, or
113 ** an error code (i.e. SQLITE_NOMEM) if an error occurs.
115 ** This API can be quite slow if used with an FTS5 table created with the
116 ** "detail=none" or "detail=column" option. If the FTS5 table is created
117 ** with either "detail=none" or "detail=column" and "content=" option
118 ** (i.e. if it is a contentless table), then this API always returns 0.
121 ** Query for the details of phrase match iIdx within the current row.
122 ** Phrase matches are numbered starting from zero, so the iIdx argument
123 ** should be greater than or equal to zero and smaller than the value
124 ** output by xInstCount(). If iIdx is less than zero or greater than
125 ** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
127 ** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
128 ** to the column in which it occurs and *piOff the token offset of the
129 ** first token of the phrase. SQLITE_OK is returned if successful, or an
130 ** error code (i.e. SQLITE_NOMEM) if an error occurs.
132 ** This API can be quite slow if used with an FTS5 table created with the
133 ** "detail=none" or "detail=column" option.
136 ** Returns the rowid of the current row.
139 ** Tokenize text using the tokenizer belonging to the FTS5 table.
141 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
142 ** This API function is used to query the FTS table for phrase iPhrase
143 ** of the current query. Specifically, a query equivalent to:
145 ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
147 ** with $p set to a phrase equivalent to the phrase iPhrase of the
148 ** current query is executed. Any column filter that applies to
149 ** phrase iPhrase of the current query is included in $p. For each
150 ** row visited, the callback function passed as the fourth argument
151 ** is invoked. The context and API objects passed to the callback
152 ** function may be used to access the properties of each matched row.
153 ** Invoking Api.xUserData() returns a copy of the pointer passed as
154 ** the third argument to pUserData.
156 ** If parameter iPhrase is less than zero, or greater than or equal to
157 ** the number of phrases in the query, as returned by xPhraseCount(),
158 ** this function returns SQLITE_RANGE.
160 ** If the callback function returns any value other than SQLITE_OK, the
161 ** query is abandoned and the xQueryPhrase function returns immediately.
162 ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
163 ** Otherwise, the error code is propagated upwards.
165 ** If the query runs to completion without incident, SQLITE_OK is returned.
166 ** Or, if some error occurs before the query completes or is aborted by
167 ** the callback, an SQLite error code is returned.
170 ** xSetAuxdata(pFts5, pAux, xDelete)
172 ** Save the pointer passed as the second argument as the extension function's
173 ** "auxiliary data". The pointer may then be retrieved by the current or any
174 ** future invocation of the same fts5 extension function made as part of
175 ** the same MATCH query using the xGetAuxdata() API.
177 ** Each extension function is allocated a single auxiliary data slot for
178 ** each FTS query (MATCH expression). If the extension function is invoked
179 ** more than once for a single FTS query, then all invocations share a
180 ** single auxiliary data context.
182 ** If there is already an auxiliary data pointer when this function is
183 ** invoked, then it is replaced by the new pointer. If an xDelete callback
184 ** was specified along with the original pointer, it is invoked at this
187 ** The xDelete callback, if one is specified, is also invoked on the
188 ** auxiliary data pointer after the FTS5 query has finished.
190 ** If an error (e.g. an OOM condition) occurs within this function,
191 ** the auxiliary data is set to NULL and an error code returned. If the
192 ** xDelete parameter was not NULL, it is invoked on the auxiliary data
193 ** pointer before returning.
196 ** xGetAuxdata(pFts5, bClear)
198 ** Returns the current auxiliary data pointer for the fts5 extension
199 ** function. See the xSetAuxdata() method for details.
201 ** If the bClear argument is non-zero, then the auxiliary data is cleared
202 ** (set to NULL) before this function returns. In this case the xDelete,
203 ** if any, is not invoked.
206 ** xRowCount(pFts5, pnRow)
208 ** This function is used to retrieve the total number of rows in the table.
209 ** In other words, the same value that would be returned by:
211 ** SELECT count(*) FROM ftstable;
214 ** This function is used, along with type Fts5PhraseIter and the xPhraseNext
215 ** method, to iterate through all instances of a single query phrase within
216 ** the current row. This is the same information as is accessible via the
217 ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
218 ** to use, this API may be faster under some circumstances. To iterate
219 ** through instances of phrase iPhrase, use the following code:
221 ** Fts5PhraseIter iter;
223 ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
225 ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
227 ** // An instance of phrase iPhrase at offset iOff of column iCol
230 ** The Fts5PhraseIter structure is defined above. Applications should not
231 ** modify this structure directly - it should only be used as shown above
232 ** with the xPhraseFirst() and xPhraseNext() API methods (and by
233 ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
235 ** This API can be quite slow if used with an FTS5 table created with the
236 ** "detail=none" or "detail=column" option. If the FTS5 table is created
237 ** with either "detail=none" or "detail=column" and "content=" option
238 ** (i.e. if it is a contentless table), then this API always iterates
239 ** through an empty set (all calls to xPhraseFirst() set iCol to -1).
242 ** See xPhraseFirst above.
244 ** xPhraseFirstColumn()
245 ** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
246 ** and xPhraseNext() APIs described above. The difference is that instead
247 ** of iterating through all instances of a phrase in the current row, these
248 ** APIs are used to iterate through the set of columns in the current row
249 ** that contain one or more instances of a specified phrase. For example:
251 ** Fts5PhraseIter iter;
253 ** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
255 ** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
257 ** // Column iCol contains at least one instance of phrase iPhrase
260 ** This API can be quite slow if used with an FTS5 table created with the
261 ** "detail=none" option. If the FTS5 table is created with either
262 ** "detail=none" "content=" option (i.e. if it is a contentless table),
263 ** then this API always iterates through an empty set (all calls to
264 ** xPhraseFirstColumn() set iCol to -1).
266 ** The information accessed using this API and its companion
267 ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
268 ** (or xInst/xInstCount). The chief advantage of this API is that it is
269 ** significantly more efficient than those alternatives when used with
270 ** "detail=column" tables.
272 ** xPhraseNextColumn()
273 ** See xPhraseFirstColumn above.
275 ** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
276 ** This is used to access token iToken of phrase iPhrase of the current
277 ** query. Before returning, output parameter *ppToken is set to point
278 ** to a buffer containing the requested token, and *pnToken to the
279 ** size of this buffer in bytes.
281 ** If iPhrase or iToken are less than zero, or if iPhrase is greater than
282 ** or equal to the number of phrases in the query as reported by
283 ** xPhraseCount(), or if iToken is equal to or greater than the number of
284 ** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
287 ** The output text is not a copy of the query text that specified the
288 ** token. It is the output of the tokenizer module. For tokendata=1
289 ** tables, this includes any embedded 0x00 and trailing data.
291 ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
292 ** This is used to access token iToken of phrase hit iIdx within the
293 ** current row. If iIdx is less than zero or greater than or equal to the
294 ** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
295 ** output variable (*ppToken) is set to point to a buffer containing the
296 ** matching document token, and (*pnToken) to the size of that buffer in
297 ** bytes. This API is not available if the specified token matches a
298 ** prefix query term. In that case both output variables are always set
301 ** The output text is not a copy of the document text that was tokenized.
302 ** It is the output of the tokenizer module. For tokendata=1 tables, this
303 ** includes any embedded 0x00 and trailing data.
305 ** This API can be quite slow if used with an FTS5 table created with the
306 ** "detail=none" or "detail=column" option.
308 struct Fts5ExtensionApi
{
309 int iVersion
; /* Currently always set to 3 */
311 void *(*xUserData
)(Fts5Context
*);
313 int (*xColumnCount
)(Fts5Context
*);
314 int (*xRowCount
)(Fts5Context
*, sqlite3_int64
*pnRow
);
315 int (*xColumnTotalSize
)(Fts5Context
*, int iCol
, sqlite3_int64
*pnToken
);
317 int (*xTokenize
)(Fts5Context
*,
318 const char *pText
, int nText
, /* Text to tokenize */
319 void *pCtx
, /* Context passed to xToken() */
320 int (*xToken
)(void*, int, const char*, int, int, int) /* Callback */
323 int (*xPhraseCount
)(Fts5Context
*);
324 int (*xPhraseSize
)(Fts5Context
*, int iPhrase
);
326 int (*xInstCount
)(Fts5Context
*, int *pnInst
);
327 int (*xInst
)(Fts5Context
*, int iIdx
, int *piPhrase
, int *piCol
, int *piOff
);
329 sqlite3_int64 (*xRowid
)(Fts5Context
*);
330 int (*xColumnText
)(Fts5Context
*, int iCol
, const char **pz
, int *pn
);
331 int (*xColumnSize
)(Fts5Context
*, int iCol
, int *pnToken
);
333 int (*xQueryPhrase
)(Fts5Context
*, int iPhrase
, void *pUserData
,
334 int(*)(const Fts5ExtensionApi
*,Fts5Context
*,void*)
336 int (*xSetAuxdata
)(Fts5Context
*, void *pAux
, void(*xDelete
)(void*));
337 void *(*xGetAuxdata
)(Fts5Context
*, int bClear
);
339 int (*xPhraseFirst
)(Fts5Context
*, int iPhrase
, Fts5PhraseIter
*, int*, int*);
340 void (*xPhraseNext
)(Fts5Context
*, Fts5PhraseIter
*, int *piCol
, int *piOff
);
342 int (*xPhraseFirstColumn
)(Fts5Context
*, int iPhrase
, Fts5PhraseIter
*, int*);
343 void (*xPhraseNextColumn
)(Fts5Context
*, Fts5PhraseIter
*, int *piCol
);
345 /* Below this point are iVersion>=3 only */
346 int (*xQueryToken
)(Fts5Context
*,
347 int iPhrase
, int iToken
,
348 const char **ppToken
, int *pnToken
350 int (*xInstToken
)(Fts5Context
*, int iIdx
, int iToken
, const char**, int*);
354 ** CUSTOM AUXILIARY FUNCTIONS
355 *************************************************************************/
357 /*************************************************************************
360 ** Applications may also register custom tokenizer types. A tokenizer
361 ** is registered by providing fts5 with a populated instance of the
362 ** following structure. All structure methods must be defined, setting
363 ** any member of the fts5_tokenizer struct to NULL leads to undefined
364 ** behaviour. The structure methods are expected to function as follows:
367 ** This function is used to allocate and initialize a tokenizer instance.
368 ** A tokenizer instance is required to actually tokenize text.
370 ** The first argument passed to this function is a copy of the (void*)
371 ** pointer provided by the application when the fts5_tokenizer object
372 ** was registered with FTS5 (the third argument to xCreateTokenizer()).
373 ** The second and third arguments are an array of nul-terminated strings
374 ** containing the tokenizer arguments, if any, specified following the
375 ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
376 ** to create the FTS5 table.
378 ** The final argument is an output variable. If successful, (*ppOut)
379 ** should be set to point to the new tokenizer handle and SQLITE_OK
380 ** returned. If an error occurs, some value other than SQLITE_OK should
381 ** be returned. In this case, fts5 assumes that the final value of *ppOut
385 ** This function is invoked to delete a tokenizer handle previously
386 ** allocated using xCreate(). Fts5 guarantees that this function will
387 ** be invoked exactly once for each successful call to xCreate().
390 ** This function is expected to tokenize the nText byte string indicated
391 ** by argument pText. pText may or may not be nul-terminated. The first
392 ** argument passed to this function is a pointer to an Fts5Tokenizer object
393 ** returned by an earlier call to xCreate().
395 ** The second argument indicates the reason that FTS5 is requesting
396 ** tokenization of the supplied text. This is always one of the following
399 ** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
400 ** or removed from the FTS table. The tokenizer is being invoked to
401 ** determine the set of tokens to add to (or delete from) the
404 ** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
405 ** against the FTS index. The tokenizer is being called to tokenize
406 ** a bareword or quoted string specified as part of the query.
408 ** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
409 ** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
410 ** followed by a "*" character, indicating that the last token
411 ** returned by the tokenizer will be treated as a token prefix.
413 ** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
414 ** satisfy an fts5_api.xTokenize() request made by an auxiliary
415 ** function. Or an fts5_api.xColumnSize() request made by the same
416 ** on a columnsize=0 database.
419 ** For each token in the input string, the supplied callback xToken() must
420 ** be invoked. The first argument to it should be a copy of the pointer
421 ** passed as the second argument to xTokenize(). The third and fourth
422 ** arguments are a pointer to a buffer containing the token text, and the
423 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets
424 ** of the first byte of and first byte immediately following the text from
425 ** which the token is derived within the input.
427 ** The second argument passed to the xToken() callback ("tflags") should
428 ** normally be set to 0. The exception is if the tokenizer supports
429 ** synonyms. In this case see the discussion below for details.
431 ** FTS5 assumes the xToken() callback is invoked for each token in the
432 ** order that they occur within the input text.
434 ** If an xToken() callback returns any value other than SQLITE_OK, then
435 ** the tokenization should be abandoned and the xTokenize() method should
436 ** immediately return a copy of the xToken() return value. Or, if the
437 ** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
438 ** if an error occurs with the xTokenize() implementation itself, it
439 ** may abandon the tokenization and return any error code other than
440 ** SQLITE_OK or SQLITE_DONE.
444 ** Custom tokenizers may also support synonyms. Consider a case in which a
445 ** user wishes to query for a phrase such as "first place". Using the
446 ** built-in tokenizers, the FTS5 query 'first + place' will match instances
447 ** of "first place" within the document set, but not alternative forms
448 ** such as "1st place". In some applications, it would be better to match
449 ** all instances of "first place" or "1st place" regardless of which form
450 ** the user specified in the MATCH query text.
452 ** There are several ways to approach this in FTS5:
454 ** <ol><li> By mapping all synonyms to a single token. In this case, using
455 ** the above example, this means that the tokenizer returns the
456 ** same token for inputs "first" and "1st". Say that token is in
457 ** fact "first", so that when the user inserts the document "I won
458 ** 1st place" entries are added to the index for tokens "i", "won",
459 ** "first" and "place". If the user then queries for '1st + place',
460 ** the tokenizer substitutes "first" for "1st" and the query works
463 ** <li> By querying the index for all synonyms of each query term
464 ** separately. In this case, when tokenizing query text, the
465 ** tokenizer may provide multiple synonyms for a single term
466 ** within the document. FTS5 then queries the index for each
467 ** synonym individually. For example, faced with the query:
470 ** ... MATCH 'first place'</codeblock>
472 ** the tokenizer offers both "1st" and "first" as synonyms for the
473 ** first token in the MATCH query and FTS5 effectively runs a query
477 ** ... MATCH '(first OR 1st) place'</codeblock>
479 ** except that, for the purposes of auxiliary functions, the query
480 ** still appears to contain just two phrases - "(first OR 1st)"
481 ** being treated as a single phrase.
483 ** <li> By adding multiple synonyms for a single term to the FTS index.
484 ** Using this method, when tokenizing document text, the tokenizer
485 ** provides multiple synonyms for each token. So that when a
486 ** document such as "I won first place" is tokenized, entries are
487 ** added to the FTS index for "i", "won", "first", "1st" and
490 ** This way, even if the tokenizer does not provide synonyms
491 ** when tokenizing query text (it should not - to do so would be
492 ** inefficient), it doesn't matter if the user queries for
493 ** 'first + place' or '1st + place', as there are entries in the
494 ** FTS index corresponding to both forms of the first token.
497 ** Whether it is parsing document or query text, any call to xToken that
498 ** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
499 ** is considered to supply a synonym for the previous token. For example,
500 ** when parsing the document "I won first place", a tokenizer that supports
501 ** synonyms would call xToken() 5 times, as follows:
504 ** xToken(pCtx, 0, "i", 1, 0, 1);
505 ** xToken(pCtx, 0, "won", 3, 2, 5);
506 ** xToken(pCtx, 0, "first", 5, 6, 11);
507 ** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
508 ** xToken(pCtx, 0, "place", 5, 12, 17);
511 ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
512 ** xToken() is called. Multiple synonyms may be specified for a single token
513 ** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
514 ** There is no limit to the number of synonyms that may be provided for a
517 ** In many cases, method (1) above is the best approach. It does not add
518 ** extra data to the FTS index or require FTS5 to query for multiple terms,
519 ** so it is efficient in terms of disk space and query speed. However, it
520 ** does not support prefix queries very well. If, as suggested above, the
521 ** token "first" is substituted for "1st" by the tokenizer, then the query:
524 ** ... MATCH '1s*'</codeblock>
526 ** will not match documents that contain the token "1st" (as the tokenizer
527 ** will probably not map "1s" to any prefix of "first").
529 ** For full prefix support, method (3) may be preferred. In this case,
530 ** because the index contains entries for both "first" and "1st", prefix
531 ** queries such as 'fi*' or '1s*' will match correctly. However, because
532 ** extra entries are added to the FTS index, this method uses more space
533 ** within the database.
535 ** Method (2) offers a midpoint between (1) and (3). Using this method,
536 ** a query such as '1s*' will match documents that contain the literal
537 ** token "1st", but not "first" (assuming the tokenizer is not able to
538 ** provide synonyms for prefixes). However, a non-prefix query like '1st'
539 ** will match against "1st" and "first". This method does not require
540 ** extra disk space, as no extra entries are added to the FTS index.
541 ** On the other hand, it may require more CPU cycles to run MATCH queries,
542 ** as separate queries of the FTS index are required for each synonym.
544 ** When using methods (2) or (3), it is important that the tokenizer only
545 ** provide synonyms when tokenizing document text (method (3)) or query
546 ** text (method (2)), not both. Doing so will not cause any errors, but is
549 typedef struct Fts5Tokenizer Fts5Tokenizer
;
550 typedef struct fts5_tokenizer fts5_tokenizer
;
551 struct fts5_tokenizer
{
552 int (*xCreate
)(void*, const char **azArg
, int nArg
, Fts5Tokenizer
**ppOut
);
553 void (*xDelete
)(Fts5Tokenizer
*);
554 int (*xTokenize
)(Fts5Tokenizer
*,
556 int flags
, /* Mask of FTS5_TOKENIZE_* flags */
557 const char *pText
, int nText
,
559 void *pCtx
, /* Copy of 2nd argument to xTokenize() */
560 int tflags
, /* Mask of FTS5_TOKEN_* flags */
561 const char *pToken
, /* Pointer to buffer containing token */
562 int nToken
, /* Size of token in bytes */
563 int iStart
, /* Byte offset of token within input text */
564 int iEnd
/* Byte offset of end of token within input text */
569 /* Flags that may be passed as the third argument to xTokenize() */
570 #define FTS5_TOKENIZE_QUERY 0x0001
571 #define FTS5_TOKENIZE_PREFIX 0x0002
572 #define FTS5_TOKENIZE_DOCUMENT 0x0004
573 #define FTS5_TOKENIZE_AUX 0x0008
575 /* Flags that may be passed by the tokenizer implementation back to FTS5
576 ** as the third argument to the supplied xToken callback. */
577 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
580 ** END OF CUSTOM TOKENIZERS
581 *************************************************************************/
583 /*************************************************************************
584 ** FTS5 EXTENSION REGISTRATION API
586 typedef struct fts5_api fts5_api
;
588 int iVersion
; /* Currently always set to 2 */
590 /* Create a new tokenizer */
591 int (*xCreateTokenizer
)(
595 fts5_tokenizer
*pTokenizer
,
596 void (*xDestroy
)(void*)
599 /* Find an existing tokenizer */
600 int (*xFindTokenizer
)(
604 fts5_tokenizer
*pTokenizer
607 /* Create a new auxiliary function */
608 int (*xCreateFunction
)(
612 fts5_extension_function xFunction
,
613 void (*xDestroy
)(void*)
618 ** END OF REGISTRATION API
619 *************************************************************************/
622 } /* end of the 'extern "C"' block */