4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 ******************************************************************************
13 ** This is part of an SQLite module implementing full-text search.
14 ** This particular file implements the generic tokenizer interface.
18 ** The code in this file is only compiled if:
20 ** * The FTS3 module is being built as an extension
21 ** (in which case SQLITE_CORE is not defined), or
23 ** * The FTS3 module is being built into the core of
24 ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
33 ** Return true if the two-argument version of fts3_tokenizer()
34 ** has been activated via a prior call to sqlite3_db_config(db,
35 ** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
37 static int fts3TokenizerEnabled(sqlite3_context
*context
){
38 sqlite3
*db
= sqlite3_context_db_handle(context
);
40 sqlite3_db_config(db
,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
,-1,&isEnabled
);
45 ** Implementation of the SQL scalar function for accessing the underlying
46 ** hash table. This function may be called as follows:
48 ** SELECT <function-name>(<key-name>);
49 ** SELECT <function-name>(<key-name>, <pointer>);
51 ** where <function-name> is the name passed as the second argument
52 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
54 ** If the <pointer> argument is specified, it must be a blob value
55 ** containing a pointer to be stored as the hash data corresponding
56 ** to the string <key-name>. If <pointer> is not specified, then
57 ** the string <key-name> must already exist in the has table. Otherwise,
58 ** an error is returned.
60 ** Whether or not the <pointer> argument is specified, the value returned
61 ** is a blob containing the pointer stored as the hash data corresponding
62 ** to string <key-name> (after the hash-table is updated, if applicable).
64 static void fts3TokenizerFunc(
65 sqlite3_context
*context
,
71 const unsigned char *zName
;
74 assert( argc
==1 || argc
==2 );
76 pHash
= (Fts3Hash
*)sqlite3_user_data(context
);
78 zName
= sqlite3_value_text(argv
[0]);
79 nName
= sqlite3_value_bytes(argv
[0])+1;
82 if( fts3TokenizerEnabled(context
) ){
84 int n
= sqlite3_value_bytes(argv
[1]);
85 if( zName
==0 || n
!=sizeof(pPtr
) ){
86 sqlite3_result_error(context
, "argument type mismatch", -1);
89 pPtr
= *(void **)sqlite3_value_blob(argv
[1]);
90 pOld
= sqlite3Fts3HashInsert(pHash
, (void *)zName
, nName
, pPtr
);
92 sqlite3_result_error(context
, "out of memory", -1);
95 sqlite3_result_error(context
, "fts3tokenize disabled", -1);
100 pPtr
= sqlite3Fts3HashFind(pHash
, zName
, nName
);
103 char *zErr
= sqlite3_mprintf("unknown tokenizer: %s", zName
);
104 sqlite3_result_error(context
, zErr
, -1);
109 sqlite3_result_blob(context
, (void *)&pPtr
, sizeof(pPtr
), SQLITE_TRANSIENT
);
112 int sqlite3Fts3IsIdChar(char c
){
113 static const char isFtsIdChar
[] = {
114 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
115 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
116 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
117 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
118 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
119 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
120 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
121 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
123 return (c
&0x80 || isFtsIdChar
[(int)(c
)]);
126 const char *sqlite3Fts3NextToken(const char *zStr
, int *pn
){
130 /* Find the start of the next token. */
135 case '\0': return 0; /* No more tokens here */
140 while( *++z2
&& (*z2
!=c
|| *++z2
==c
) );
145 while( *z2
&& z2
[0]!=']' ) z2
++;
150 if( sqlite3Fts3IsIdChar(*z1
) ){
152 while( sqlite3Fts3IsIdChar(*z2
) ) z2
++;
163 int sqlite3Fts3InitTokenizer(
164 Fts3Hash
*pHash
, /* Tokenizer hash table */
165 const char *zArg
, /* Tokenizer name */
166 sqlite3_tokenizer
**ppTok
, /* OUT: Tokenizer (if applicable) */
167 char **pzErr
/* OUT: Set to malloced error message */
170 char *z
= (char *)zArg
;
173 char *zEnd
; /* Pointer to nul-term of zCopy */
174 sqlite3_tokenizer_module
*m
;
176 zCopy
= sqlite3_mprintf("%s", zArg
);
177 if( !zCopy
) return SQLITE_NOMEM
;
178 zEnd
= &zCopy
[strlen(zCopy
)];
180 z
= (char *)sqlite3Fts3NextToken(zCopy
, &n
);
186 sqlite3Fts3Dequote(z
);
188 m
= (sqlite3_tokenizer_module
*)sqlite3Fts3HashFind(pHash
,z
,(int)strlen(z
)+1);
190 sqlite3Fts3ErrMsg(pzErr
, "unknown tokenizer: %s", z
);
193 char const **aArg
= 0;
196 while( z
<zEnd
&& (NULL
!=(z
= (char *)sqlite3Fts3NextToken(z
, &n
))) ){
197 int nNew
= sizeof(char *)*(iArg
+1);
198 char const **aNew
= (const char **)sqlite3_realloc((void *)aArg
, nNew
);
201 sqlite3_free((void *)aArg
);
207 sqlite3Fts3Dequote(z
);
210 rc
= m
->xCreate(iArg
, aArg
, ppTok
);
211 assert( rc
!=SQLITE_OK
|| *ppTok
);
213 sqlite3Fts3ErrMsg(pzErr
, "unknown tokenizer");
215 (*ppTok
)->pModule
= m
;
217 sqlite3_free((void *)aArg
);
227 #if defined(INCLUDE_SQLITE_TCL_H)
228 # include "sqlite_tcl.h"
235 ** Implementation of a special SQL scalar function for testing tokenizers
236 ** designed to be used in concert with the Tcl testing framework. This
237 ** function must be called with two or more arguments:
239 ** SELECT <function-name>(<key-name>, ..., <input-string>);
241 ** where <function-name> is the name passed as the second argument
242 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
243 ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
245 ** The return value is a string that may be interpreted as a Tcl
246 ** list. For each token in the <input-string>, three elements are
247 ** added to the returned list. The first is the token position, the
248 ** second is the token text (folded, stemmed, etc.) and the third is the
249 ** substring of <input-string> associated with the token. For example,
250 ** using the built-in "simple" tokenizer:
252 ** SELECT fts_tokenizer_test('simple', 'I don't see how');
254 ** will return the string:
256 ** "{0 i I 1 dont don't 2 see see 3 how how}"
259 static void testFunc(
260 sqlite3_context
*context
,
265 sqlite3_tokenizer_module
*p
;
266 sqlite3_tokenizer
*pTokenizer
= 0;
267 sqlite3_tokenizer_cursor
*pCsr
= 0;
269 const char *zErr
= 0;
276 const char *azArg
[64];
288 sqlite3_result_error(context
, "insufficient arguments", -1);
292 nName
= sqlite3_value_bytes(argv
[0]);
293 zName
= (const char *)sqlite3_value_text(argv
[0]);
294 nInput
= sqlite3_value_bytes(argv
[argc
-1]);
295 zInput
= (const char *)sqlite3_value_text(argv
[argc
-1]);
297 pHash
= (Fts3Hash
*)sqlite3_user_data(context
);
298 p
= (sqlite3_tokenizer_module
*)sqlite3Fts3HashFind(pHash
, zName
, nName
+1);
301 char *zErr2
= sqlite3_mprintf("unknown tokenizer: %s", zName
);
302 sqlite3_result_error(context
, zErr2
, -1);
308 Tcl_IncrRefCount(pRet
);
310 for(i
=1; i
<argc
-1; i
++){
311 azArg
[i
-1] = (const char *)sqlite3_value_text(argv
[i
]);
314 if( SQLITE_OK
!=p
->xCreate(argc
-2, azArg
, &pTokenizer
) ){
315 zErr
= "error in xCreate()";
318 pTokenizer
->pModule
= p
;
319 if( sqlite3Fts3OpenTokenizer(pTokenizer
, 0, zInput
, nInput
, &pCsr
) ){
320 zErr
= "error in xOpen()";
324 while( SQLITE_OK
==p
->xNext(pCsr
, &zToken
, &nToken
, &iStart
, &iEnd
, &iPos
) ){
325 Tcl_ListObjAppendElement(0, pRet
, Tcl_NewIntObj(iPos
));
326 Tcl_ListObjAppendElement(0, pRet
, Tcl_NewStringObj(zToken
, nToken
));
327 zToken
= &zInput
[iStart
];
328 nToken
= iEnd
-iStart
;
329 Tcl_ListObjAppendElement(0, pRet
, Tcl_NewStringObj(zToken
, nToken
));
332 if( SQLITE_OK
!=p
->xClose(pCsr
) ){
333 zErr
= "error in xClose()";
336 if( SQLITE_OK
!=p
->xDestroy(pTokenizer
) ){
337 zErr
= "error in xDestroy()";
343 sqlite3_result_error(context
, zErr
, -1);
345 sqlite3_result_text(context
, Tcl_GetString(pRet
), -1, SQLITE_TRANSIENT
);
347 Tcl_DecrRefCount(pRet
);
351 int registerTokenizer(
354 const sqlite3_tokenizer_module
*p
358 const char zSql
[] = "SELECT fts3_tokenizer(?, ?)";
360 rc
= sqlite3_prepare_v2(db
, zSql
, -1, &pStmt
, 0);
365 sqlite3_bind_text(pStmt
, 1, zName
, -1, SQLITE_STATIC
);
366 sqlite3_bind_blob(pStmt
, 2, &p
, sizeof(p
), SQLITE_STATIC
);
369 return sqlite3_finalize(pStmt
);
377 const sqlite3_tokenizer_module
**pp
381 const char zSql
[] = "SELECT fts3_tokenizer(?)";
384 rc
= sqlite3_prepare_v2(db
, zSql
, -1, &pStmt
, 0);
389 sqlite3_bind_text(pStmt
, 1, zName
, -1, SQLITE_STATIC
);
390 if( SQLITE_ROW
==sqlite3_step(pStmt
) ){
391 if( sqlite3_column_type(pStmt
, 0)==SQLITE_BLOB
){
392 memcpy((void *)pp
, sqlite3_column_blob(pStmt
, 0), sizeof(*pp
));
396 return sqlite3_finalize(pStmt
);
399 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module
const**ppModule
);
402 ** Implementation of the scalar function fts3_tokenizer_internal_test().
403 ** This function is used for testing only, it is not included in the
404 ** build unless SQLITE_TEST is defined.
406 ** The purpose of this is to test that the fts3_tokenizer() function
407 ** can be used as designed by the C-code in the queryTokenizer and
408 ** registerTokenizer() functions above. These two functions are repeated
409 ** in the README.tokenizer file as an example, so it is important to
412 ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
413 ** function with no arguments. An assert() will fail if a problem is
416 ** SELECT fts3_tokenizer_internal_test();
419 static void intTestFunc(
420 sqlite3_context
*context
,
425 const sqlite3_tokenizer_module
*p1
;
426 const sqlite3_tokenizer_module
*p2
;
427 sqlite3
*db
= (sqlite3
*)sqlite3_user_data(context
);
429 UNUSED_PARAMETER(argc
);
430 UNUSED_PARAMETER(argv
);
432 /* Test the query function */
433 sqlite3Fts3SimpleTokenizerModule(&p1
);
434 rc
= queryTokenizer(db
, "simple", &p2
);
435 assert( rc
==SQLITE_OK
);
437 rc
= queryTokenizer(db
, "nosuchtokenizer", &p2
);
438 assert( rc
==SQLITE_ERROR
);
440 assert( 0==strcmp(sqlite3_errmsg(db
), "unknown tokenizer: nosuchtokenizer") );
442 /* Test the storage function */
443 if( fts3TokenizerEnabled(context
) ){
444 rc
= registerTokenizer(db
, "nosuchtokenizer", p1
);
445 assert( rc
==SQLITE_OK
);
446 rc
= queryTokenizer(db
, "nosuchtokenizer", &p2
);
447 assert( rc
==SQLITE_OK
);
451 sqlite3_result_text(context
, "ok", -1, SQLITE_STATIC
);
457 ** Set up SQL objects in database db used to access the contents of
458 ** the hash table pointed to by argument pHash. The hash table must
459 ** been initialized to use string keys, and to take a private copy
460 ** of the key when a value is inserted. i.e. by a call similar to:
462 ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
464 ** This function adds a scalar function (see header comment above
465 ** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
466 ** defined at compilation time, a temporary virtual table (see header
467 ** comment above struct HashTableVtab) to the database schema. Both
468 ** provide read/write access to the contents of *pHash.
470 ** The third argument to this function, zName, is used as the name
471 ** of both the scalar and, if created, the virtual table.
473 int sqlite3Fts3InitHashTable(
479 void *p
= (void *)pHash
;
480 const int any
= SQLITE_ANY
;
485 void *pdb
= (void *)db
;
486 zTest
= sqlite3_mprintf("%s_test", zName
);
487 zTest2
= sqlite3_mprintf("%s_internal_test", zName
);
488 if( !zTest
|| !zTest2
){
494 rc
= sqlite3_create_function(db
, zName
, 1, any
, p
, fts3TokenizerFunc
, 0, 0);
497 rc
= sqlite3_create_function(db
, zName
, 2, any
, p
, fts3TokenizerFunc
, 0, 0);
501 rc
= sqlite3_create_function(db
, zTest
, -1, any
, p
, testFunc
, 0, 0);
504 rc
= sqlite3_create_function(db
, zTest2
, 0, any
, pdb
, intTestFunc
, 0, 0);
510 sqlite3_free(zTest2
);
516 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */