ext/fts3/fts3_tokenizer.c

   1 /*
   2 ** 2007 June 22
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This is part of an SQLite module implementing full-text search.
  14 ** This particular file implements the generic tokenizer interface.
  15 */
  16
  17 /*
  18 ** The code in this file is only compiled if:
  19 **
  20 **     * The FTS3 module is being built as an extension
  21 **       (in which case SQLITE_CORE is not defined), or
  22 **
  23 **     * The FTS3 module is being built into the core of
  24 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
  25 */
  26 #include "fts3Int.h"
  27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
  28
  29 #include <assert.h>
  30 #include <string.h>
  31
  32 /*
  33 ** Return true if the two-argument version of fts3_tokenizer()
  34 ** has been activated via a prior call to sqlite3_db_config(db,
  35 ** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
  36 */
  37 static int fts3TokenizerEnabled(sqlite3_context *context){
  38   sqlite3 *db = sqlite3_context_db_handle(context);
  39   int isEnabled = 0;
  40   sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
  41   return isEnabled;
  42 }
  43
  44 /*
  45 ** Implementation of the SQL scalar function for accessing the underlying
  46 ** hash table. This function may be called as follows:
  47 **
  48 **   SELECT <function-name>(<key-name>);
  49 **   SELECT <function-name>(<key-name>, <pointer>);
  50 **
  51 ** where <function-name> is the name passed as the second argument
  52 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
  53 **
  54 ** If the <pointer> argument is specified, it must be a blob value
  55 ** containing a pointer to be stored as the hash data corresponding
  56 ** to the string <key-name>. If <pointer> is not specified, then
  57 ** the string <key-name> must already exist in the has table. Otherwise,
  58 ** an error is returned.
  59 **
  60 ** Whether or not the <pointer> argument is specified, the value returned
  61 ** is a blob containing the pointer stored as the hash data corresponding
  62 ** to string <key-name> (after the hash-table is updated, if applicable).
  63 */
  64 static void fts3TokenizerFunc(
  65   sqlite3_context *context,
  66   int argc,
  67   sqlite3_value **argv
  68 ){
  69   Fts3Hash *pHash;
  70   void *pPtr = 0;
  71   const unsigned char *zName;
  72   int nName;
  73
  74   assert( argc==1 || argc==2 );
  75
  76   pHash = (Fts3Hash *)sqlite3_user_data(context);
  77
  78   zName = sqlite3_value_text(argv[0]);
  79   nName = sqlite3_value_bytes(argv[0])+1;
  80
  81   if( argc==2 ){
  82     if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){
  83       void *pOld;
  84       int n = sqlite3_value_bytes(argv[1]);
  85       if( zName==0 || n!=sizeof(pPtr) ){
  86         sqlite3_result_error(context, "argument type mismatch", -1);
  87         return;
  88       }
  89       pPtr = *(void **)sqlite3_value_blob(argv[1]);
  90       pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
  91       if( pOld==pPtr ){
  92         sqlite3_result_error(context, "out of memory", -1);
  93       }
  94     }else{
  95       sqlite3_result_error(context, "fts3tokenize disabled", -1);
  96       return;
  97     }
  98   }else{
  99     if( zName ){
 100       pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
 101     }
 102     if( !pPtr ){
 103       char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
 104       sqlite3_result_error(context, zErr, -1);
 105       sqlite3_free(zErr);
 106       return;
 107     }
 108   }
 109   if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
 110     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
 111   }
 112 }
 113
 114 int sqlite3Fts3IsIdChar(char c){
 115   static const char isFtsIdChar[] = {
 116       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
 117       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
 118       0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
 119       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
 120       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
 121       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
 122       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
 123       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 124   };
 125   return (c&0x80 || isFtsIdChar[(int)(c)]);
 126 }
 127
 128 const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
 129   const char *z1;
 130   const char *z2 = 0;
 131
 132   /* Find the start of the next token. */
 133   z1 = zStr;
 134   while( z2==0 ){
 135     char c = *z1;
 136     switch( c ){
 137       case '\0': return 0;        /* No more tokens here */
 138       case '\'':
 139       case '"':
 140       case '`': {
 141         z2 = z1;
 142         while( *++z2 && (*z2!=c || *++z2==c) );
 143         break;
 144       }
 145       case '[':
 146         z2 = &z1[1];
 147         while( *z2 && z2[0]!=']' ) z2++;
 148         if( *z2 ) z2++;
 149         break;
 150
 151       default:
 152         if( sqlite3Fts3IsIdChar(*z1) ){
 153           z2 = &z1[1];
 154           while( sqlite3Fts3IsIdChar(*z2) ) z2++;
 155         }else{
 156           z1++;
 157         }
 158     }
 159   }
 160
 161   *pn = (int)(z2-z1);
 162   return z1;
 163 }
 164
 165 int sqlite3Fts3InitTokenizer(
 166   Fts3Hash *pHash,                /* Tokenizer hash table */
 167   const char *zArg,               /* Tokenizer name */
 168   sqlite3_tokenizer **ppTok,      /* OUT: Tokenizer (if applicable) */
 169   char **pzErr                    /* OUT: Set to malloced error message */
 170 ){
 171   int rc;
 172   char *z = (char *)zArg;
 173   int n = 0;
 174   char *zCopy;
 175   char *zEnd;                     /* Pointer to nul-term of zCopy */
 176   sqlite3_tokenizer_module *m;
 177
 178   zCopy = sqlite3_mprintf("%s", zArg);
 179   if( !zCopy ) return SQLITE_NOMEM;
 180   zEnd = &zCopy[strlen(zCopy)];
 181
 182   z = (char *)sqlite3Fts3NextToken(zCopy, &n);
 183   if( z==0 ){
 184     assert( n==0 );
 185     z = zCopy;
 186   }
 187   z[n] = '\0';
 188   sqlite3Fts3Dequote(z);
 189
 190   m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
 191   if( !m ){
 192     sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z);
 193     rc = SQLITE_ERROR;
 194   }else{
 195     char const **aArg = 0;
 196     int iArg = 0;
 197     z = &z[n+1];
 198     while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
 199       sqlite3_int64 nNew = sizeof(char *)*(iArg+1);
 200       char const **aNew = (const char **)sqlite3_realloc64((void *)aArg, nNew);
 201       if( !aNew ){
 202         sqlite3_free(zCopy);
 203         sqlite3_free((void *)aArg);
 204         return SQLITE_NOMEM;
 205       }
 206       aArg = aNew;
 207       aArg[iArg++] = z;
 208       z[n] = '\0';
 209       sqlite3Fts3Dequote(z);
 210       z = &z[n+1];
 211     }
 212     rc = m->xCreate(iArg, aArg, ppTok);
 213     assert( rc!=SQLITE_OK || *ppTok );
 214     if( rc!=SQLITE_OK ){
 215       sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer");
 216     }else{
 217       (*ppTok)->pModule = m;
 218     }
 219     sqlite3_free((void *)aArg);
 220   }
 221
 222   sqlite3_free(zCopy);
 223   return rc;
 224 }
 225
 226
 227 #ifdef SQLITE_TEST
 228
 229 #if defined(INCLUDE_SQLITE_TCL_H)
 230 #  include "sqlite_tcl.h"
 231 #else
 232 #  include "tcl.h"
 233 #endif
 234 #include <string.h>
 235
 236 /*
 237 ** Implementation of a special SQL scalar function for testing tokenizers
 238 ** designed to be used in concert with the Tcl testing framework. This
 239 ** function must be called with two or more arguments:
 240 **
 241 **   SELECT <function-name>(<key-name>, ..., <input-string>);
 242 **
 243 ** where <function-name> is the name passed as the second argument
 244 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
 245 ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
 246 **
 247 ** The return value is a string that may be interpreted as a Tcl
 248 ** list. For each token in the <input-string>, three elements are
 249 ** added to the returned list. The first is the token position, the
 250 ** second is the token text (folded, stemmed, etc.) and the third is the
 251 ** substring of <input-string> associated with the token. For example,
 252 ** using the built-in "simple" tokenizer:
 253 **
 254 **   SELECT fts_tokenizer_test('simple', 'I don't see how');
 255 **
 256 ** will return the string:
 257 **
 258 **   "{0 i I 1 dont don't 2 see see 3 how how}"
 259 **
 260 */
 261 static void testFunc(
 262   sqlite3_context *context,
 263   int argc,
 264   sqlite3_value **argv
 265 ){
 266   Fts3Hash *pHash;
 267   sqlite3_tokenizer_module *p;
 268   sqlite3_tokenizer *pTokenizer = 0;
 269   sqlite3_tokenizer_cursor *pCsr = 0;
 270
 271   const char *zErr = 0;
 272
 273   const char *zName;
 274   int nName;
 275   const char *zInput;
 276   int nInput;
 277
 278   const char *azArg[64];
 279
 280   const char *zToken;
 281   int nToken = 0;
 282   int iStart = 0;
 283   int iEnd = 0;
 284   int iPos = 0;
 285   int i;
 286
 287   Tcl_Obj *pRet;
 288
 289   if( argc<2 ){
 290     sqlite3_result_error(context, "insufficient arguments", -1);
 291     return;
 292   }
 293
 294   nName = sqlite3_value_bytes(argv[0]);
 295   zName = (const char *)sqlite3_value_text(argv[0]);
 296   nInput = sqlite3_value_bytes(argv[argc-1]);
 297   zInput = (const char *)sqlite3_value_text(argv[argc-1]);
 298
 299   pHash = (Fts3Hash *)sqlite3_user_data(context);
 300   p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
 301
 302   if( !p ){
 303     char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName);
 304     sqlite3_result_error(context, zErr2, -1);
 305     sqlite3_free(zErr2);
 306     return;
 307   }
 308
 309   pRet = Tcl_NewObj();
 310   Tcl_IncrRefCount(pRet);
 311
 312   for(i=1; i<argc-1; i++){
 313     azArg[i-1] = (const char *)sqlite3_value_text(argv[i]);
 314   }
 315
 316   if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){
 317     zErr = "error in xCreate()";
 318     goto finish;
 319   }
 320   pTokenizer->pModule = p;
 321   if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){
 322     zErr = "error in xOpen()";
 323     goto finish;
 324   }
 325
 326   while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
 327     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
 328     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 329     zToken = &zInput[iStart];
 330     nToken = iEnd-iStart;
 331     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 332   }
 333
 334   if( SQLITE_OK!=p->xClose(pCsr) ){
 335     zErr = "error in xClose()";
 336     goto finish;
 337   }
 338   if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
 339     zErr = "error in xDestroy()";
 340     goto finish;
 341   }
 342
 343 finish:
 344   if( zErr ){
 345     sqlite3_result_error(context, zErr, -1);
 346   }else{
 347     sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
 348   }
 349   Tcl_DecrRefCount(pRet);
 350 }
 351
 352 static
 353 int registerTokenizer(
 354   sqlite3 *db,
 355   char *zName,
 356   const sqlite3_tokenizer_module *p
 357 ){
 358   int rc;
 359   sqlite3_stmt *pStmt;
 360   const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
 361
 362   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 363   if( rc!=SQLITE_OK ){
 364     return rc;
 365   }
 366
 367   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 368   sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
 369   sqlite3_step(pStmt);
 370
 371   return sqlite3_finalize(pStmt);
 372 }
 373
 374
 375 static
 376 int queryTokenizer(
 377   sqlite3 *db,
 378   char *zName,
 379   const sqlite3_tokenizer_module **pp
 380 ){
 381   int rc;
 382   sqlite3_stmt *pStmt;
 383   const char zSql[] = "SELECT fts3_tokenizer(?)";
 384
 385   *pp = 0;
 386   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 387   if( rc!=SQLITE_OK ){
 388     return rc;
 389   }
 390
 391   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 392   if( SQLITE_ROW==sqlite3_step(pStmt) ){
 393     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB
 394      && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp)
 395     ){
 396       memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
 397     }
 398   }
 399
 400   return sqlite3_finalize(pStmt);
 401 }
 402
 403 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 404
 405 /*
 406 ** Implementation of the scalar function fts3_tokenizer_internal_test().
 407 ** This function is used for testing only, it is not included in the
 408 ** build unless SQLITE_TEST is defined.
 409 **
 410 ** The purpose of this is to test that the fts3_tokenizer() function
 411 ** can be used as designed by the C-code in the queryTokenizer and
 412 ** registerTokenizer() functions above. These two functions are repeated
 413 ** in the README.tokenizer file as an example, so it is important to
 414 ** test them.
 415 **
 416 ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
 417 ** function with no arguments. An assert() will fail if a problem is
 418 ** detected. i.e.:
 419 **
 420 **     SELECT fts3_tokenizer_internal_test();
 421 **
 422 */
 423 static void intTestFunc(
 424   sqlite3_context *context,
 425   int argc,
 426   sqlite3_value **argv
 427 ){
 428   int rc;
 429   const sqlite3_tokenizer_module *p1;
 430   const sqlite3_tokenizer_module *p2;
 431   sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
 432
 433   UNUSED_PARAMETER(argc);
 434   UNUSED_PARAMETER(argv);
 435
 436   /* Test the query function */
 437   sqlite3Fts3SimpleTokenizerModule(&p1);
 438   rc = queryTokenizer(db, "simple", &p2);
 439   assert( rc==SQLITE_OK );
 440   assert( p1==p2 );
 441   rc = queryTokenizer(db, "nosuchtokenizer", &p2);
 442   assert( rc==SQLITE_ERROR );
 443   assert( p2==0 );
 444   assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
 445
 446   /* Test the storage function */
 447   if( fts3TokenizerEnabled(context) ){
 448     rc = registerTokenizer(db, "nosuchtokenizer", p1);
 449     assert( rc==SQLITE_OK );
 450     rc = queryTokenizer(db, "nosuchtokenizer", &p2);
 451     assert( rc==SQLITE_OK );
 452     assert( p2==p1 );
 453   }
 454
 455   sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 456 }
 457
 458 #endif
 459
 460 /*
 461 ** Set up SQL objects in database db used to access the contents of
 462 ** the hash table pointed to by argument pHash. The hash table must
 463 ** been initialized to use string keys, and to take a private copy
 464 ** of the key when a value is inserted. i.e. by a call similar to:
 465 **
 466 **    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
 467 **
 468 ** This function adds a scalar function (see header comment above
 469 ** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
 470 ** defined at compilation time, a temporary virtual table (see header
 471 ** comment above struct HashTableVtab) to the database schema. Both
 472 ** provide read/write access to the contents of *pHash.
 473 **
 474 ** The third argument to this function, zName, is used as the name
 475 ** of both the scalar and, if created, the virtual table.
 476 */
 477 int sqlite3Fts3InitHashTable(
 478   sqlite3 *db,
 479   Fts3Hash *pHash,
 480   const char *zName
 481 ){
 482   int rc = SQLITE_OK;
 483   void *p = (void *)pHash;
 484   const int any = SQLITE_UTF8|SQLITE_DIRECTONLY;
 485
 486 #ifdef SQLITE_TEST
 487   char *zTest = 0;
 488   char *zTest2 = 0;
 489   void *pdb = (void *)db;
 490   zTest = sqlite3_mprintf("%s_test", zName);
 491   zTest2 = sqlite3_mprintf("%s_internal_test", zName);
 492   if( !zTest || !zTest2 ){
 493     rc = SQLITE_NOMEM;
 494   }
 495 #endif
 496
 497   if( SQLITE_OK==rc ){
 498     rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
 499   }
 500   if( SQLITE_OK==rc ){
 501     rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
 502   }
 503 #ifdef SQLITE_TEST
 504   if( SQLITE_OK==rc ){
 505     rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
 506   }
 507   if( SQLITE_OK==rc ){
 508     rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
 509   }
 510 #endif
 511
 512 #ifdef SQLITE_TEST
 513   sqlite3_free(zTest);
 514   sqlite3_free(zTest2);
 515 #endif
 516
 517   return rc;
 518 }
 519
 520 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */