ext/fts5/fts5_tcl.c

   1 /*
   2 ** 2014 Dec 01
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14
  15
  16 #ifdef SQLITE_TEST
  17 #if defined(INCLUDE_SQLITE_TCL_H)
  18 #  include "sqlite_tcl.h"
  19 #else
  20 #  include "tcl.h"
  21 #  ifndef SQLITE_TCLAPI
  22 #    define SQLITE_TCLAPI
  23 #  endif
  24 #endif
  25
  26 #ifdef SQLITE_ENABLE_FTS5
  27
  28 #include "fts5.h"
  29 #include <string.h>
  30 #include <assert.h>
  31
  32 #ifdef SQLITE_DEBUG
  33 extern int sqlite3_fts5_may_be_corrupt;
  34 #endif
  35 extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
  36 extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);
  37
  38 /*************************************************************************
  39 ** This is a copy of the first part of the SqliteDb structure in
  40 ** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
  41 ** can extract the sqlite3* pointer from an existing Tcl SQLite
  42 ** connection.
  43 */
  44
  45 extern const char *sqlite3ErrName(int);
  46
  47 struct SqliteDb {
  48   sqlite3 *db;
  49 };
  50
  51 /*
  52 ** Decode a pointer to an sqlite3 object.
  53 */
  54 static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){
  55   struct SqliteDb *p;
  56   Tcl_CmdInfo cmdInfo;
  57   char *z = Tcl_GetString(pObj);
  58   if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){
  59     p = (struct SqliteDb*)cmdInfo.objClientData;
  60     *ppDb = p->db;
  61     return TCL_OK;
  62   }
  63   return TCL_ERROR;
  64 }
  65
  66 /* End of code that accesses the SqliteDb struct.
  67 **************************************************************************/
  68
  69 static int f5tResultToErrorCode(const char *zRes){
  70   struct ErrorCode {
  71     int rc;
  72     const char *zError;
  73   } aErr[] = {
  74     { SQLITE_DONE,  "SQLITE_DONE" },
  75     { SQLITE_ERROR, "SQLITE_ERROR" },
  76     { SQLITE_OK,    "SQLITE_OK" },
  77     { SQLITE_OK,    "" },
  78   };
  79   int i;
  80
  81   for(i=0; i<sizeof(aErr)/sizeof(aErr[0]); i++){
  82     if( 0==sqlite3_stricmp(zRes, aErr[i].zError) ){
  83       return aErr[i].rc;
  84     }
  85   }
  86
  87   return SQLITE_ERROR;
  88 }
  89
  90 static int SQLITE_TCLAPI f5tDbAndApi(
  91   Tcl_Interp *interp,
  92   Tcl_Obj *pObj,
  93   sqlite3 **ppDb,
  94   fts5_api **ppApi
  95 ){
  96   sqlite3 *db = 0;
  97   int rc = f5tDbPointer(interp, pObj, &db);
  98   if( rc!=TCL_OK ){
  99     return TCL_ERROR;
 100   }else{
 101     sqlite3_stmt *pStmt = 0;
 102     fts5_api *pApi = 0;
 103
 104     rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
 105     if( rc!=SQLITE_OK ){
 106       Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
 107       return TCL_ERROR;
 108     }
 109     sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
 110     sqlite3_step(pStmt);
 111
 112     if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
 113       Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
 114       return TCL_ERROR;
 115     }
 116
 117     *ppDb = db;
 118     *ppApi = pApi;
 119   }
 120
 121   return TCL_OK;
 122 }
 123
 124 typedef struct F5tFunction F5tFunction;
 125 struct F5tFunction {
 126   Tcl_Interp *interp;
 127   Tcl_Obj *pScript;
 128 };
 129
 130 typedef struct F5tApi F5tApi;
 131 struct F5tApi {
 132   const Fts5ExtensionApi *pApi;
 133   Fts5Context *pFts;
 134 };
 135
 136 /*
 137 ** An object of this type is used with the xSetAuxdata() and xGetAuxdata()
 138 ** API test wrappers. The tcl interface allows a single tcl value to be
 139 ** saved using xSetAuxdata(). Instead of simply storing a pointer to the
 140 ** tcl object, the code in this file wraps it in an sqlite3_malloc'd
 141 ** instance of the following struct so that if the destructor is not
 142 ** correctly invoked it will be reported as an SQLite memory leak.
 143 */
 144 typedef struct F5tAuxData F5tAuxData;
 145 struct F5tAuxData {
 146   Tcl_Obj *pObj;
 147 };
 148
 149 static int xTokenizeCb(
 150   void *pCtx,
 151   int tflags,
 152   const char *zToken, int nToken,
 153   int iStart, int iEnd
 154 ){
 155   F5tFunction *p = (F5tFunction*)pCtx;
 156   Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript);
 157   int rc;
 158
 159   Tcl_IncrRefCount(pEval);
 160   Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
 161   Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
 162   Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));
 163
 164   rc = Tcl_EvalObjEx(p->interp, pEval, 0);
 165   Tcl_DecrRefCount(pEval);
 166   if( rc==TCL_OK ){
 167     rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
 168   }
 169
 170   return rc;
 171 }
 172
 173 static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
 174
 175 static int xQueryPhraseCb(
 176   const Fts5ExtensionApi *pApi,
 177   Fts5Context *pFts,
 178   void *pCtx
 179 ){
 180   F5tFunction *p = (F5tFunction*)pCtx;
 181   static sqlite3_int64 iCmd = 0;
 182   Tcl_Obj *pEval;
 183   int rc;
 184
 185   char zCmd[64];
 186   F5tApi sApi;
 187
 188   sApi.pApi = pApi;
 189   sApi.pFts = pFts;
 190   sprintf(zCmd, "f5t_2_%lld", iCmd++);
 191   Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);
 192
 193   pEval = Tcl_DuplicateObj(p->pScript);
 194   Tcl_IncrRefCount(pEval);
 195   Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
 196   rc = Tcl_EvalObjEx(p->interp, pEval, 0);
 197   Tcl_DecrRefCount(pEval);
 198   Tcl_DeleteCommand(p->interp, zCmd);
 199
 200   if( rc==TCL_OK ){
 201     rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
 202   }
 203
 204   return rc;
 205 }
 206
 207 static void xSetAuxdataDestructor(void *p){
 208   F5tAuxData *pData = (F5tAuxData*)p;
 209   Tcl_DecrRefCount(pData->pObj);
 210   sqlite3_free(pData);
 211 }
 212
 213 /*
 214 **      api sub-command...
 215 **
 216 ** Description...
 217 */
 218 static int SQLITE_TCLAPI xF5tApi(
 219   void * clientData,
 220   Tcl_Interp *interp,
 221   int objc,
 222   Tcl_Obj *CONST objv[]
 223 ){
 224   struct Sub {
 225     const char *zName;
 226     int nArg;
 227     const char *zMsg;
 228   } aSub[] = {
 229     { "xColumnCount",      0, "" },                   /*  0 */
 230     { "xRowCount",         0, "" },                   /*  1 */
 231     { "xColumnTotalSize",  1, "COL" },                /*  2 */
 232     { "xTokenize",         2, "TEXT SCRIPT" },        /*  3 */
 233     { "xPhraseCount",      0, "" },                   /*  4 */
 234     { "xPhraseSize",       1, "PHRASE" },             /*  5 */
 235     { "xInstCount",        0, "" },                   /*  6 */
 236     { "xInst",             1, "IDX" },                /*  7 */
 237     { "xRowid",            0, "" },                   /*  8 */
 238     { "xColumnText",       1, "COL" },                /*  9 */
 239     { "xColumnSize",       1, "COL" },                /* 10 */
 240     { "xQueryPhrase",      2, "PHRASE SCRIPT" },      /* 11 */
 241     { "xSetAuxdata",       1, "VALUE" },              /* 12 */
 242     { "xGetAuxdata",       1, "CLEAR" },              /* 13 */
 243     { "xSetAuxdataInt",    1, "INTEGER" },            /* 14 */
 244     { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
 245     { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
 246     { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
 247
 248     { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
 249     { "xInstToken",        2, "IDX ITERM" },          /* 19 */
 250     { 0, 0, 0}
 251   };
 252
 253   int rc;
 254   int iSub = 0;
 255   F5tApi *p = (F5tApi*)clientData;
 256
 257   if( objc<2 ){
 258     Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");
 259     return TCL_ERROR;
 260   }
 261
 262   rc = Tcl_GetIndexFromObjStruct(
 263       interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub
 264   );
 265   if( rc!=TCL_OK ) return rc;
 266   if( aSub[iSub].nArg!=objc-2 ){
 267     Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);
 268     return TCL_ERROR;
 269   }
 270
 271 #define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 );
 272   switch( iSub ){
 273     CASE(0, "xColumnCount") {
 274       int nCol;
 275       nCol = p->pApi->xColumnCount(p->pFts);
 276       if( rc==SQLITE_OK ){
 277         Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
 278       }
 279       break;
 280     }
 281     CASE(1, "xRowCount") {
 282       sqlite3_int64 nRow;
 283       rc = p->pApi->xRowCount(p->pFts, &nRow);
 284       if( rc==SQLITE_OK ){
 285         Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow));
 286       }
 287       break;
 288     }
 289     CASE(2, "xColumnTotalSize") {
 290       int iCol;
 291       sqlite3_int64 nSize;
 292       if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR;
 293       rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize);
 294       if( rc==SQLITE_OK ){
 295         Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
 296       }
 297       break;
 298     }
 299     CASE(3, "xTokenize") {
 300       int nText;
 301       char *zText = Tcl_GetStringFromObj(objv[2], &nText);
 302       F5tFunction ctx;
 303       ctx.interp = interp;
 304       ctx.pScript = objv[3];
 305       rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
 306       if( rc==SQLITE_OK ){
 307         Tcl_ResetResult(interp);
 308       }
 309       return rc;
 310     }
 311     CASE(4, "xPhraseCount") {
 312       int nPhrase;
 313       nPhrase = p->pApi->xPhraseCount(p->pFts);
 314       if( rc==SQLITE_OK ){
 315         Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase));
 316       }
 317       break;
 318     }
 319     CASE(5, "xPhraseSize") {
 320       int iPhrase;
 321       int sz;
 322       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
 323         return TCL_ERROR;
 324       }
 325       sz = p->pApi->xPhraseSize(p->pFts, iPhrase);
 326       if( rc==SQLITE_OK ){
 327         Tcl_SetObjResult(interp, Tcl_NewIntObj(sz));
 328       }
 329       break;
 330     }
 331     CASE(6, "xInstCount") {
 332       int nInst;
 333       rc = p->pApi->xInstCount(p->pFts, &nInst);
 334       if( rc==SQLITE_OK ){
 335         Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));
 336       }
 337       break;
 338     }
 339     CASE(7, "xInst") {
 340       int iIdx, ip, ic, io;
 341       if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
 342         return TCL_ERROR;
 343       }
 344       rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);
 345       if( rc==SQLITE_OK ){
 346         Tcl_Obj *pList = Tcl_NewObj();
 347         Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));
 348         Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));
 349         Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));
 350         Tcl_SetObjResult(interp, pList);
 351       }
 352       break;
 353     }
 354     CASE(8, "xRowid") {
 355       sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);
 356       Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));
 357       break;
 358     }
 359     CASE(9, "xColumnText") {
 360       const char *z = 0;
 361       int n = 0;
 362       int iCol;
 363       if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
 364         return TCL_ERROR;
 365       }
 366       rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);
 367       if( rc==SQLITE_OK ){
 368         Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
 369       }
 370       break;
 371     }
 372     CASE(10, "xColumnSize") {
 373       int n = 0;
 374       int iCol;
 375       if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
 376         return TCL_ERROR;
 377       }
 378       rc = p->pApi->xColumnSize(p->pFts, iCol, &n);
 379       if( rc==SQLITE_OK ){
 380         Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
 381       }
 382       break;
 383     }
 384     CASE(11, "xQueryPhrase") {
 385       int iPhrase;
 386       F5tFunction ctx;
 387       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
 388         return TCL_ERROR;
 389       }
 390       ctx.interp = interp;
 391       ctx.pScript = objv[3];
 392       rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb);
 393       if( rc==SQLITE_OK ){
 394         Tcl_ResetResult(interp);
 395       }
 396       break;
 397     }
 398     CASE(12, "xSetAuxdata") {
 399       F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));
 400       if( pData==0 ){
 401         Tcl_AppendResult(interp, "out of memory", 0);
 402         return TCL_ERROR;
 403       }
 404       pData->pObj = objv[2];
 405       Tcl_IncrRefCount(pData->pObj);
 406       rc = p->pApi->xSetAuxdata(p->pFts, pData, xSetAuxdataDestructor);
 407       break;
 408     }
 409     CASE(13, "xGetAuxdata") {
 410       F5tAuxData *pData;
 411       int bClear;
 412       if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ){
 413         return TCL_ERROR;
 414       }
 415       pData = (F5tAuxData*)p->pApi->xGetAuxdata(p->pFts, bClear);
 416       if( pData==0 ){
 417         Tcl_ResetResult(interp);
 418       }else{
 419         Tcl_SetObjResult(interp, pData->pObj);
 420         if( bClear ){
 421           xSetAuxdataDestructor((void*)pData);
 422         }
 423       }
 424       break;
 425     }
 426
 427     /* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the
 428     ** xSetAuxdata and xGetAuxdata methods implemented above. The difference
 429     ** is that they may only save an integer value as auxiliary data, and
 430     ** do not specify a destructor function.  */
 431     CASE(14, "xSetAuxdataInt") {
 432       int iVal;
 433       if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;
 434       rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
 435       break;
 436     }
 437     CASE(15, "xGetAuxdataInt") {
 438       int iVal;
 439       int bClear;
 440       if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
 441       iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
 442       Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
 443       break;
 444     }
 445
 446     CASE(16, "xPhraseForeach") {
 447       int iPhrase;
 448       int iCol;
 449       int iOff;
 450       const char *zColvar;
 451       const char *zOffvar;
 452       Tcl_Obj *pScript = objv[5];
 453       Fts5PhraseIter iter;
 454
 455       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
 456       zColvar = Tcl_GetString(objv[3]);
 457       zOffvar = Tcl_GetString(objv[4]);
 458
 459       rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
 460       if( rc!=SQLITE_OK ){
 461         Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
 462         return TCL_ERROR;
 463       }
 464       for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
 465         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
 466         Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
 467         rc = Tcl_EvalObjEx(interp, pScript, 0);
 468         if( rc==TCL_CONTINUE ) rc = TCL_OK;
 469         if( rc!=TCL_OK ){
 470           if( rc==TCL_BREAK ) rc = TCL_OK;
 471           break;
 472         }
 473       }
 474
 475       break;
 476     }
 477
 478     CASE(17, "xPhraseColumnForeach") {
 479       int iPhrase;
 480       int iCol;
 481       const char *zColvar;
 482       Tcl_Obj *pScript = objv[4];
 483       Fts5PhraseIter iter;
 484
 485       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
 486       zColvar = Tcl_GetString(objv[3]);
 487
 488       rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
 489       if( rc!=SQLITE_OK ){
 490         Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
 491         return TCL_ERROR;
 492       }
 493       for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
 494         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
 495         rc = Tcl_EvalObjEx(interp, pScript, 0);
 496         if( rc==TCL_CONTINUE ) rc = TCL_OK;
 497         if( rc!=TCL_OK ){
 498           if( rc==TCL_BREAK ) rc = TCL_OK;
 499           break;
 500         }
 501       }
 502
 503       break;
 504     }
 505
 506     CASE(18, "xQueryToken") {
 507       const char *pTerm = 0;
 508       int nTerm = 0;
 509       int iPhrase = 0;
 510       int iTerm = 0;
 511
 512       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
 513       if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
 514       rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
 515       if( rc==SQLITE_OK ){
 516         Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
 517       }
 518
 519       break;
 520     }
 521
 522     CASE(19, "xInstToken") {
 523       const char *pTerm = 0;
 524       int nTerm = 0;
 525       int iIdx = 0;
 526       int iTerm = 0;
 527
 528       if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
 529       if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
 530       rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
 531       if( rc==SQLITE_OK ){
 532         Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
 533       }
 534
 535       break;
 536     }
 537
 538     default:
 539       assert( 0 );
 540       break;
 541   }
 542 #undef CASE
 543
 544   if( rc!=SQLITE_OK ){
 545     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
 546     return TCL_ERROR;
 547   }
 548
 549   return TCL_OK;
 550 }
 551
 552 static void xF5tFunction(
 553   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
 554   Fts5Context *pFts,              /* First arg to pass to pApi functions */
 555   sqlite3_context *pCtx,          /* Context for returning result/error */
 556   int nVal,                       /* Number of values in apVal[] array */
 557   sqlite3_value **apVal           /* Array of trailing arguments */
 558 ){
 559   F5tFunction *p = (F5tFunction*)pApi->xUserData(pFts);
 560   Tcl_Obj *pEval;                 /* Script to evaluate */
 561   int i;
 562   int rc;
 563
 564   static sqlite3_int64 iCmd = 0;
 565   char zCmd[64];
 566   F5tApi sApi;
 567   sApi.pApi = pApi;
 568   sApi.pFts = pFts;
 569
 570   sprintf(zCmd, "f5t_%lld", iCmd++);
 571   Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);
 572   pEval = Tcl_DuplicateObj(p->pScript);
 573   Tcl_IncrRefCount(pEval);
 574   Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
 575
 576   for(i=0; i<nVal; i++){
 577     Tcl_Obj *pObj = 0;
 578     switch( sqlite3_value_type(apVal[i]) ){
 579       case SQLITE_TEXT:
 580         pObj = Tcl_NewStringObj((const char*)sqlite3_value_text(apVal[i]), -1);
 581         break;
 582       case SQLITE_BLOB:
 583         pObj = Tcl_NewByteArrayObj(
 584             sqlite3_value_blob(apVal[i]), sqlite3_value_bytes(apVal[i])
 585         );
 586         break;
 587       case SQLITE_INTEGER:
 588         pObj = Tcl_NewWideIntObj(sqlite3_value_int64(apVal[i]));
 589         break;
 590       case SQLITE_FLOAT:
 591         pObj = Tcl_NewDoubleObj(sqlite3_value_double(apVal[i]));
 592         break;
 593       default:
 594         pObj = Tcl_NewObj();
 595         break;
 596     }
 597     Tcl_ListObjAppendElement(p->interp, pEval, pObj);
 598   }
 599
 600   rc = Tcl_EvalObjEx(p->interp, pEval, TCL_GLOBAL_ONLY);
 601   Tcl_DecrRefCount(pEval);
 602   Tcl_DeleteCommand(p->interp, zCmd);
 603
 604   if( rc!=TCL_OK ){
 605     sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
 606   }else{
 607     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
 608     int n;
 609     const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
 610     char c = zType[0];
 611     if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
 612       /* Only return a BLOB type if the Tcl variable is a bytearray and
 613       ** has no string representation. */
 614       unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
 615       sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
 616     }else if( c=='b' && strcmp(zType,"boolean")==0 ){
 617       Tcl_GetIntFromObj(0, pVar, &n);
 618       sqlite3_result_int(pCtx, n);
 619     }else if( c=='d' && strcmp(zType,"double")==0 ){
 620       double r;
 621       Tcl_GetDoubleFromObj(0, pVar, &r);
 622       sqlite3_result_double(pCtx, r);
 623     }else if( (c=='w' && strcmp(zType,"wideInt")==0) ||
 624           (c=='i' && strcmp(zType,"int")==0) ){
 625       Tcl_WideInt v;
 626       Tcl_GetWideIntFromObj(0, pVar, &v);
 627       sqlite3_result_int64(pCtx, v);
 628     }else{
 629       unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
 630       sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
 631     }
 632   }
 633 }
 634
 635 static void xF5tDestroy(void *pCtx){
 636   F5tFunction *p = (F5tFunction*)pCtx;
 637   Tcl_DecrRefCount(p->pScript);
 638   ckfree((char *)p);
 639 }
 640
 641 /*
 642 **      sqlite3_fts5_create_function DB NAME SCRIPT
 643 **
 644 ** Description...
 645 */
 646 static int SQLITE_TCLAPI f5tCreateFunction(
 647   void * clientData,
 648   Tcl_Interp *interp,
 649   int objc,
 650   Tcl_Obj *CONST objv[]
 651 ){
 652   char *zName;
 653   Tcl_Obj *pScript;
 654   sqlite3 *db = 0;
 655   fts5_api *pApi = 0;
 656   F5tFunction *pCtx = 0;
 657   int rc;
 658
 659   if( objc!=4 ){
 660     Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
 661     return TCL_ERROR;
 662   }
 663   if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
 664
 665   zName = Tcl_GetString(objv[2]);
 666   pScript = objv[3];
 667   pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction));
 668   pCtx->interp = interp;
 669   pCtx->pScript = pScript;
 670   Tcl_IncrRefCount(pScript);
 671
 672   rc = pApi->xCreateFunction(
 673       pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
 674   );
 675   if( rc!=SQLITE_OK ){
 676     Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
 677     return TCL_ERROR;
 678   }
 679
 680   return TCL_OK;
 681 }
 682
 683 typedef struct F5tTokenizeCtx F5tTokenizeCtx;
 684 struct F5tTokenizeCtx {
 685   Tcl_Obj *pRet;
 686   int bSubst;
 687   const char *zInput;
 688 };
 689
 690 static int xTokenizeCb2(
 691   void *pCtx,
 692   int tflags,
 693   const char *zToken, int nToken,
 694   int iStart, int iEnd
 695 ){
 696   F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx;
 697   if( p->bSubst ){
 698     Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));
 699     Tcl_ListObjAppendElement(
 700         0, p->pRet, Tcl_NewStringObj(&p->zInput[iStart], iEnd-iStart)
 701     );
 702   }else{
 703     Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));
 704     Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iStart));
 705     Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iEnd));
 706   }
 707   return SQLITE_OK;
 708 }
 709
 710
 711 /*
 712 **      sqlite3_fts5_tokenize DB TOKENIZER TEXT
 713 **
 714 ** Description...
 715 */
 716 static int SQLITE_TCLAPI f5tTokenize(
 717   void * clientData,
 718   Tcl_Interp *interp,
 719   int objc,
 720   Tcl_Obj *CONST objv[]
 721 ){
 722   char *zText;
 723   int nText;
 724   sqlite3 *db = 0;
 725   fts5_api *pApi = 0;
 726   Fts5Tokenizer *pTok = 0;
 727   fts5_tokenizer tokenizer;
 728   Tcl_Obj *pRet = 0;
 729   void *pUserdata;
 730   int rc;
 731
 732   int nArg;
 733   const char **azArg;
 734   F5tTokenizeCtx ctx;
 735
 736   if( objc!=4 && objc!=5 ){
 737     Tcl_WrongNumArgs(interp, 1, objv, "?-subst? DB NAME TEXT");
 738     return TCL_ERROR;
 739   }
 740   if( objc==5 ){
 741     char *zOpt = Tcl_GetString(objv[1]);
 742     if( strcmp("-subst", zOpt) ){
 743       Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0);
 744       return TCL_ERROR;
 745     }
 746   }
 747   if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ) return TCL_ERROR;
 748   if( Tcl_SplitList(interp, Tcl_GetString(objv[objc-2]), &nArg, &azArg) ){
 749     return TCL_ERROR;
 750   }
 751   if( nArg==0 ){
 752     Tcl_AppendResult(interp, "no such tokenizer: ", 0);
 753     Tcl_Free((void*)azArg);
 754     return TCL_ERROR;
 755   }
 756   zText = Tcl_GetStringFromObj(objv[objc-1], &nText);
 757
 758   rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
 759   if( rc!=SQLITE_OK ){
 760     Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0);
 761     return TCL_ERROR;
 762   }
 763
 764   rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
 765   if( rc!=SQLITE_OK ){
 766     Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
 767     return TCL_ERROR;
 768   }
 769
 770   pRet = Tcl_NewObj();
 771   Tcl_IncrRefCount(pRet);
 772   ctx.bSubst = (objc==5);
 773   ctx.pRet = pRet;
 774   ctx.zInput = zText;
 775   rc = tokenizer.xTokenize(
 776       pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
 777   );
 778   tokenizer.xDelete(pTok);
 779   if( rc!=SQLITE_OK ){
 780     Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
 781     Tcl_DecrRefCount(pRet);
 782     return TCL_ERROR;
 783   }
 784
 785
 786   Tcl_Free((void*)azArg);
 787   Tcl_SetObjResult(interp, pRet);
 788   Tcl_DecrRefCount(pRet);
 789   return TCL_OK;
 790 }
 791
 792 /*************************************************************************
 793 ** Start of tokenizer wrapper.
 794 */
 795
 796 typedef struct F5tTokenizerContext F5tTokenizerContext;
 797 typedef struct F5tTokenizerCb F5tTokenizerCb;
 798 typedef struct F5tTokenizerModule F5tTokenizerModule;
 799 typedef struct F5tTokenizerInstance F5tTokenizerInstance;
 800
 801 struct F5tTokenizerContext {
 802   void *pCtx;
 803   int (*xToken)(void*, int, const char*, int, int, int);
 804 };
 805
 806 struct F5tTokenizerModule {
 807   Tcl_Interp *interp;
 808   Tcl_Obj *pScript;
 809   F5tTokenizerContext *pContext;
 810 };
 811
 812 struct F5tTokenizerInstance {
 813   Tcl_Interp *interp;
 814   Tcl_Obj *pScript;
 815   F5tTokenizerContext *pContext;
 816 };
 817
 818 static int f5tTokenizerCreate(
 819   void *pCtx,
 820   const char **azArg,
 821   int nArg,
 822   Fts5Tokenizer **ppOut
 823 ){
 824   F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
 825   Tcl_Obj *pEval;
 826   int rc = TCL_OK;
 827   int i;
 828
 829   pEval = Tcl_DuplicateObj(pMod->pScript);
 830   Tcl_IncrRefCount(pEval);
 831   for(i=0; rc==TCL_OK && i<nArg; i++){
 832     Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1);
 833     rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj);
 834   }
 835
 836   if( rc==TCL_OK ){
 837     rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY);
 838   }
 839   Tcl_DecrRefCount(pEval);
 840
 841   if( rc==TCL_OK ){
 842     F5tTokenizerInstance *pInst;
 843     pInst = (F5tTokenizerInstance*)ckalloc(sizeof(F5tTokenizerInstance));
 844     memset(pInst, 0, sizeof(F5tTokenizerInstance));
 845     pInst->interp = pMod->interp;
 846     pInst->pScript = Tcl_GetObjResult(pMod->interp);
 847     pInst->pContext = pMod->pContext;
 848     Tcl_IncrRefCount(pInst->pScript);
 849     *ppOut = (Fts5Tokenizer*)pInst;
 850   }
 851
 852   return rc;
 853 }
 854
 855
 856 static void f5tTokenizerDelete(Fts5Tokenizer *p){
 857   F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
 858   Tcl_DecrRefCount(pInst->pScript);
 859   ckfree((char *)pInst);
 860 }
 861
 862 static int f5tTokenizerTokenize(
 863   Fts5Tokenizer *p,
 864   void *pCtx,
 865   int flags,
 866   const char *pText, int nText,
 867   int (*xToken)(void*, int, const char*, int, int, int)
 868 ){
 869   F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
 870   void *pOldCtx;
 871   int (*xOldToken)(void*, int, const char*, int, int, int);
 872   Tcl_Obj *pEval;
 873   int rc;
 874   const char *zFlags;
 875
 876   pOldCtx = pInst->pContext->pCtx;
 877   xOldToken = pInst->pContext->xToken;
 878
 879   pInst->pContext->pCtx = pCtx;
 880   pInst->pContext->xToken = xToken;
 881
 882   assert(
 883       flags==FTS5_TOKENIZE_DOCUMENT
 884    || flags==FTS5_TOKENIZE_AUX
 885    || flags==FTS5_TOKENIZE_QUERY
 886    || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)
 887   );
 888   pEval = Tcl_DuplicateObj(pInst->pScript);
 889   Tcl_IncrRefCount(pEval);
 890   switch( flags ){
 891     case FTS5_TOKENIZE_DOCUMENT:
 892       zFlags = "document";
 893       break;
 894     case FTS5_TOKENIZE_AUX:
 895       zFlags = "aux";
 896       break;
 897     case FTS5_TOKENIZE_QUERY:
 898       zFlags = "query";
 899       break;
 900     case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY):
 901       zFlags = "prefixquery";
 902       break;
 903     default:
 904       assert( 0 );
 905       zFlags = "invalid";
 906       break;
 907   }
 908
 909   Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1));
 910   Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText));
 911   rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
 912   Tcl_DecrRefCount(pEval);
 913
 914   pInst->pContext->pCtx = pOldCtx;
 915   pInst->pContext->xToken = xOldToken;
 916   return rc;
 917 }
 918
 919 /*
 920 ** sqlite3_fts5_token ?-colocated? TEXT START END
 921 */
 922 static int SQLITE_TCLAPI f5tTokenizerReturn(
 923   void * clientData,
 924   Tcl_Interp *interp,
 925   int objc,
 926   Tcl_Obj *CONST objv[]
 927 ){
 928   F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
 929   int iStart;
 930   int iEnd;
 931   int nToken;
 932   int tflags = 0;
 933   char *zToken;
 934   int rc;
 935
 936   if( objc==5 ){
 937     int nArg;
 938     char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
 939     if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
 940       tflags |= FTS5_TOKEN_COLOCATED;
 941     }else{
 942       goto usage;
 943     }
 944   }else if( objc!=4 ){
 945     goto usage;
 946   }
 947
 948   zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken);
 949   if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart)
 950    || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd)
 951   ){
 952     return TCL_ERROR;
 953   }
 954
 955   if( p->xToken==0 ){
 956     Tcl_AppendResult(interp,
 957         "sqlite3_fts5_token may only be used by tokenizer callback", 0
 958     );
 959     return TCL_ERROR;
 960   }
 961
 962   rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
 963   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
 964   return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 965
 966  usage:
 967   Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
 968   return TCL_ERROR;
 969 }
 970
 971 static void f5tDelTokenizer(void *pCtx){
 972   F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
 973   Tcl_DecrRefCount(pMod->pScript);
 974   ckfree((char *)pMod);
 975 }
 976
 977 /*
 978 **      sqlite3_fts5_create_tokenizer DB NAME SCRIPT
 979 **
 980 ** Register a tokenizer named NAME implemented by script SCRIPT. When
 981 ** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer
 982 ** arguments are appended to SCRIPT and the result executed.
 983 **
 984 ** The value returned by (SCRIPT + args) is itself a tcl script. This
 985 ** script - call it SCRIPT2 - is executed to tokenize text using the
 986 ** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize
 987 ** text SCRIPT2 is invoked with a single argument appended to it - the
 988 ** text to tokenize.
 989 **
 990 ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
 991 ** token within the tokenized text.
 992 */
 993 static int SQLITE_TCLAPI f5tCreateTokenizer(
 994   ClientData clientData,
 995   Tcl_Interp *interp,
 996   int objc,
 997   Tcl_Obj *CONST objv[]
 998 ){
 999   F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData;
1000   sqlite3 *db;
1001   fts5_api *pApi;
1002   char *zName;
1003   Tcl_Obj *pScript;
1004   fts5_tokenizer t;
1005   F5tTokenizerModule *pMod;
1006   int rc;
1007
1008   if( objc!=4 ){
1009     Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
1010     return TCL_ERROR;
1011   }
1012   if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
1013     return TCL_ERROR;
1014   }
1015   zName = Tcl_GetString(objv[2]);
1016   pScript = objv[3];
1017
1018   t.xCreate = f5tTokenizerCreate;
1019   t.xTokenize = f5tTokenizerTokenize;
1020   t.xDelete = f5tTokenizerDelete;
1021
1022   pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
1023   pMod->interp = interp;
1024   pMod->pScript = pScript;
1025   pMod->pContext = pContext;
1026   Tcl_IncrRefCount(pScript);
1027   rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
1028   if( rc!=SQLITE_OK ){
1029     Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
1030     return TCL_ERROR;
1031   }
1032
1033   return TCL_OK;
1034 }
1035
1036 static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
1037   ckfree(clientData);
1038 }
1039
1040 /*
1041 **      sqlite3_fts5_may_be_corrupt BOOLEAN
1042 **
1043 ** Set or clear the global "may-be-corrupt" flag. Return the old value.
1044 */
1045 static int SQLITE_TCLAPI f5tMayBeCorrupt(
1046   void * clientData,
1047   Tcl_Interp *interp,
1048   int objc,
1049   Tcl_Obj *CONST objv[]
1050 ){
1051 #ifdef SQLITE_DEBUG
1052   int bOld = sqlite3_fts5_may_be_corrupt;
1053
1054   if( objc!=2 && objc!=1 ){
1055     Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
1056     return TCL_ERROR;
1057   }
1058   if( objc==2 ){
1059     int bNew;
1060     if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
1061     sqlite3_fts5_may_be_corrupt = bNew;
1062   }
1063
1064   Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
1065 #endif
1066   return TCL_OK;
1067 }
1068
1069
1070 static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){
1071   int i;
1072   unsigned int h = 13;
1073   for(i=n-1; i>=0; i--){
1074     h = (h << 3) ^ h ^ p[i];
1075   }
1076   return (h % nSlot);
1077 }
1078
1079 static int SQLITE_TCLAPI f5tTokenHash(
1080   void * clientData,
1081   Tcl_Interp *interp,
1082   int objc,
1083   Tcl_Obj *CONST objv[]
1084 ){
1085   char *z;
1086   int n;
1087   unsigned int iVal;
1088   int nSlot;
1089
1090   if( objc!=3 ){
1091     Tcl_WrongNumArgs(interp, 1, objv, "NSLOT TOKEN");
1092     return TCL_ERROR;
1093   }
1094   if( Tcl_GetIntFromObj(interp, objv[1], &nSlot) ){
1095     return TCL_ERROR;
1096   }
1097   z = Tcl_GetStringFromObj(objv[2], &n);
1098
1099   iVal = f5t_fts5HashKey(nSlot, z, n);
1100   Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
1101   return TCL_OK;
1102 }
1103
1104 static int SQLITE_TCLAPI f5tRegisterMatchinfo(
1105   void * clientData,
1106   Tcl_Interp *interp,
1107   int objc,
1108   Tcl_Obj *CONST objv[]
1109 ){
1110   int rc;
1111   sqlite3 *db = 0;
1112
1113   if( objc!=2 ){
1114     Tcl_WrongNumArgs(interp, 1, objv, "DB");
1115     return TCL_ERROR;
1116   }
1117   if( f5tDbPointer(interp, objv[1], &db) ){
1118     return TCL_ERROR;
1119   }
1120
1121   rc = sqlite3Fts5TestRegisterMatchinfo(db);
1122   if( rc!=SQLITE_OK ){
1123     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
1124     return TCL_ERROR;
1125   }
1126   return TCL_OK;
1127 }
1128
1129 static int SQLITE_TCLAPI f5tRegisterTok(
1130   void * clientData,
1131   Tcl_Interp *interp,
1132   int objc,
1133   Tcl_Obj *CONST objv[]
1134 ){
1135   int rc;
1136   sqlite3 *db = 0;
1137   fts5_api *pApi = 0;
1138
1139   if( objc!=2 ){
1140     Tcl_WrongNumArgs(interp, 1, objv, "DB");
1141     return TCL_ERROR;
1142   }
1143   if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
1144     return TCL_ERROR;
1145   }
1146
1147   rc = sqlite3Fts5TestRegisterTok(db, pApi);
1148   if( rc!=SQLITE_OK ){
1149     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
1150     return TCL_ERROR;
1151   }
1152   return TCL_OK;
1153 }
1154
1155 typedef struct OriginTextCtx OriginTextCtx;
1156 struct OriginTextCtx {
1157   sqlite3 *db;
1158   fts5_api *pApi;
1159 };
1160
1161 typedef struct OriginTextTokenizer OriginTextTokenizer;
1162 struct OriginTextTokenizer {
1163   Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
1164   fts5_tokenizer tokapi;          /* API implementation for pTok */
1165 };
1166
1167 /*
1168 ** Delete the OriginTextCtx object indicated by the only argument.
1169 */
1170 static void f5tOrigintextTokenizerDelete(void *pCtx){
1171   OriginTextCtx *p = (OriginTextCtx*)pCtx;
1172   ckfree((char*)p);
1173 }
1174
1175 static int f5tOrigintextCreate(
1176   void *pCtx,
1177   const char **azArg,
1178   int nArg,
1179   Fts5Tokenizer **ppOut
1180 ){
1181   OriginTextCtx *p = (OriginTextCtx*)pCtx;
1182   OriginTextTokenizer *pTok = 0;
1183   void *pTokCtx = 0;
1184   int rc = SQLITE_OK;
1185
1186   pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
1187   if( pTok==0 ){
1188     rc = SQLITE_NOMEM;
1189   }else if( nArg<1 ){
1190     rc = SQLITE_ERROR;
1191   }else{
1192     /* Locate the underlying tokenizer */
1193     rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
1194   }
1195
1196   /* Create the new tokenizer instance */
1197   if( rc==SQLITE_OK ){
1198     rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
1199   }
1200
1201   if( rc!=SQLITE_OK ){
1202     sqlite3_free(pTok);
1203     pTok = 0;
1204   }
1205   *ppOut = (Fts5Tokenizer*)pTok;
1206   return rc;
1207 }
1208
1209 static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
1210   OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
1211   if( p->pTok ){
1212     p->tokapi.xDelete(p->pTok);
1213   }
1214   sqlite3_free(p);
1215 }
1216
1217 typedef struct OriginTextCb OriginTextCb;
1218 struct OriginTextCb {
1219   void *pCtx;
1220   const char *pText;
1221   int nText;
1222   int (*xToken)(void *, int, const char *, int, int, int);
1223
1224   char *aBuf;                     /* Buffer to use */
1225   int nBuf;                       /* Allocated size of aBuf[] */
1226 };
1227
1228 static int xOriginToken(
1229   void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
1230   int tflags,                     /* Mask of FTS5_TOKEN_* flags */
1231   const char *pToken,             /* Pointer to buffer containing token */
1232   int nToken,                     /* Size of token in bytes */
1233   int iStart,                     /* Byte offset of token within input text */
1234   int iEnd                        /* Byte offset of end of token within input */
1235 ){
1236   OriginTextCb *p = (OriginTextCb*)pCtx;
1237   int ret = 0;
1238
1239   if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
1240     /* Token exactly matches document text. Pass it through as is. */
1241     ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
1242   }else{
1243     int nReq = nToken + 1 + (iEnd-iStart);
1244     if( nReq>p->nBuf ){
1245       sqlite3_free(p->aBuf);
1246       p->aBuf = sqlite3_malloc(nReq*2);
1247       if( p->aBuf==0 ) return SQLITE_NOMEM;
1248       p->nBuf = nReq*2;
1249     }
1250
1251     memcpy(p->aBuf, pToken, nToken);
1252     p->aBuf[nToken] = '\0';
1253     memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
1254     ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
1255   }
1256
1257   return ret;
1258 }
1259
1260
1261 static int f5tOrigintextTokenize(
1262   Fts5Tokenizer *pTokenizer,
1263   void *pCtx,
1264   int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
1265   const char *pText, int nText,
1266   int (*xToken)(void *, int, const char *, int, int, int)
1267 ){
1268   OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
1269   OriginTextCb cb;
1270   int ret;
1271
1272   memset(&cb, 0, sizeof(cb));
1273   cb.pCtx = pCtx;
1274   cb.pText = pText;
1275   cb.nText = nText;
1276   cb.xToken = xToken;
1277
1278   ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
1279   sqlite3_free(cb.aBuf);
1280   return ret;
1281 }
1282
1283 /*
1284 **      sqlite3_fts5_register_origintext DB
1285 **
1286 ** Description...
1287 */
1288 static int SQLITE_TCLAPI f5tRegisterOriginText(
1289   void * clientData,
1290   Tcl_Interp *interp,
1291   int objc,
1292   Tcl_Obj *CONST objv[]
1293 ){
1294   sqlite3 *db = 0;
1295   fts5_api *pApi = 0;
1296   int rc;
1297   fts5_tokenizer tok = {0, 0, 0};
1298   OriginTextCtx *pCtx = 0;
1299
1300   if( objc!=2 ){
1301     Tcl_WrongNumArgs(interp, 1, objv, "DB");
1302     return TCL_ERROR;
1303   }
1304   if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
1305
1306   pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
1307   pCtx->db = db;
1308   pCtx->pApi = pApi;
1309
1310   tok.xCreate = f5tOrigintextCreate;
1311   tok.xDelete = f5tOrigintextDelete;
1312   tok.xTokenize = f5tOrigintextTokenize;
1313   rc = pApi->xCreateTokenizer(
1314       pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
1315   );
1316
1317   Tcl_ResetResult(interp);
1318   if( rc!=SQLITE_OK ){
1319     Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
1320     return TCL_ERROR;
1321   }
1322   return TCL_OK;
1323 }
1324
1325 /*
1326 ** Entry point.
1327 */
1328 int Fts5tcl_Init(Tcl_Interp *interp){
1329   static struct Cmd {
1330     char *zName;
1331     Tcl_ObjCmdProc *xProc;
1332     int bTokenizeCtx;
1333   } aCmd[] = {
1334     { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
1335     { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
1336     { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
1337     { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
1338     { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
1339     { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
1340     { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
1341     { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
1342     { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
1343   };
1344   int i;
1345   F5tTokenizerContext *pContext;
1346
1347   pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext));
1348   memset(pContext, 0, sizeof(*pContext));
1349
1350   for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
1351     struct Cmd *p = &aCmd[i];
1352     void *pCtx = 0;
1353     if( p->bTokenizeCtx ) pCtx = (void*)pContext;
1354     Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
1355   }
1356
1357   return TCL_OK;
1358 }
1359 #else  /* SQLITE_ENABLE_FTS5 */
1360 int Fts5tcl_Init(Tcl_Interp *interp){
1361   return TCL_OK;
1362 }
1363 #endif /* SQLITE_ENABLE_FTS5 */
1364 #endif /* SQLITE_TEST */