third_party/sqlite/sqlite-src-3070603/ext/fts3/fts3_tokenizer1.c

   1 /*
   2 ** 2006 Oct 10
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** Implementation of the "simple" full-text-search tokenizer.
  14 */
  15
  16 /*
  17 ** The code in this file is only compiled if:
  18 **
  19 **     * The FTS3 module is being built as an extension
  20 **       (in which case SQLITE_CORE is not defined), or
  21 **
  22 **     * The FTS3 module is being built into the core of
  23 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
  24 */
  25 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
  26
  27 #include "fts3Int.h"
  28
  29 #include <assert.h>
  30 #include <stdlib.h>
  31 #include <stdio.h>
  32 #include <string.h>
  33
  34 #include "fts3_tokenizer.h"
  35
  36 typedef struct simple_tokenizer {
  37   sqlite3_tokenizer base;
  38   char delim[128];             /* flag ASCII delimiters */
  39 } simple_tokenizer;
  40
  41 typedef struct simple_tokenizer_cursor {
  42   sqlite3_tokenizer_cursor base;
  43   const char *pInput;          /* input we are tokenizing */
  44   int nBytes;                  /* size of the input */
  45   int iOffset;                 /* current position in pInput */
  46   int iToken;                  /* index of next token to be returned */
  47   char *pToken;                /* storage for current token */
  48   int nTokenAllocated;         /* space allocated to zToken buffer */
  49 } simple_tokenizer_cursor;
  50
  51
  52 static int simpleDelim(simple_tokenizer *t, unsigned char c){
  53   return c<0x80 && t->delim[c];
  54 }
  55 static int fts3_isalnum(int x){
  56   return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
  57 }
  58
  59 /*
  60 ** Create a new tokenizer instance.
  61 */
  62 static int simpleCreate(
  63   int argc, const char * const *argv,
  64   sqlite3_tokenizer **ppTokenizer
  65 ){
  66   simple_tokenizer *t;
  67
  68   t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
  69   if( t==NULL ) return SQLITE_NOMEM;
  70   memset(t, 0, sizeof(*t));
  71
  72   /* TODO(shess) Delimiters need to remain the same from run to run,
  73   ** else we need to reindex.  One solution would be a meta-table to
  74   ** track such information in the database, then we'd only want this
  75   ** information on the initial create.
  76   */
  77   if( argc>1 ){
  78     int i, n = (int)strlen(argv[1]);
  79     for(i=0; i<n; i++){
  80       unsigned char ch = argv[1][i];
  81       /* We explicitly don't support UTF-8 delimiters for now. */
  82       if( ch>=0x80 ){
  83         sqlite3_free(t);
  84         return SQLITE_ERROR;
  85       }
  86       t->delim[ch] = 1;
  87     }
  88   } else {
  89     /* Mark non-alphanumeric ASCII characters as delimiters */
  90     int i;
  91     for(i=1; i<0x80; i++){
  92       t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
  93     }
  94   }
  95
  96   *ppTokenizer = &t->base;
  97   return SQLITE_OK;
  98 }
  99
 100 /*
 101 ** Destroy a tokenizer
 102 */
 103 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
 104   sqlite3_free(pTokenizer);
 105   return SQLITE_OK;
 106 }
 107
 108 /*
 109 ** Prepare to begin tokenizing a particular string.  The input
 110 ** string to be tokenized is pInput[0..nBytes-1].  A cursor
 111 ** used to incrementally tokenize this string is returned in
 112 ** *ppCursor.
 113 */
 114 static int simpleOpen(
 115   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
 116   const char *pInput, int nBytes,        /* String to be tokenized */
 117   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
 118 ){
 119   simple_tokenizer_cursor *c;
 120
 121   UNUSED_PARAMETER(pTokenizer);
 122
 123   c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
 124   if( c==NULL ) return SQLITE_NOMEM;
 125
 126   c->pInput = pInput;
 127   if( pInput==0 ){
 128     c->nBytes = 0;
 129   }else if( nBytes<0 ){
 130     c->nBytes = (int)strlen(pInput);
 131   }else{
 132     c->nBytes = nBytes;
 133   }
 134   c->iOffset = 0;                 /* start tokenizing at the beginning */
 135   c->iToken = 0;
 136   c->pToken = NULL;               /* no space allocated, yet. */
 137   c->nTokenAllocated = 0;
 138
 139   *ppCursor = &c->base;
 140   return SQLITE_OK;
 141 }
 142
 143 /*
 144 ** Close a tokenization cursor previously opened by a call to
 145 ** simpleOpen() above.
 146 */
 147 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
 148   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 149   sqlite3_free(c->pToken);
 150   sqlite3_free(c);
 151   return SQLITE_OK;
 152 }
 153
 154 /*
 155 ** Extract the next token from a tokenization cursor.  The cursor must
 156 ** have been opened by a prior call to simpleOpen().
 157 */
 158 static int simpleNext(
 159   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
 160   const char **ppToken,               /* OUT: *ppToken is the token text */
 161   int *pnBytes,                       /* OUT: Number of bytes in token */
 162   int *piStartOffset,                 /* OUT: Starting offset of token */
 163   int *piEndOffset,                   /* OUT: Ending offset of token */
 164   int *piPosition                     /* OUT: Position integer of token */
 165 ){
 166   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 167   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
 168   unsigned char *p = (unsigned char *)c->pInput;
 169
 170   while( c->iOffset<c->nBytes ){
 171     int iStartOffset;
 172
 173     /* Scan past delimiter characters */
 174     while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
 175       c->iOffset++;
 176     }
 177
 178     /* Count non-delimiter characters. */
 179     iStartOffset = c->iOffset;
 180     while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
 181       c->iOffset++;
 182     }
 183
 184     if( c->iOffset>iStartOffset ){
 185       int i, n = c->iOffset-iStartOffset;
 186       if( n>c->nTokenAllocated ){
 187         char *pNew;
 188         c->nTokenAllocated = n+20;
 189         pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
 190         if( !pNew ) return SQLITE_NOMEM;
 191         c->pToken = pNew;
 192       }
 193       for(i=0; i<n; i++){
 194         /* TODO(shess) This needs expansion to handle UTF-8
 195         ** case-insensitivity.
 196         */
 197         unsigned char ch = p[iStartOffset+i];
 198         c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
 199       }
 200       *ppToken = c->pToken;
 201       *pnBytes = n;
 202       *piStartOffset = iStartOffset;
 203       *piEndOffset = c->iOffset;
 204       *piPosition = c->iToken++;
 205
 206       return SQLITE_OK;
 207     }
 208   }
 209   return SQLITE_DONE;
 210 }
 211
 212 /*
 213 ** The set of routines that implement the simple tokenizer
 214 */
 215 static const sqlite3_tokenizer_module simpleTokenizerModule = {
 216   0,
 217   simpleCreate,
 218   simpleDestroy,
 219   simpleOpen,
 220   simpleClose,
 221   simpleNext,
 222 };
 223
 224 /*
 225 ** Allocate a new simple tokenizer.  Return a pointer to the new
 226 ** tokenizer in *ppModule
 227 */
 228 void sqlite3Fts3SimpleTokenizerModule(
 229   sqlite3_tokenizer_module const**ppModule
 230 ){
 231   *ppModule = &simpleTokenizerModule;
 232 }
 233
 234 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */