ext/fts1/simple_tokenizer.c

   1 /*
   2 ** The author disclaims copyright to this source code.
   3 **
   4 *************************************************************************
   5 ** Implementation of the "simple" full-text-search tokenizer.
   6 */
   7
   8 #include <assert.h>
   9 #if !defined(__APPLE__)
  10 #include <malloc.h>
  11 #else
  12 #include <stdlib.h>
  13 #endif
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <ctype.h>
  17
  18 #include "tokenizer.h"
  19
  20 /* Duplicate a string; the caller must free() the returned string.
  21  * (We don't use strdup() since it's not part of the standard C library and
  22  * may not be available everywhere.) */
  23 /* TODO(shess) Copied from fulltext.c, consider util.c for such
  24 ** things. */
  25 static char *string_dup(const char *s){
  26   char *str = malloc(strlen(s) + 1);
  27   strcpy(str, s);
  28   return str;
  29 }
  30
  31 typedef struct simple_tokenizer {
  32   sqlite3_tokenizer base;
  33   const char *zDelim;          /* token delimiters */
  34 } simple_tokenizer;
  35
  36 typedef struct simple_tokenizer_cursor {
  37   sqlite3_tokenizer_cursor base;
  38   const char *pInput;          /* input we are tokenizing */
  39   int nBytes;                  /* size of the input */
  40   const char *pCurrent;        /* current position in pInput */
  41   int iToken;                  /* index of next token to be returned */
  42   char *zToken;                /* storage for current token */
  43   int nTokenBytes;             /* actual size of current token */
  44   int nTokenAllocated;         /* space allocated to zToken buffer */
  45 } simple_tokenizer_cursor;
  46
  47 static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
  48
  49 static int simpleCreate(
  50   int argc, const char **argv,
  51   sqlite3_tokenizer **ppTokenizer
  52 ){
  53   simple_tokenizer *t;
  54
  55   t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
  56   /* TODO(shess) Delimiters need to remain the same from run to run,
  57   ** else we need to reindex.  One solution would be a meta-table to
  58   ** track such information in the database, then we'd only want this
  59   ** information on the initial create.
  60   */
  61   if( argc>1 ){
  62     t->zDelim = string_dup(argv[1]);
  63   } else {
  64     /* Build a string excluding alphanumeric ASCII characters */
  65     char zDelim[0x80];               /* nul-terminated, so nul not a member */
  66     int i, j;
  67     for(i=1, j=0; i<0x80; i++){
  68       if( !isalnum(i) ){
  69         zDelim[j++] = i;
  70       }
  71     }
  72     zDelim[j++] = '\0';
  73     assert( j<=sizeof(zDelim) );
  74     t->zDelim = string_dup(zDelim);
  75   }
  76
  77   *ppTokenizer = &t->base;
  78   return SQLITE_OK;
  79 }
  80
  81 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
  82   simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
  83
  84   free((void *) t->zDelim);
  85   free(t);
  86
  87   return SQLITE_OK;
  88 }
  89
  90 static int simpleOpen(
  91   sqlite3_tokenizer *pTokenizer,
  92   const char *pInput, int nBytes,
  93   sqlite3_tokenizer_cursor **ppCursor
  94 ){
  95   simple_tokenizer_cursor *c;
  96
  97   c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
  98   c->pInput = pInput;
  99   c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
 100   c->pCurrent = c->pInput;        /* start tokenizing at the beginning */
 101   c->iToken = 0;
 102   c->zToken = NULL;               /* no space allocated, yet. */
 103   c->nTokenBytes = 0;
 104   c->nTokenAllocated = 0;
 105
 106   *ppCursor = &c->base;
 107   return SQLITE_OK;
 108 }
 109
 110 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
 111   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 112
 113   if( NULL!=c->zToken ){
 114     free(c->zToken);
 115   }
 116   free(c);
 117
 118   return SQLITE_OK;
 119 }
 120
 121 static int simpleNext(
 122   sqlite3_tokenizer_cursor *pCursor,
 123   const char **ppToken, int *pnBytes,
 124   int *piStartOffset, int *piEndOffset, int *piPosition
 125 ){
 126   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 127   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
 128   int ii;
 129
 130   while( c->pCurrent-c->pInput<c->nBytes ){
 131     int n = (int) strcspn(c->pCurrent, t->zDelim);
 132     if( n>0 ){
 133       if( n+1>c->nTokenAllocated ){
 134         c->zToken = realloc(c->zToken, n+1);
 135       }
 136       for(ii=0; ii<n; ii++){
 137         /* TODO(shess) This needs expansion to handle UTF-8
 138         ** case-insensitivity.
 139         */
 140         char ch = c->pCurrent[ii];
 141         c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
 142       }
 143       c->zToken[n] = '\0';
 144       *ppToken = c->zToken;
 145       *pnBytes = n;
 146       *piStartOffset = (int) (c->pCurrent-c->pInput);
 147       *piEndOffset = *piStartOffset+n;
 148       *piPosition = c->iToken++;
 149       c->pCurrent += n + 1;
 150
 151       return SQLITE_OK;
 152     }
 153     c->pCurrent += n + 1;
 154     /* TODO(shess) could strspn() to skip delimiters en masse.  Needs
 155     ** to happen in two places, though, which is annoying.
 156     */
 157   }
 158   return SQLITE_DONE;
 159 }
 160
 161 static sqlite3_tokenizer_module simpleTokenizerModule = {
 162   0,
 163   simpleCreate,
 164   simpleDestroy,
 165   simpleOpen,
 166   simpleClose,
 167   simpleNext,
 168 };
 169
 170 void get_simple_tokenizer_module(
 171   sqlite3_tokenizer_module **ppModule
 172 ){
 173   *ppModule = &simpleTokenizerModule;
 174 }