ext/fts3/fts3_tokenizer.h

   1 /*
   2 ** 2006 July 10
   3 **
   4 ** The author disclaims copyright to this source code.
   5 **
   6 *************************************************************************
   7 ** Defines the interface to tokenizers used by fulltext-search.  There
   8 ** are three basic components:
   9 **
  10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer
  11 ** interface functions.  This is essentially the class structure for
  12 ** tokenizers.
  13 **
  14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
  15 ** including customization information defined at creation time.
  16 **
  17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
  18 ** tokens from a particular input.
  19 */
  20 #ifndef _FTS3_TOKENIZER_H_
  21 #define _FTS3_TOKENIZER_H_
  22
  23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
  24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then
  25 ** we will need a way to register the API consistently.
  26 */
  27 #include "sqlite3.h"
  28
  29 /*
  30 ** Structures used by the tokenizer interface. When a new tokenizer
  31 ** implementation is registered, the caller provides a pointer to
  32 ** an sqlite3_tokenizer_module containing pointers to the callback
  33 ** functions that make up an implementation.
  34 **
  35 ** When an fts3 table is created, it passes any arguments passed to
  36 ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
  37 ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
  38 ** implementation. The xCreate() function in turn returns an
  39 ** sqlite3_tokenizer structure representing the specific tokenizer to
  40 ** be used for the fts3 table (customized by the tokenizer clause arguments).
  41 **
  42 ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
  43 ** method is called. It returns an sqlite3_tokenizer_cursor object
  44 ** that may be used to tokenize a specific input buffer based on
  45 ** the tokenization rules supplied by a specific sqlite3_tokenizer
  46 ** object.
  47 */
  48 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
  49 typedef struct sqlite3_tokenizer sqlite3_tokenizer;
  50 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
  51
  52 struct sqlite3_tokenizer_module {
  53
  54   /*
  55   ** Structure version. Should always be set to 0 or 1.
  56   */
  57   int iVersion;
  58
  59   /*
  60   ** Create a new tokenizer. The values in the argv[] array are the
  61   ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
  62   ** TABLE statement that created the fts3 table. For example, if
  63   ** the following SQL is executed:
  64   **
  65   **   CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
  66   **
  67   ** then argc is set to 2, and the argv[] array contains pointers
  68   ** to the strings "arg1" and "arg2".
  69   **
  70   ** This method should return either SQLITE_OK (0), or an SQLite error
  71   ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
  72   ** to point at the newly created tokenizer structure. The generic
  73   ** sqlite3_tokenizer.pModule variable should not be initialized by
  74   ** this callback. The caller will do so.
  75   */
  76   int (*xCreate)(
  77     int argc,                           /* Size of argv array */
  78     const char *const*argv,             /* Tokenizer argument strings */
  79     sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
  80   );
  81
  82   /*
  83   ** Destroy an existing tokenizer. The fts3 module calls this method
  84   ** exactly once for each successful call to xCreate().
  85   */
  86   int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
  87
  88   /*
  89   ** Create a tokenizer cursor to tokenize an input buffer. The caller
  90   ** is responsible for ensuring that the input buffer remains valid
  91   ** until the cursor is closed (using the xClose() method).
  92   */
  93   int (*xOpen)(
  94     sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
  95     const char *pInput, int nBytes,      /* Input buffer */
  96     sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
  97   );
  98
  99   /*
 100   ** Destroy an existing tokenizer cursor. The fts3 module calls this
 101   ** method exactly once for each successful call to xOpen().
 102   */
 103   int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
 104
 105   /*
 106   ** Retrieve the next token from the tokenizer cursor pCursor. This
 107   ** method should either return SQLITE_OK and set the values of the
 108   ** "OUT" variables identified below, or SQLITE_DONE to indicate that
 109   ** the end of the buffer has been reached, or an SQLite error code.
 110   **
 111   ** *ppToken should be set to point at a buffer containing the
 112   ** normalized version of the token (i.e. after any case-folding and/or
 113   ** stemming has been performed). *pnBytes should be set to the length
 114   ** of this buffer in bytes. The input text that generated the token is
 115   ** identified by the byte offsets returned in *piStartOffset and
 116   ** *piEndOffset. *piStartOffset should be set to the index of the first
 117   ** byte of the token in the input buffer. *piEndOffset should be set
 118   ** to the index of the first byte just past the end of the token in
 119   ** the input buffer.
 120   **
 121   ** The buffer *ppToken is set to point at is managed by the tokenizer
 122   ** implementation. It is only required to be valid until the next call
 123   ** to xNext() or xClose().
 124   */
 125   /* TODO(shess) current implementation requires pInput to be
 126   ** nul-terminated.  This should either be fixed, or pInput/nBytes
 127   ** should be converted to zInput.
 128   */
 129   int (*xNext)(
 130     sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
 131     const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
 132     int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
 133     int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
 134     int *piPosition      /* OUT: Number of tokens returned before this one */
 135   );
 136
 137   /***********************************************************************
 138   ** Methods below this point are only available if iVersion>=1.
 139   */
 140
 141   /*
 142   ** Configure the language id of a tokenizer cursor.
 143   */
 144   int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
 145 };
 146
 147 struct sqlite3_tokenizer {
 148   const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
 149   /* Tokenizer implementations will typically add additional fields */
 150 };
 151
 152 struct sqlite3_tokenizer_cursor {
 153   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
 154   /* Tokenizer implementations will typically add additional fields */
 155 };
 156
 157 int fts3_global_term_cnt(int iTerm, int iCol);
 158 int fts3_term_cnt(int iTerm, int iCol);
 159
 160
 161 #endif /* _FTS3_TOKENIZER_H_ */