dlls/msi/tokenize.c

   1 /*
   2 ** 2001 September 15
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 *************************************************************************
  12 ** An tokenizer for SQL
  13 **
  14 ** This file contains C code that splits an SQL input string up into
  15 ** individual tokens and sends those tokens one-by-one over to the
  16 ** parser for analysis.
  17 */
  18
  19 #include "winbase.h"
  20 #include <ctype.h>
  21 #include <stdlib.h>
  22 #include "wine/debug.h"
  23 #include "winnls.h"
  24 #include "query.h"
  25 #include "y.tab.h"
  26
  27 WINE_DEFAULT_DEBUG_CHANNEL(msi);
  28
  29 /*
  30 ** All the keywords of the SQL language are stored as in a hash
  31 ** table composed of instances of the following structure.
  32 */
  33 typedef struct Keyword Keyword;
  34 struct Keyword {
  35   const char *zName;             /* The keyword name */
  36   int tokenType;           /* The token value for this keyword */
  37 };
  38
  39 /*
  40 ** These are the keywords
  41 */
  42 static const Keyword aKeywordTable[] = {
  43   { "ABORT", TK_ABORT },
  44   { "AFTER", TK_AFTER },
  45   { "ALL", TK_ALL },
  46   { "AND", TK_AND },
  47   { "AS", TK_AS },
  48   { "ASC", TK_ASC },
  49   { "BEFORE", TK_BEFORE },
  50   { "BEGIN", TK_BEGIN },
  51   { "BETWEEN", TK_BETWEEN },
  52   { "BY", TK_BY },
  53   { "CASCADE", TK_CASCADE },
  54   { "CASE", TK_CASE },
  55   { "CHECK", TK_CHECK },
  56   { "CLUSTER", TK_CLUSTER },
  57   { "COLLATE", TK_COLLATE },
  58   { "COMMIT", TK_COMMIT },
  59   { "CONFLICT", TK_CONFLICT },
  60   { "CONSTRAINT", TK_CONSTRAINT },
  61   { "COPY", TK_COPY },
  62   { "CREATE", TK_CREATE },
  63   { "CROSS", TK_JOIN_KW },
  64   { "DEFAULT", TK_DEFAULT },
  65   { "DEFERRED", TK_DEFERRED },
  66   { "DEFERRABLE", TK_DEFERRABLE },
  67   { "DELETE", TK_DELETE },
  68   { "DELIMITERS", TK_DELIMITERS },
  69   { "DESC", TK_DESC },
  70   { "DISTINCT", TK_DISTINCT },
  71   { "DROP", TK_DROP },
  72   { "END", TK_END },
  73   { "EACH", TK_EACH },
  74   { "ELSE", TK_ELSE },
  75   { "EXCEPT", TK_EXCEPT },
  76   { "EXPLAIN", TK_EXPLAIN },
  77   { "FAIL", TK_FAIL },
  78   { "FOR", TK_FOR },
  79   { "FOREIGN", TK_FOREIGN },
  80   { "FROM", TK_FROM },
  81   { "FULL", TK_JOIN_KW },
  82   { "GLOB", TK_GLOB },
  83   { "GROUP", TK_GROUP },
  84   { "HAVING", TK_HAVING },
  85   { "IGNORE", TK_IGNORE },
  86   { "IMMEDIATE", TK_IMMEDIATE },
  87   { "IN", TK_IN },
  88   { "INDEX", TK_INDEX },
  89   { "INITIALLY", TK_INITIALLY },
  90   { "INNER", TK_JOIN_KW },
  91   { "INSERT", TK_INSERT },
  92   { "INSTEAD", TK_INSTEAD },
  93   { "INTERSECT", TK_INTERSECT },
  94   { "INTO", TK_INTO },
  95   { "IS", TK_IS },
  96   { "ISNULL", TK_ISNULL },
  97   { "JOIN", TK_JOIN },
  98   { "KEY", TK_KEY },
  99   { "LEFT", TK_JOIN_KW },
 100   { "LIKE", TK_LIKE },
 101   { "LIMIT", TK_LIMIT },
 102   { "MATCH", TK_MATCH },
 103   { "NATURAL", TK_JOIN_KW },
 104   { "NOT", TK_NOT },
 105   { "NOTNULL", TK_NOTNULL },
 106   { "NULL", TK_NULL },
 107   { "OF", TK_OF },
 108   { "OFFSET", TK_OFFSET },
 109   { "ON", TK_ON },
 110   { "OR", TK_OR },
 111   { "ORDER", TK_ORDER },
 112   { "OUTER", TK_JOIN_KW },
 113   { "PRAGMA", TK_PRAGMA },
 114   { "PRIMARY", TK_PRIMARY },
 115   { "RAISE", TK_RAISE },
 116   { "REFERENCES", TK_REFERENCES },
 117   { "REPLACE", TK_REPLACE },
 118   { "RESTRICT", TK_RESTRICT },
 119   { "RIGHT", TK_JOIN_KW },
 120   { "ROLLBACK", TK_ROLLBACK },
 121   { "ROW", TK_ROW },
 122   { "SELECT", TK_SELECT },
 123   { "SET", TK_SET },
 124   { "STATEMENT", TK_STATEMENT },
 125   { "TABLE", TK_TABLE },
 126   { "TEMP", TK_TEMP },
 127   { "TEMPORARY", TK_TEMP },
 128   { "THEN", TK_THEN },
 129   { "TRANSACTION", TK_TRANSACTION },
 130   { "TRIGGER", TK_TRIGGER },
 131   { "UNION", TK_UNION },
 132   { "UNIQUE", TK_UNIQUE },
 133   { "UPDATE", TK_UPDATE },
 134   { "USING", TK_USING },
 135   { "VACUUM", TK_VACUUM },
 136   { "VALUES", TK_VALUES },
 137   { "VIEW", TK_VIEW },
 138   { "WHEN", TK_WHEN },
 139   { "WHERE", TK_WHERE },
 140 };
 141
 142 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
 143
 144 /*
 145 ** This function looks up an identifier to determine if it is a
 146 ** keyword.  If it is a keyword, the token code of that keyword is
 147 ** returned.  If the input is not a keyword, TK_ID is returned.
 148 */
 149 int sqliteKeywordCode(const WCHAR *z, int n){
 150   int i, len;
 151   char buffer[0x10];
 152
 153   len = WideCharToMultiByte( CP_ACP, 0, z, n, buffer, sizeof buffer, NULL, NULL );
 154   for(i=0; i<KEYWORD_COUNT; i++)
 155   {
 156       if(memcmp(buffer, aKeywordTable[i].zName, len))
 157           continue;
 158       if(strlen(aKeywordTable[i].zName) == len )
 159           return aKeywordTable[i].tokenType;
 160   }
 161   return TK_ID;
 162 }
 163
 164
 165 /*
 166 ** If X is a character that can be used in an identifier then
 167 ** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
 168 **
 169 ** In this implementation, an identifier can be a string of
 170 ** alphabetic characters, digits, and "_" plus any character
 171 ** with the high-order bit set.  The latter rule means that
 172 ** any sequence of UTF-8 characters or characters taken from
 173 ** an extended ISO8859 character set can form an identifier.
 174 */
 175 static const char isIdChar[] = {
 176 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
 177     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
 178     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
 179     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
 180     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
 181     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
 182     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
 183     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
 184     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 185     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
 186     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
 187     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
 188     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
 189     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
 190     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
 191     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
 192     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
 193 };
 194
 195
 196 /*
 197 ** Return the length of the token that begins at z[0].  Return
 198 ** -1 if the token is (or might be) incomplete.  Store the token
 199 ** type in *tokenType before returning.
 200 */
 201 int sqliteGetToken(const WCHAR *z, int *tokenType){
 202   int i;
 203   switch( *z ){
 204     case ' ': case '\t': case '\n': case '\f': case '\r': {
 205       for(i=1; isspace(z[i]); i++){}
 206       *tokenType = TK_SPACE;
 207       return i;
 208     }
 209     case '-': {
 210       if( z[1]==0 ) return -1;
 211       if( z[1]=='-' ){
 212         for(i=2; z[i] && z[i]!='\n'; i++){}
 213         *tokenType = TK_COMMENT;
 214         return i;
 215       }
 216       *tokenType = TK_MINUS;
 217       return 1;
 218     }
 219     case '(': {
 220       if( z[1]=='+' && z[2]==')' ){
 221         *tokenType = TK_ORACLE_OUTER_JOIN;
 222         return 3;
 223       }else{
 224         *tokenType = TK_LP;
 225         return 1;
 226       }
 227     }
 228     case ')': {
 229       *tokenType = TK_RP;
 230       return 1;
 231     }
 232     case ';': {
 233       *tokenType = TK_SEMI;
 234       return 1;
 235     }
 236     case '+': {
 237       *tokenType = TK_PLUS;
 238       return 1;
 239     }
 240     case '*': {
 241       *tokenType = TK_STAR;
 242       return 1;
 243     }
 244     case '/': {
 245       if( z[1]!='*' || z[2]==0 ){
 246         *tokenType = TK_SLASH;
 247         return 1;
 248       }
 249       for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
 250       if( z[i] ) i++;
 251       *tokenType = TK_COMMENT;
 252       return i;
 253     }
 254     case '%': {
 255       *tokenType = TK_REM;
 256       return 1;
 257     }
 258     case '=': {
 259       *tokenType = TK_EQ;
 260       return 1 + (z[1]=='=');
 261     }
 262     case '<': {
 263       if( z[1]=='=' ){
 264         *tokenType = TK_LE;
 265         return 2;
 266       }else if( z[1]=='>' ){
 267         *tokenType = TK_NE;
 268         return 2;
 269       }else if( z[1]=='<' ){
 270         *tokenType = TK_LSHIFT;
 271         return 2;
 272       }else{
 273         *tokenType = TK_LT;
 274         return 1;
 275       }
 276     }
 277     case '>': {
 278       if( z[1]=='=' ){
 279         *tokenType = TK_GE;
 280         return 2;
 281       }else if( z[1]=='>' ){
 282         *tokenType = TK_RSHIFT;
 283         return 2;
 284       }else{
 285         *tokenType = TK_GT;
 286         return 1;
 287       }
 288     }
 289     case '!': {
 290       if( z[1]!='=' ){
 291         *tokenType = TK_ILLEGAL;
 292         return 2;
 293       }else{
 294         *tokenType = TK_NE;
 295         return 2;
 296       }
 297     }
 298     case '|': {
 299       if( z[1]!='|' ){
 300         *tokenType = TK_BITOR;
 301         return 1;
 302       }else{
 303         *tokenType = TK_CONCAT;
 304         return 2;
 305       }
 306     }
 307     case ',': {
 308       *tokenType = TK_COMMA;
 309       return 1;
 310     }
 311     case '&': {
 312       *tokenType = TK_BITAND;
 313       return 1;
 314     }
 315     case '~': {
 316       *tokenType = TK_BITNOT;
 317       return 1;
 318     }
 319     case '`': case '\'': case '"': {
 320       int delim = z[0];
 321       for(i=1; z[i]; i++){
 322         if( z[i]==delim ){
 323           if( z[i+1]==delim ){
 324             i++;
 325           }else{
 326             break;
 327           }
 328         }
 329       }
 330       if( z[i] ) i++;
 331       *tokenType = TK_STRING;
 332       return i;
 333     }
 334     case '.': {
 335       if( !isdigit(z[1]) ){
 336         *tokenType = TK_DOT;
 337         return 1;
 338       }
 339       /* Fall thru into the next case */
 340     }
 341     case '0': case '1': case '2': case '3': case '4':
 342     case '5': case '6': case '7': case '8': case '9': {
 343       *tokenType = TK_INTEGER;
 344       for(i=1; isdigit(z[i]); i++){}
 345       if( z[i]=='.' ){
 346         i++;
 347         while( isdigit(z[i]) ){ i++; }
 348         *tokenType = TK_FLOAT;
 349       }
 350       if( (z[i]=='e' || z[i]=='E') &&
 351            ( isdigit(z[i+1])
 352             || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
 353            )
 354       ){
 355         i += 2;
 356         while( isdigit(z[i]) ){ i++; }
 357         *tokenType = TK_FLOAT;
 358       }else if( z[0]=='.' ){
 359         *tokenType = TK_FLOAT;
 360       }
 361       return i;
 362     }
 363     case '[': {
 364       for(i=1; z[i] && z[i-1]!=']'; i++){}
 365       *tokenType = TK_ID;
 366       return i;
 367     }
 368     default: {
 369       if( !isIdChar[*z] ){
 370         break;
 371       }
 372       for(i=1; isIdChar[z[i]]; i++){}
 373       *tokenType = sqliteKeywordCode(z, i);
 374       return i;
 375     }
 376   }
 377   *tokenType = TK_ILLEGAL;
 378   return 1;
 379 }