dlls/msi/tokenize.c

   1 /*
   2 ** 2001 September 15
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 *************************************************************************
  12 ** An tokenizer for SQL
  13 **
  14 ** This file contains C code that splits an SQL input string up into
  15 ** individual tokens and sends those tokens one-by-one over to the
  16 ** parser for analysis.
  17 */
  18
  19 #include <ctype.h>
  20 #include <stdarg.h>
  21 #include <stdlib.h>
  22
  23 #include "windef.h"
  24 #include "winbase.h"
  25 #include "wine/debug.h"
  26 #include "winnls.h"
  27 #include "query.h"
  28 #include "y.tab.h"
  29
  30 WINE_DEFAULT_DEBUG_CHANNEL(msi);
  31
  32 /*
  33 ** All the keywords of the SQL language are stored as in a hash
  34 ** table composed of instances of the following structure.
  35 */
  36 typedef struct Keyword Keyword;
  37 struct Keyword {
  38   const char *zName;             /* The keyword name */
  39   int tokenType;           /* The token value for this keyword */
  40 };
  41
  42 /*
  43 ** These are the keywords
  44 */
  45 static const Keyword aKeywordTable[] = {
  46   { "ABORT", TK_ABORT },
  47   { "AFTER", TK_AFTER },
  48   { "ALL", TK_ALL },
  49   { "AND", TK_AND },
  50   { "AS", TK_AS },
  51   { "ASC", TK_ASC },
  52   { "BEFORE", TK_BEFORE },
  53   { "BEGIN", TK_BEGIN },
  54   { "BETWEEN", TK_BETWEEN },
  55   { "BY", TK_BY },
  56   { "CASCADE", TK_CASCADE },
  57   { "CASE", TK_CASE },
  58   { "CHECK", TK_CHECK },
  59   { "CLUSTER", TK_CLUSTER },
  60   { "COLLATE", TK_COLLATE },
  61   { "COMMIT", TK_COMMIT },
  62   { "CONFLICT", TK_CONFLICT },
  63   { "CONSTRAINT", TK_CONSTRAINT },
  64   { "COPY", TK_COPY },
  65   { "CREATE", TK_CREATE },
  66   { "CROSS", TK_JOIN_KW },
  67   { "DEFAULT", TK_DEFAULT },
  68   { "DEFERRED", TK_DEFERRED },
  69   { "DEFERRABLE", TK_DEFERRABLE },
  70   { "DELETE", TK_DELETE },
  71   { "DELIMITERS", TK_DELIMITERS },
  72   { "DESC", TK_DESC },
  73   { "DISTINCT", TK_DISTINCT },
  74   { "DROP", TK_DROP },
  75   { "END", TK_END },
  76   { "EACH", TK_EACH },
  77   { "ELSE", TK_ELSE },
  78   { "EXCEPT", TK_EXCEPT },
  79   { "EXPLAIN", TK_EXPLAIN },
  80   { "FAIL", TK_FAIL },
  81   { "FOR", TK_FOR },
  82   { "FOREIGN", TK_FOREIGN },
  83   { "FROM", TK_FROM },
  84   { "FULL", TK_JOIN_KW },
  85   { "GLOB", TK_GLOB },
  86   { "GROUP", TK_GROUP },
  87   { "HAVING", TK_HAVING },
  88   { "IGNORE", TK_IGNORE },
  89   { "IMMEDIATE", TK_IMMEDIATE },
  90   { "IN", TK_IN },
  91   { "INDEX", TK_INDEX },
  92   { "INITIALLY", TK_INITIALLY },
  93   { "INNER", TK_JOIN_KW },
  94   { "INSERT", TK_INSERT },
  95   { "INSTEAD", TK_INSTEAD },
  96   { "INTERSECT", TK_INTERSECT },
  97   { "INTO", TK_INTO },
  98   { "IS", TK_IS },
  99   { "ISNULL", TK_ISNULL },
 100   { "JOIN", TK_JOIN },
 101   { "KEY", TK_KEY },
 102   { "LEFT", TK_JOIN_KW },
 103   { "LIKE", TK_LIKE },
 104   { "LIMIT", TK_LIMIT },
 105   { "MATCH", TK_MATCH },
 106   { "NATURAL", TK_JOIN_KW },
 107   { "NOT", TK_NOT },
 108   { "NOTNULL", TK_NOTNULL },
 109   { "NULL", TK_NULL },
 110   { "OF", TK_OF },
 111   { "OFFSET", TK_OFFSET },
 112   { "ON", TK_ON },
 113   { "OR", TK_OR },
 114   { "ORDER", TK_ORDER },
 115   { "OUTER", TK_JOIN_KW },
 116   { "PRAGMA", TK_PRAGMA },
 117   { "PRIMARY", TK_PRIMARY },
 118   { "RAISE", TK_RAISE },
 119   { "REFERENCES", TK_REFERENCES },
 120   { "REPLACE", TK_REPLACE },
 121   { "RESTRICT", TK_RESTRICT },
 122   { "RIGHT", TK_JOIN_KW },
 123   { "ROLLBACK", TK_ROLLBACK },
 124   { "ROW", TK_ROW },
 125   { "SELECT", TK_SELECT },
 126   { "SET", TK_SET },
 127   { "STATEMENT", TK_STATEMENT },
 128   { "TABLE", TK_TABLE },
 129   { "TEMP", TK_TEMP },
 130   { "TEMPORARY", TK_TEMP },
 131   { "THEN", TK_THEN },
 132   { "TRANSACTION", TK_TRANSACTION },
 133   { "TRIGGER", TK_TRIGGER },
 134   { "UNION", TK_UNION },
 135   { "UNIQUE", TK_UNIQUE },
 136   { "UPDATE", TK_UPDATE },
 137   { "USING", TK_USING },
 138   { "VACUUM", TK_VACUUM },
 139   { "VALUES", TK_VALUES },
 140   { "VIEW", TK_VIEW },
 141   { "WHEN", TK_WHEN },
 142   { "WHERE", TK_WHERE },
 143 };
 144
 145 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
 146
 147 /*
 148 ** This function looks up an identifier to determine if it is a
 149 ** keyword.  If it is a keyword, the token code of that keyword is
 150 ** returned.  If the input is not a keyword, TK_ID is returned.
 151 */
 152 int sqliteKeywordCode(const WCHAR *z, int n){
 153   int i, len;
 154   char buffer[0x10];
 155
 156   len = WideCharToMultiByte( CP_ACP, 0, z, n, buffer, sizeof buffer, NULL, NULL );
 157   for(i=0; i<len; i++)
 158       buffer[i] = toupper(buffer[i]);
 159   for(i=0; i<KEYWORD_COUNT; i++)
 160   {
 161       if(memcmp(buffer, aKeywordTable[i].zName, len))
 162           continue;
 163       if(strlen(aKeywordTable[i].zName) == len )
 164           return aKeywordTable[i].tokenType;
 165   }
 166   return TK_ID;
 167 }
 168
 169
 170 /*
 171 ** If X is a character that can be used in an identifier then
 172 ** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
 173 **
 174 ** In this implementation, an identifier can be a string of
 175 ** alphabetic characters, digits, and "_" plus any character
 176 ** with the high-order bit set.  The latter rule means that
 177 ** any sequence of UTF-8 characters or characters taken from
 178 ** an extended ISO8859 character set can form an identifier.
 179 */
 180 static const char isIdChar[] = {
 181 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
 182     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
 183     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
 184     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
 185     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
 186     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
 187     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
 188     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
 189     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 190     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
 191     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
 192     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
 193     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
 194     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
 195     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
 196     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
 197     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
 198 };
 199
 200
 201 /*
 202 ** Return the length of the token that begins at z[0].  Return
 203 ** -1 if the token is (or might be) incomplete.  Store the token
 204 ** type in *tokenType before returning.
 205 */
 206 int sqliteGetToken(const WCHAR *z, int *tokenType){
 207   int i;
 208   switch( *z ){
 209     case ' ': case '\t': case '\n': case '\f': case '\r': {
 210       for(i=1; isspace(z[i]); i++){}
 211       *tokenType = TK_SPACE;
 212       return i;
 213     }
 214     case '-': {
 215       if( z[1]==0 ) return -1;
 216       if( z[1]=='-' ){
 217         for(i=2; z[i] && z[i]!='\n'; i++){}
 218         *tokenType = TK_COMMENT;
 219         return i;
 220       }
 221       *tokenType = TK_MINUS;
 222       return 1;
 223     }
 224     case '(': {
 225       if( z[1]=='+' && z[2]==')' ){
 226         *tokenType = TK_ORACLE_OUTER_JOIN;
 227         return 3;
 228       }else{
 229         *tokenType = TK_LP;
 230         return 1;
 231       }
 232     }
 233     case ')': {
 234       *tokenType = TK_RP;
 235       return 1;
 236     }
 237     case ';': {
 238       *tokenType = TK_SEMI;
 239       return 1;
 240     }
 241     case '+': {
 242       *tokenType = TK_PLUS;
 243       return 1;
 244     }
 245     case '*': {
 246       *tokenType = TK_STAR;
 247       return 1;
 248     }
 249     case '/': {
 250       if( z[1]!='*' || z[2]==0 ){
 251         *tokenType = TK_SLASH;
 252         return 1;
 253       }
 254       for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
 255       if( z[i] ) i++;
 256       *tokenType = TK_COMMENT;
 257       return i;
 258     }
 259     case '%': {
 260       *tokenType = TK_REM;
 261       return 1;
 262     }
 263     case '=': {
 264       *tokenType = TK_EQ;
 265       return 1 + (z[1]=='=');
 266     }
 267     case '<': {
 268       if( z[1]=='=' ){
 269         *tokenType = TK_LE;
 270         return 2;
 271       }else if( z[1]=='>' ){
 272         *tokenType = TK_NE;
 273         return 2;
 274       }else if( z[1]=='<' ){
 275         *tokenType = TK_LSHIFT;
 276         return 2;
 277       }else{
 278         *tokenType = TK_LT;
 279         return 1;
 280       }
 281     }
 282     case '>': {
 283       if( z[1]=='=' ){
 284         *tokenType = TK_GE;
 285         return 2;
 286       }else if( z[1]=='>' ){
 287         *tokenType = TK_RSHIFT;
 288         return 2;
 289       }else{
 290         *tokenType = TK_GT;
 291         return 1;
 292       }
 293     }
 294     case '!': {
 295       if( z[1]!='=' ){
 296         *tokenType = TK_ILLEGAL;
 297         return 2;
 298       }else{
 299         *tokenType = TK_NE;
 300         return 2;
 301       }
 302     }
 303     case '|': {
 304       if( z[1]!='|' ){
 305         *tokenType = TK_BITOR;
 306         return 1;
 307       }else{
 308         *tokenType = TK_CONCAT;
 309         return 2;
 310       }
 311     }
 312     case ',': {
 313       *tokenType = TK_COMMA;
 314       return 1;
 315     }
 316     case '&': {
 317       *tokenType = TK_BITAND;
 318       return 1;
 319     }
 320     case '~': {
 321       *tokenType = TK_BITNOT;
 322       return 1;
 323     }
 324     case '`': case '\'': case '"': {
 325       int delim = z[0];
 326       for(i=1; z[i]; i++){
 327         if( z[i]==delim ){
 328           if( z[i+1]==delim ){
 329             i++;
 330           }else{
 331             break;
 332           }
 333         }
 334       }
 335       if( z[i] ) i++;
 336       *tokenType = TK_STRING;
 337       return i;
 338     }
 339     case '.': {
 340       if( !isdigit(z[1]) ){
 341         *tokenType = TK_DOT;
 342         return 1;
 343       }
 344       /* Fall thru into the next case */
 345     }
 346     case '0': case '1': case '2': case '3': case '4':
 347     case '5': case '6': case '7': case '8': case '9': {
 348       *tokenType = TK_INTEGER;
 349       for(i=1; isdigit(z[i]); i++){}
 350       if( z[i]=='.' ){
 351         i++;
 352         while( isdigit(z[i]) ){ i++; }
 353         *tokenType = TK_FLOAT;
 354       }
 355       if( (z[i]=='e' || z[i]=='E') &&
 356            ( isdigit(z[i+1])
 357             || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
 358            )
 359       ){
 360         i += 2;
 361         while( isdigit(z[i]) ){ i++; }
 362         *tokenType = TK_FLOAT;
 363       }else if( z[0]=='.' ){
 364         *tokenType = TK_FLOAT;
 365       }
 366       return i;
 367     }
 368     case '[': {
 369       for(i=1; z[i] && z[i-1]!=']'; i++){}
 370       *tokenType = TK_ID;
 371       return i;
 372     }
 373     default: {
 374       if( !isIdChar[*z] ){
 375         break;
 376       }
 377       for(i=1; isIdChar[z[i]]; i++){}
 378       *tokenType = sqliteKeywordCode(z, i);
 379       return i;
 380     }
 381   }
 382   *tokenType = TK_ILLEGAL;
 383   return 1;
 384 }