dlls/msi/tokenize.c

   1 /*
   2 ** 2001 September 15
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 *************************************************************************
  12 ** An tokenizer for SQL
  13 **
  14 ** This file contains C code that splits an SQL input string up into
  15 ** individual tokens and sends those tokens one-by-one over to the
  16 ** parser for analysis.
  17 */
  18
  19 #include <ctype.h>
  20 #include <stdarg.h>
  21 #include <stdlib.h>
  22
  23 #include "windef.h"
  24 #include "winbase.h"
  25 #include "wine/debug.h"
  26 #include "winnls.h"
  27 #include "query.h"
  28 #include "sql.tab.h"
  29
  30 WINE_DEFAULT_DEBUG_CHANNEL(msi);
  31
  32 /*
  33 ** All the keywords of the SQL language are stored as in a hash
  34 ** table composed of instances of the following structure.
  35 */
  36 typedef struct Keyword Keyword;
  37 struct Keyword {
  38   const char *zName;             /* The keyword name */
  39   int tokenType;           /* The token value for this keyword */
  40 };
  41
  42 /*
  43 ** These are the keywords
  44 */
  45 static const Keyword aKeywordTable[] = {
  46   { "ABORT", TK_ABORT },
  47   { "AFTER", TK_AFTER },
  48   { "ALL", TK_ALL },
  49   { "AND", TK_AND },
  50   { "AS", TK_AS },
  51   { "ASC", TK_ASC },
  52   { "BEFORE", TK_BEFORE },
  53   { "BEGIN", TK_BEGIN },
  54   { "BETWEEN", TK_BETWEEN },
  55   { "BY", TK_BY },
  56   { "CASCADE", TK_CASCADE },
  57   { "CASE", TK_CASE },
  58   { "CHAR", TK_CHAR },
  59   { "CHARACTER", TK_CHAR },
  60   { "CHECK", TK_CHECK },
  61   { "CLUSTER", TK_CLUSTER },
  62   { "COLLATE", TK_COLLATE },
  63   { "COMMIT", TK_COMMIT },
  64   { "CONFLICT", TK_CONFLICT },
  65   { "CONSTRAINT", TK_CONSTRAINT },
  66   { "COPY", TK_COPY },
  67   { "CREATE", TK_CREATE },
  68   { "CROSS", TK_JOIN_KW },
  69   { "DEFAULT", TK_DEFAULT },
  70   { "DEFERRED", TK_DEFERRED },
  71   { "DEFERRABLE", TK_DEFERRABLE },
  72   { "DELETE", TK_DELETE },
  73   { "DELIMITERS", TK_DELIMITERS },
  74   { "DESC", TK_DESC },
  75   { "DISTINCT", TK_DISTINCT },
  76   { "DROP", TK_DROP },
  77   { "END", TK_END },
  78   { "EACH", TK_EACH },
  79   { "ELSE", TK_ELSE },
  80   { "EXCEPT", TK_EXCEPT },
  81   { "EXPLAIN", TK_EXPLAIN },
  82   { "FAIL", TK_FAIL },
  83   { "FOR", TK_FOR },
  84   { "FOREIGN", TK_FOREIGN },
  85   { "FROM", TK_FROM },
  86   { "FULL", TK_JOIN_KW },
  87   { "GLOB", TK_GLOB },
  88   { "GROUP", TK_GROUP },
  89   { "HAVING", TK_HAVING },
  90   { "HOLD", TK_HOLD },
  91   { "IGNORE", TK_IGNORE },
  92   { "IMMEDIATE", TK_IMMEDIATE },
  93   { "IN", TK_IN },
  94   { "INDEX", TK_INDEX },
  95   { "INITIALLY", TK_INITIALLY },
  96   { "INNER", TK_JOIN_KW },
  97   { "INSERT", TK_INSERT },
  98   { "INSTEAD", TK_INSTEAD },
  99   { "INT", TK_INT },
 100   { "INTERSECT", TK_INTERSECT },
 101   { "INTO", TK_INTO },
 102   { "IS", TK_IS },
 103   { "ISNULL", TK_ISNULL },
 104   { "JOIN", TK_JOIN },
 105   { "KEY", TK_KEY },
 106   { "LEFT", TK_JOIN_KW },
 107   { "LIKE", TK_LIKE },
 108   { "LIMIT", TK_LIMIT },
 109   { "LOCALIZABLE", TK_LOCALIZABLE },
 110   { "LONG", TK_LONG },
 111   { "LONGCHAR", TK_LONGCHAR },
 112   { "MATCH", TK_MATCH },
 113   { "NATURAL", TK_JOIN_KW },
 114   { "NOT", TK_NOT },
 115   { "NOTNULL", TK_NOTNULL },
 116   { "NULL", TK_NULL },
 117   { "OBJECT", TK_OBJECT },
 118   { "OF", TK_OF },
 119   { "OFFSET", TK_OFFSET },
 120   { "ON", TK_ON },
 121   { "OR", TK_OR },
 122   { "ORDER", TK_ORDER },
 123   { "OUTER", TK_JOIN_KW },
 124   { "PRAGMA", TK_PRAGMA },
 125   { "PRIMARY", TK_PRIMARY },
 126   { "RAISE", TK_RAISE },
 127   { "REFERENCES", TK_REFERENCES },
 128   { "REPLACE", TK_REPLACE },
 129   { "RESTRICT", TK_RESTRICT },
 130   { "RIGHT", TK_JOIN_KW },
 131   { "ROLLBACK", TK_ROLLBACK },
 132   { "ROW", TK_ROW },
 133   { "SELECT", TK_SELECT },
 134   { "SET", TK_SET },
 135   { "SHORT", TK_SHORT },
 136   { "STATEMENT", TK_STATEMENT },
 137   { "TABLE", TK_TABLE },
 138   { "TEMP", TK_TEMP },
 139   { "TEMPORARY", TK_TEMP },
 140   { "THEN", TK_THEN },
 141   { "TRANSACTION", TK_TRANSACTION },
 142   { "TRIGGER", TK_TRIGGER },
 143   { "UNION", TK_UNION },
 144   { "UNIQUE", TK_UNIQUE },
 145   { "UPDATE", TK_UPDATE },
 146   { "USING", TK_USING },
 147   { "VACUUM", TK_VACUUM },
 148   { "VALUES", TK_VALUES },
 149   { "VIEW", TK_VIEW },
 150   { "WHEN", TK_WHEN },
 151   { "WHERE", TK_WHERE },
 152 };
 153
 154 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
 155
 156 /*
 157 ** This function looks up an identifier to determine if it is a
 158 ** keyword.  If it is a keyword, the token code of that keyword is
 159 ** returned.  If the input is not a keyword, TK_ID is returned.
 160 */
 161 static int sqliteKeywordCode(const WCHAR *z, int n){
 162   UINT i, len;
 163   char buffer[0x10];
 164
 165   len = WideCharToMultiByte( CP_ACP, 0, z, n, buffer, sizeof buffer, NULL, NULL );
 166   for(i=0; i<len; i++)
 167       buffer[i] = toupper(buffer[i]);
 168   for(i=0; i<KEYWORD_COUNT; i++)
 169   {
 170       if(memcmp(buffer, aKeywordTable[i].zName, len))
 171           continue;
 172       if(strlen(aKeywordTable[i].zName) == len )
 173           return aKeywordTable[i].tokenType;
 174   }
 175   return TK_ID;
 176 }
 177
 178
 179 /*
 180 ** If X is a character that can be used in an identifier then
 181 ** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
 182 **
 183 ** In this implementation, an identifier can be a string of
 184 ** alphabetic characters, digits, and "_" plus any character
 185 ** with the high-order bit set.  The latter rule means that
 186 ** any sequence of UTF-8 characters or characters taken from
 187 ** an extended ISO8859 character set can form an identifier.
 188 */
 189 static const char isIdChar[] = {
 190 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
 191     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
 192     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
 193     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
 194     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
 195     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
 196     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
 197     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
 198     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 199     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
 200     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
 201     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
 202     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
 203     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
 204     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
 205     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
 206     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
 207 };
 208
 209
 210 /*
 211 ** Return the length of the token that begins at z[0].  Return
 212 ** -1 if the token is (or might be) incomplete.  Store the token
 213 ** type in *tokenType before returning.
 214 */
 215 int sqliteGetToken(const WCHAR *z, int *tokenType){
 216   int i;
 217   switch( *z ){
 218     case ' ': case '\t': case '\n': case '\f': case '\r': {
 219       for(i=1; isspace(z[i]); i++){}
 220       *tokenType = TK_SPACE;
 221       return i;
 222     }
 223     case '-': {
 224       if( z[1]==0 ) return -1;
 225       if( z[1]=='-' ){
 226         for(i=2; z[i] && z[i]!='\n'; i++){}
 227         *tokenType = TK_COMMENT;
 228         return i;
 229       }
 230       *tokenType = TK_MINUS;
 231       return 1;
 232     }
 233     case '(': {
 234       if( z[1]=='+' && z[2]==')' ){
 235         *tokenType = TK_ORACLE_OUTER_JOIN;
 236         return 3;
 237       }else{
 238         *tokenType = TK_LP;
 239         return 1;
 240       }
 241     }
 242     case ')': {
 243       *tokenType = TK_RP;
 244       return 1;
 245     }
 246     case ';': {
 247       *tokenType = TK_SEMI;
 248       return 1;
 249     }
 250     case '+': {
 251       *tokenType = TK_PLUS;
 252       return 1;
 253     }
 254     case '*': {
 255       *tokenType = TK_STAR;
 256       return 1;
 257     }
 258     case '/': {
 259       if( z[1]!='*' || z[2]==0 ){
 260         *tokenType = TK_SLASH;
 261         return 1;
 262       }
 263       for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
 264       if( z[i] ) i++;
 265       *tokenType = TK_COMMENT;
 266       return i;
 267     }
 268     case '%': {
 269       *tokenType = TK_REM;
 270       return 1;
 271     }
 272     case '=': {
 273       *tokenType = TK_EQ;
 274       return 1 + (z[1]=='=');
 275     }
 276     case '<': {
 277       if( z[1]=='=' ){
 278         *tokenType = TK_LE;
 279         return 2;
 280       }else if( z[1]=='>' ){
 281         *tokenType = TK_NE;
 282         return 2;
 283       }else if( z[1]=='<' ){
 284         *tokenType = TK_LSHIFT;
 285         return 2;
 286       }else{
 287         *tokenType = TK_LT;
 288         return 1;
 289       }
 290     }
 291     case '>': {
 292       if( z[1]=='=' ){
 293         *tokenType = TK_GE;
 294         return 2;
 295       }else if( z[1]=='>' ){
 296         *tokenType = TK_RSHIFT;
 297         return 2;
 298       }else{
 299         *tokenType = TK_GT;
 300         return 1;
 301       }
 302     }
 303     case '!': {
 304       if( z[1]!='=' ){
 305         *tokenType = TK_ILLEGAL;
 306         return 2;
 307       }else{
 308         *tokenType = TK_NE;
 309         return 2;
 310       }
 311     }
 312     case '|': {
 313       if( z[1]!='|' ){
 314         *tokenType = TK_BITOR;
 315         return 1;
 316       }else{
 317         *tokenType = TK_CONCAT;
 318         return 2;
 319       }
 320     }
 321     case '?': {
 322       *tokenType = TK_WILDCARD;
 323       return 1;
 324     }
 325     case ',': {
 326       *tokenType = TK_COMMA;
 327       return 1;
 328     }
 329     case '&': {
 330       *tokenType = TK_BITAND;
 331       return 1;
 332     }
 333     case '~': {
 334       *tokenType = TK_BITNOT;
 335       return 1;
 336     }
 337     case '`': case '\'': case '"': {
 338       int delim = z[0];
 339       for(i=1; z[i]; i++){
 340         if( z[i]==delim ){
 341           if( z[i+1]==delim ){
 342             i++;
 343           }else{
 344             break;
 345           }
 346         }
 347       }
 348       if( z[i] ) i++;
 349       if( delim == '`' )
 350         *tokenType = TK_ID;
 351       else
 352         *tokenType = TK_STRING;
 353       return i;
 354     }
 355     case '.': {
 356       if( !isdigit(z[1]) ){
 357         *tokenType = TK_DOT;
 358         return 1;
 359       }
 360       /* Fall thru into the next case */
 361     }
 362     case '0': case '1': case '2': case '3': case '4':
 363     case '5': case '6': case '7': case '8': case '9': {
 364       *tokenType = TK_INTEGER;
 365       for(i=1; isdigit(z[i]); i++){}
 366       if( z[i]=='.' ){
 367         i++;
 368         while( isdigit(z[i]) ){ i++; }
 369         *tokenType = TK_FLOAT;
 370       }
 371       if( (z[i]=='e' || z[i]=='E') &&
 372            ( isdigit(z[i+1])
 373             || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
 374            )
 375       ){
 376         i += 2;
 377         while( isdigit(z[i]) ){ i++; }
 378         *tokenType = TK_FLOAT;
 379       }else if( z[0]=='.' ){
 380         *tokenType = TK_FLOAT;
 381       }
 382       return i;
 383     }
 384     case '[': {
 385       for(i=1; z[i] && z[i-1]!=']'; i++){}
 386       *tokenType = TK_ID;
 387       return i;
 388     }
 389     default: {
 390       if( !isIdChar[*z] ){
 391         break;
 392       }
 393       for(i=1; isIdChar[z[i]]; i++){}
 394       *tokenType = sqliteKeywordCode(z, i);
 395       return i;
 396     }
 397   }
 398   *tokenType = TK_ILLEGAL;
 399   return 1;
 400 }