4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** An tokenizer for SQL
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
25 #include "wine/debug.h"
30 WINE_DEFAULT_DEBUG_CHANNEL(msi
);
33 ** All the keywords of the SQL language are stored as in a hash
34 ** table composed of instances of the following structure.
36 typedef struct Keyword Keyword
;
38 const char *zName
; /* The keyword name */
39 int tokenType
; /* The token value for this keyword */
43 ** These are the keywords
45 static const Keyword aKeywordTable
[] = {
46 { "ABORT", TK_ABORT
},
47 { "AFTER", TK_AFTER
},
52 { "BEFORE", TK_BEFORE
},
53 { "BEGIN", TK_BEGIN
},
54 { "BETWEEN", TK_BETWEEN
},
56 { "CASCADE", TK_CASCADE
},
58 { "CHECK", TK_CHECK
},
59 { "CLUSTER", TK_CLUSTER
},
60 { "COLLATE", TK_COLLATE
},
61 { "COMMIT", TK_COMMIT
},
62 { "CONFLICT", TK_CONFLICT
},
63 { "CONSTRAINT", TK_CONSTRAINT
},
65 { "CREATE", TK_CREATE
},
66 { "CROSS", TK_JOIN_KW
},
67 { "DEFAULT", TK_DEFAULT
},
68 { "DEFERRED", TK_DEFERRED
},
69 { "DEFERRABLE", TK_DEFERRABLE
},
70 { "DELETE", TK_DELETE
},
71 { "DELIMITERS", TK_DELIMITERS
},
73 { "DISTINCT", TK_DISTINCT
},
78 { "EXCEPT", TK_EXCEPT
},
79 { "EXPLAIN", TK_EXPLAIN
},
82 { "FOREIGN", TK_FOREIGN
},
84 { "FULL", TK_JOIN_KW
},
86 { "GROUP", TK_GROUP
},
87 { "HAVING", TK_HAVING
},
88 { "IGNORE", TK_IGNORE
},
89 { "IMMEDIATE", TK_IMMEDIATE
},
91 { "INDEX", TK_INDEX
},
92 { "INITIALLY", TK_INITIALLY
},
93 { "INNER", TK_JOIN_KW
},
94 { "INSERT", TK_INSERT
},
95 { "INSTEAD", TK_INSTEAD
},
96 { "INTERSECT", TK_INTERSECT
},
99 { "ISNULL", TK_ISNULL
},
102 { "LEFT", TK_JOIN_KW
},
104 { "LIMIT", TK_LIMIT
},
105 { "MATCH", TK_MATCH
},
106 { "NATURAL", TK_JOIN_KW
},
108 { "NOTNULL", TK_NOTNULL
},
111 { "OFFSET", TK_OFFSET
},
114 { "ORDER", TK_ORDER
},
115 { "OUTER", TK_JOIN_KW
},
116 { "PRAGMA", TK_PRAGMA
},
117 { "PRIMARY", TK_PRIMARY
},
118 { "RAISE", TK_RAISE
},
119 { "REFERENCES", TK_REFERENCES
},
120 { "REPLACE", TK_REPLACE
},
121 { "RESTRICT", TK_RESTRICT
},
122 { "RIGHT", TK_JOIN_KW
},
123 { "ROLLBACK", TK_ROLLBACK
},
125 { "SELECT", TK_SELECT
},
127 { "STATEMENT", TK_STATEMENT
},
128 { "TABLE", TK_TABLE
},
130 { "TEMPORARY", TK_TEMP
},
132 { "TRANSACTION", TK_TRANSACTION
},
133 { "TRIGGER", TK_TRIGGER
},
134 { "UNION", TK_UNION
},
135 { "UNIQUE", TK_UNIQUE
},
136 { "UPDATE", TK_UPDATE
},
137 { "USING", TK_USING
},
138 { "VACUUM", TK_VACUUM
},
139 { "VALUES", TK_VALUES
},
142 { "WHERE", TK_WHERE
},
145 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
148 ** This function looks up an identifier to determine if it is a
149 ** keyword. If it is a keyword, the token code of that keyword is
150 ** returned. If the input is not a keyword, TK_ID is returned.
152 int sqliteKeywordCode(const WCHAR
*z
, int n
){
156 len
= WideCharToMultiByte( CP_ACP
, 0, z
, n
, buffer
, sizeof buffer
, NULL
, NULL
);
158 buffer
[i
] = toupper(buffer
[i
]);
159 for(i
=0; i
<KEYWORD_COUNT
; i
++)
161 if(memcmp(buffer
, aKeywordTable
[i
].zName
, len
))
163 if(strlen(aKeywordTable
[i
].zName
) == len
)
164 return aKeywordTable
[i
].tokenType
;
171 ** If X is a character that can be used in an identifier then
172 ** isIdChar[X] will be 1. Otherwise isIdChar[X] will be 0.
174 ** In this implementation, an identifier can be a string of
175 ** alphabetic characters, digits, and "_" plus any character
176 ** with the high-order bit set. The latter rule means that
177 ** any sequence of UTF-8 characters or characters taken from
178 ** an extended ISO8859 character set can form an identifier.
180 static const char isIdChar
[] = {
181 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
182 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
183 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
184 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
185 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
186 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
188 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
190 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8x */
191 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9x */
192 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ax */
193 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Bx */
194 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Cx */
195 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Dx */
196 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ex */
197 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Fx */
202 ** Return the length of the token that begins at z[0]. Return
203 ** -1 if the token is (or might be) incomplete. Store the token
204 ** type in *tokenType before returning.
206 int sqliteGetToken(const WCHAR
*z
, int *tokenType
){
209 case ' ': case '\t': case '\n': case '\f': case '\r': {
210 for(i
=1; isspace(z
[i
]); i
++){}
211 *tokenType
= TK_SPACE
;
215 if( z
[1]==0 ) return -1;
217 for(i
=2; z
[i
] && z
[i
]!='\n'; i
++){}
218 *tokenType
= TK_COMMENT
;
221 *tokenType
= TK_MINUS
;
225 if( z
[1]=='+' && z
[2]==')' ){
226 *tokenType
= TK_ORACLE_OUTER_JOIN
;
238 *tokenType
= TK_SEMI
;
242 *tokenType
= TK_PLUS
;
246 *tokenType
= TK_STAR
;
250 if( z
[1]!='*' || z
[2]==0 ){
251 *tokenType
= TK_SLASH
;
254 for(i
=3; z
[i
] && (z
[i
]!='/' || z
[i
-1]!='*'); i
++){}
256 *tokenType
= TK_COMMENT
;
265 return 1 + (z
[1]=='=');
271 }else if( z
[1]=='>' ){
274 }else if( z
[1]=='<' ){
275 *tokenType
= TK_LSHIFT
;
286 }else if( z
[1]=='>' ){
287 *tokenType
= TK_RSHIFT
;
296 *tokenType
= TK_ILLEGAL
;
305 *tokenType
= TK_BITOR
;
308 *tokenType
= TK_CONCAT
;
313 *tokenType
= TK_COMMA
;
317 *tokenType
= TK_BITAND
;
321 *tokenType
= TK_BITNOT
;
324 case '`': case '\'': case '"': {
336 *tokenType
= TK_STRING
;
340 if( !isdigit(z
[1]) ){
344 /* Fall thru into the next case */
346 case '0': case '1': case '2': case '3': case '4':
347 case '5': case '6': case '7': case '8': case '9': {
348 *tokenType
= TK_INTEGER
;
349 for(i
=1; isdigit(z
[i
]); i
++){}
352 while( isdigit(z
[i
]) ){ i
++; }
353 *tokenType
= TK_FLOAT
;
355 if( (z
[i
]=='e' || z
[i
]=='E') &&
357 || ((z
[i
+1]=='+' || z
[i
+1]=='-') && isdigit(z
[i
+2]))
361 while( isdigit(z
[i
]) ){ i
++; }
362 *tokenType
= TK_FLOAT
;
363 }else if( z
[0]=='.' ){
364 *tokenType
= TK_FLOAT
;
369 for(i
=1; z
[i
] && z
[i
-1]!=']'; i
++){}
377 for(i
=1; isIdChar
[z
[i
]]; i
++){}
378 *tokenType
= sqliteKeywordCode(z
, i
);
382 *tokenType
= TK_ILLEGAL
;