?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
(DISTFILES): Comment out a few missing files.
[mono-project.git] / mcs / mcs / cs-tokenizer.cs
blob7c0ae198036152c79acb333fe95c30288c24dc6a
1 //
2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
3 // This also implements the preprocessor
4 //
5 // Author: Miguel de Icaza (miguel@gnu.org)
6 //
7 // Licensed under the terms of the GNU GPL
8 //
9 // (C) 2001, 2002 Ximian, Inc (http://www.ximian.com)
10 //
11
12 /*
13 * TODO:
14 * Make sure we accept the proper Unicode ranges, per the spec.
15 * Report error 1032
16 */
17
18 using System;
19 using System.Text;
20 using System.Collections;
21 using System.IO;
22 using System.Globalization;
23 using System.Reflection;
24
25 namespace Mono.CSharp
26 {
27 /// <summary>
28 /// Tokenizer for C# source code.
29 /// </summary>
30
31 public class Tokenizer : yyParser.yyInput
32 {
33 SeekableStreamReader reader;
34 public SourceFile ref_name;
35 public SourceFile file_name;
36 public int ref_line = 1;
37 public int line = 1;
38 public int col = 1;
39 public int current_token;
40 bool handle_get_set = false;
41 bool handle_remove_add = false;
42 bool handle_assembly = false;
43
44 //
45 // Whether tokens have been seen on this line
46 //
47 bool tokens_seen = false;
48
49 //
50 // Whether a token has been seen on the file
51 // This is needed because `define' is not allowed to be used
52 // after a token has been seen.
53 //
54 bool any_token_seen = false;
55 static Hashtable tokenValues;
56
57 private static Hashtable TokenValueName
58 {
59 get {
60 if (tokenValues == null)
61 tokenValues = GetTokenValueNameHash ();
62
63 return tokenValues;
64 }
65 }
66
67 private static Hashtable GetTokenValueNameHash ()
68 {
69 Type t = typeof (Token);
70 FieldInfo [] fields = t.GetFields ();
71 Hashtable hash = new Hashtable ();
72 foreach (FieldInfo field in fields) {
73 if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int))
74 hash.Add (field.GetValue (null), field.Name);
75 }
76 return hash;
77 }
78
79 //
80 // Returns a verbose representation of the current location
81 //
82 public string location {
83 get {
84 string det;
85
86 if (current_token == Token.ERROR)
87 det = "detail: " + error_details;
88 else
89 det = "";
90
91 // return "Line: "+line+" Col: "+col + "\n" +
92 // "VirtLine: "+ref_line +
93 // " Token: "+current_token + " " + det;
94 string current_token_name = TokenValueName [current_token] as string;
95 if (current_token_name == null)
96 current_token_name = current_token.ToString ();
97
98 return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name.Name,
99 ref_line,
100 col,
101 current_token_name,
102 det);
103 }
104 }
105
106 public bool PropertyParsing {
107 get {
108 return handle_get_set;
109 }
110
111 set {
112 handle_get_set = value;
113 }
114 }
115
116 public bool AssemblyTargetParsing {
117 get {
118 return handle_assembly;
119 }
120
121 set {
122 handle_assembly = value;
123 }
124 }
125
126 public bool EventParsing {
127 get {
128 return handle_remove_add;
129 }
130
131 set {
132 handle_remove_add = value;
133 }
134 }
135
136 //
137 // Class variables
138 //
139 static CharArrayHashtable[] keywords;
140 static NumberStyles styles;
141 static NumberFormatInfo csharp_format_info;
142
143 //
144 // Values for the associated token returned
145 //
146 int putback_char;
147 Object val;
148
149 //
150 // Pre-processor
151 //
152 Hashtable defines;
153
154 const int TAKING = 1;
155 const int TAKEN_BEFORE = 2;
156 const int ELSE_SEEN = 4;
157 const int PARENT_TAKING = 8;
158 const int REGION = 16;
159
160 //
161 // pre-processor if stack state:
162 //
163 Stack ifstack;
164
165 static System.Text.StringBuilder string_builder;
166
167 const int max_id_size = 512;
168 static char [] id_builder = new char [max_id_size];
169
170 static CharArrayHashtable [] identifiers = new CharArrayHashtable [max_id_size + 1];
171
172 const int max_number_size = 128;
173 static char [] number_builder = new char [max_number_size];
174 static int number_pos;
175
176 //
177 // Details about the error encoutered by the tokenizer
178 //
179 string error_details;
180
181 public string error {
182 get {
183 return error_details;
184 }
185 }
186
187 public int Line {
188 get {
189 return ref_line;
190 }
191 }
192
193 public int Col {
194 get {
195 return col;
196 }
197 }
198
199 static void AddKeyword (string kw, int token) {
200 if (keywords [kw.Length] == null) {
201 keywords [kw.Length] = new CharArrayHashtable (kw.Length);
202 }
203 keywords [kw.Length] [kw.ToCharArray ()] = token;
204 }
205
206 static void InitTokens ()
207 {
208 keywords = new CharArrayHashtable [64];
209
210 AddKeyword ("__arglist", Token.ARGLIST);
211 AddKeyword ("abstract", Token.ABSTRACT);
212 AddKeyword ("as", Token.AS);
213 AddKeyword ("add", Token.ADD);
214 AddKeyword ("assembly", Token.ASSEMBLY);
215 AddKeyword ("base", Token.BASE);
216 AddKeyword ("bool", Token.BOOL);
217 AddKeyword ("break", Token.BREAK);
218 AddKeyword ("byte", Token.BYTE);
219 AddKeyword ("case", Token.CASE);
220 AddKeyword ("catch", Token.CATCH);
221 AddKeyword ("char", Token.CHAR);
222 AddKeyword ("checked", Token.CHECKED);
223 AddKeyword ("class", Token.CLASS);
224 AddKeyword ("const", Token.CONST);
225 AddKeyword ("continue", Token.CONTINUE);
226 AddKeyword ("decimal", Token.DECIMAL);
227 AddKeyword ("default", Token.DEFAULT);
228 AddKeyword ("delegate", Token.DELEGATE);
229 AddKeyword ("do", Token.DO);
230 AddKeyword ("double", Token.DOUBLE);
231 AddKeyword ("else", Token.ELSE);
232 AddKeyword ("enum", Token.ENUM);
233 AddKeyword ("event", Token.EVENT);
234 AddKeyword ("explicit", Token.EXPLICIT);
235 AddKeyword ("extern", Token.EXTERN);
236 AddKeyword ("false", Token.FALSE);
237 AddKeyword ("finally", Token.FINALLY);
238 AddKeyword ("fixed", Token.FIXED);
239 AddKeyword ("float", Token.FLOAT);
240 AddKeyword ("for", Token.FOR);
241 AddKeyword ("foreach", Token.FOREACH);
242 AddKeyword ("goto", Token.GOTO);
243 AddKeyword ("get", Token.GET);
244 AddKeyword ("if", Token.IF);
245 AddKeyword ("implicit", Token.IMPLICIT);
246 AddKeyword ("in", Token.IN);
247 AddKeyword ("int", Token.INT);
248 AddKeyword ("interface", Token.INTERFACE);
249 AddKeyword ("internal", Token.INTERNAL);
250 AddKeyword ("is", Token.IS);
251 AddKeyword ("lock", Token.LOCK);
252 AddKeyword ("long", Token.LONG);
253 AddKeyword ("namespace", Token.NAMESPACE);
254 AddKeyword ("new", Token.NEW);
255 AddKeyword ("null", Token.NULL);
256 AddKeyword ("object", Token.OBJECT);
257 AddKeyword ("operator", Token.OPERATOR);
258 AddKeyword ("out", Token.OUT);
259 AddKeyword ("override", Token.OVERRIDE);
260 AddKeyword ("params", Token.PARAMS);
261 AddKeyword ("private", Token.PRIVATE);
262 AddKeyword ("protected", Token.PROTECTED);
263 AddKeyword ("public", Token.PUBLIC);
264 AddKeyword ("readonly", Token.READONLY);
265 AddKeyword ("ref", Token.REF);
266 AddKeyword ("remove", Token.REMOVE);
267 AddKeyword ("return", Token.RETURN);
268 AddKeyword ("sbyte", Token.SBYTE);
269 AddKeyword ("sealed", Token.SEALED);
270 AddKeyword ("set", Token.SET);
271 AddKeyword ("short", Token.SHORT);
272 AddKeyword ("sizeof", Token.SIZEOF);
273 AddKeyword ("stackalloc", Token.STACKALLOC);
274 AddKeyword ("static", Token.STATIC);
275 AddKeyword ("string", Token.STRING);
276 AddKeyword ("struct", Token.STRUCT);
277 AddKeyword ("switch", Token.SWITCH);
278 AddKeyword ("this", Token.THIS);
279 AddKeyword ("throw", Token.THROW);
280 AddKeyword ("true", Token.TRUE);
281 AddKeyword ("try", Token.TRY);
282 AddKeyword ("typeof", Token.TYPEOF);
283 AddKeyword ("uint", Token.UINT);
284 AddKeyword ("ulong", Token.ULONG);
285 AddKeyword ("unchecked", Token.UNCHECKED);
286 AddKeyword ("unsafe", Token.UNSAFE);
287 AddKeyword ("ushort", Token.USHORT);
288 AddKeyword ("using", Token.USING);
289 AddKeyword ("virtual", Token.VIRTUAL);
290 AddKeyword ("void", Token.VOID);
291 AddKeyword ("volatile", Token.VOLATILE);
292 AddKeyword ("while", Token.WHILE);
293 AddKeyword ("partial", Token.PARTIAL);
294 }
295
296 //
297 // Class initializer
298 //
299 static Tokenizer ()
300 {
301 InitTokens ();
302 csharp_format_info = NumberFormatInfo.InvariantInfo;
303 styles = NumberStyles.Float;
304
305 string_builder = new System.Text.StringBuilder ();
306 }
307
308 int GetKeyword (char[] id, int id_len)
309 {
310 /*
311 * Keywords are stored in an array of hashtables grouped by their
312 * length.
313 */
314
315 if ((id_len >= keywords.Length) || (keywords [id_len] == null))
316 return -1;
317 object o = keywords [id_len] [id];
318
319 if (o == null)
320 return -1;
321
322 int res = (int) o;
323
324 if (handle_get_set == false && (res == Token.GET || res == Token.SET))
325 return -1;
326 if (handle_remove_add == false && (res == Token.REMOVE || res == Token.ADD))
327 return -1;
328 if (handle_assembly == false && res == Token.ASSEMBLY)
329 return -1;
330
331 return res;
332
333 }
334
335 public Location Location {
336 get {
337 return new Location (ref_line);
338 }
339 }
340
341 void define (string def)
342 {
343 if (!RootContext.AllDefines.Contains (def)){
344 RootContext.AllDefines [def] = true;
345 }
346 if (defines.Contains (def))
347 return;
348 defines [def] = true;
349 }
350
351 public Tokenizer (SeekableStreamReader input, SourceFile file, ArrayList defs)
352 {
353 this.ref_name = file;
354 this.file_name = file;
355 reader = input;
356
357 putback_char = -1;
358
359 if (defs != null){
360 defines = new Hashtable ();
361 foreach (string def in defs)
362 define (def);
363 }
364
365 //
366 // FIXME: This could be `Location.Push' but we have to
367 // find out why the MS compiler allows this
368 //
369 Mono.CSharp.Location.Push (file);
370 }
371
372 public static void Cleanup () {
373 identifiers = null;
374 }
375
376 static bool is_identifier_start_character (char c)
377 {
378 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c);
379 }
380
381 static bool is_identifier_part_character (char c)
382 {
383 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c);
384 }
385
386 public static bool IsValidIdentifier (string s)
387 {
388 if (s == null || s.Length == 0)
389 return false;
390
391 if (!is_identifier_start_character (s [0]))
392 return false;
393
394 for (int i = 1; i < s.Length; i ++)
395 if (! is_identifier_part_character (s [i]))
396 return false;
397
398 return true;
399 }
400
401 int is_punct (char c, ref bool doread)
402 {
403 int d;
404 int t;
405
406 doread = false;
407
408 switch (c){
409 case '{':
410 return Token.OPEN_BRACE;
411 case '}':
412 return Token.CLOSE_BRACE;
413 case '[':
414 return Token.OPEN_BRACKET;
415 case ']':
416 return Token.CLOSE_BRACKET;
417 case '(':
418 return Token.OPEN_PARENS;
419 case ')': {
420 if (deambiguate_close_parens == 0)
421 return Token.CLOSE_PARENS;
422
423 --deambiguate_close_parens;
424
425 // Save current position and parse next token.
426 int old = reader.Position;
427 int new_token = token ();
428 reader.Position = old;
429 putback_char = -1;
430
431 if (new_token == Token.OPEN_PARENS)
432 return Token.CLOSE_PARENS_OPEN_PARENS;
433 else if (new_token == Token.MINUS)
434 return Token.CLOSE_PARENS_MINUS;
435 else if (IsCastToken (new_token))
436 return Token.CLOSE_PARENS_CAST;
437 else
438 return Token.CLOSE_PARENS_NO_CAST;
439 }
440
441 case ',':
442 return Token.COMMA;
443 case ':':
444 return Token.COLON;
445 case ';':
446 return Token.SEMICOLON;
447 case '~':
448 return Token.TILDE;
449 case '?':
450 return Token.INTERR;
451 }
452
453 d = peekChar ();
454 if (c == '+'){
455
456 if (d == '+')
457 t = Token.OP_INC;
458 else if (d == '=')
459 t = Token.OP_ADD_ASSIGN;
460 else
461 return Token.PLUS;
462 doread = true;
463 return t;
464 }
465 if (c == '-'){
466 if (d == '-')
467 t = Token.OP_DEC;
468 else if (d == '=')
469 t = Token.OP_SUB_ASSIGN;
470 else if (d == '>')
471 t = Token.OP_PTR;
472 else
473 return Token.MINUS;
474 doread = true;
475 return t;
476 }
477
478 if (c == '!'){
479 if (d == '='){
480 doread = true;
481 return Token.OP_NE;
482 }
483 return Token.BANG;
484 }
485
486 if (c == '='){
487 if (d == '='){
488 doread = true;
489 return Token.OP_EQ;
490 }
491 return Token.ASSIGN;
492 }
493
494 if (c == '&'){
495 if (d == '&'){
496 doread = true;
497 return Token.OP_AND;
498 } else if (d == '='){
499 doread = true;
500 return Token.OP_AND_ASSIGN;
501 }
502 return Token.BITWISE_AND;
503 }
504
505 if (c == '|'){
506 if (d == '|'){
507 doread = true;
508 return Token.OP_OR;
509 } else if (d == '='){
510 doread = true;
511 return Token.OP_OR_ASSIGN;
512 }
513 return Token.BITWISE_OR;
514 }
515
516 if (c == '*'){
517 if (d == '='){
518 doread = true;
519 return Token.OP_MULT_ASSIGN;
520 }
521 return Token.STAR;
522 }
523
524 if (c == '/'){
525 if (d == '='){
526 doread = true;
527 return Token.OP_DIV_ASSIGN;
528 }
529 return Token.DIV;
530 }
531
532 if (c == '%'){
533 if (d == '='){
534 doread = true;
535 return Token.OP_MOD_ASSIGN;
536 }
537 return Token.PERCENT;
538 }
539
540 if (c == '^'){
541 if (d == '='){
542 doread = true;
543 return Token.OP_XOR_ASSIGN;
544 }
545 return Token.CARRET;
546 }
547
548 if (c == '<'){
549 if (d == '<'){
550 getChar ();
551 d = peekChar ();
552
553 if (d == '='){
554 doread = true;
555 return Token.OP_SHIFT_LEFT_ASSIGN;
556 }
557 return Token.OP_SHIFT_LEFT;
558 } else if (d == '='){
559 doread = true;
560 return Token.OP_LE;
561 }
562 return Token.OP_LT;
563 }
564
565 if (c == '>'){
566 if (d == '>'){
567 getChar ();
568 d = peekChar ();
569
570 if (d == '='){
571 doread = true;
572 return Token.OP_SHIFT_RIGHT_ASSIGN;
573 }
574 return Token.OP_SHIFT_RIGHT;
575 } else if (d == '='){
576 doread = true;
577 return Token.OP_GE;
578 }
579 return Token.OP_GT;
580 }
581 return Token.ERROR;
582 }
583
584 int deambiguate_close_parens = 0;
585
586 public void Deambiguate_CloseParens ()
587 {
588 putback (')');
589 deambiguate_close_parens++;
590 }
591
592 void Error_NumericConstantTooLong ()
593 {
594 Report.Error (1021, Location, "Numeric constant too long");
595 }
596
597 bool decimal_digits (int c)
598 {
599 int d;
600 bool seen_digits = false;
601
602 if (c != -1){
603 if (number_pos == max_number_size)
604 Error_NumericConstantTooLong ();
605 number_builder [number_pos++] = (char) c;
606 }
607
608 //
609 // We use peekChar2, because decimal_digits needs to do a
610 // 2-character look-ahead (5.ToString for example).
611 //
612 while ((d = peekChar2 ()) != -1){
613 if (d >= '0' && d <= '9'){
614 if (number_pos == max_number_size)
615 Error_NumericConstantTooLong ();
616 number_builder [number_pos++] = (char) d;
617 getChar ();
618 seen_digits = true;
619 } else
620 break;
621 }
622
623 return seen_digits;
624 }
625
626 bool is_hex (int e)
627 {
628 return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
629 }
630
631 void hex_digits (int c)
632 {
633 if (c != -1)
634 number_builder [number_pos++] = (char) c;
635
636 }
637
638 int real_type_suffix (int c)
639 {
640 int t;
641
642 switch (c){
643 case 'F': case 'f':
644 t = Token.LITERAL_FLOAT;
645 break;
646 case 'D': case 'd':
647 t = Token.LITERAL_DOUBLE;
648 break;
649 case 'M': case 'm':
650 t= Token.LITERAL_DECIMAL;
651 break;
652 default:
653 return Token.NONE;
654 }
655 return t;
656 }
657
658 int integer_type_suffix (ulong ul, int c)
659 {
660 bool is_unsigned = false;
661 bool is_long = false;
662
663 if (c != -1){
664 bool scanning = true;
665 do {
666 switch (c){
667 case 'U': case 'u':
668 if (is_unsigned)
669 scanning = false;
670 is_unsigned = true;
671 getChar ();
672 break;
673
674 case 'l':
675 if (!is_unsigned && (RootContext.WarningLevel >= 4)){
676 //
677 // if we have not seen anything in between
678 // report this error
679 //
680 Report.Warning (78, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
681 }
682 goto case 'L';
683
684 case 'L':
685 if (is_long)
686 scanning = false;
687 is_long = true;
688 getChar ();
689 break;
690
691 default:
692 scanning = false;
693 break;
694 }
695 c = peekChar ();
696 } while (scanning);
697 }
698
699 if (is_long && is_unsigned){
700 val = ul;
701 return Token.LITERAL_INTEGER;
702 } else if (is_unsigned){
703 // uint if possible, or ulong else.
704
705 if ((ul & 0xffffffff00000000) == 0)
706 val = (uint) ul;
707 else
708 val = ul;
709 } else if (is_long){
710 // long if possible, ulong otherwise
711 if ((ul & 0x8000000000000000) != 0)
712 val = ul;
713 else
714 val = (long) ul;
715 } else {
716 // int, uint, long or ulong in that order
717 if ((ul & 0xffffffff00000000) == 0){
718 uint ui = (uint) ul;
719
720 if ((ui & 0x80000000) != 0)
721 val = ui;
722 else
723 val = (int) ui;
724 } else {
725 if ((ul & 0x8000000000000000) != 0)
726 val = ul;
727 else
728 val = (long) ul;
729 }
730 }
731 return Token.LITERAL_INTEGER;
732 }
733
734 //
735 // given `c' as the next char in the input decide whether
736 // we need to convert to a special type, and then choose
737 // the best representation for the integer
738 //
739 int adjust_int (int c)
740 {
741 try {
742 if (number_pos > 9){
743 ulong ul = (uint) (number_builder [0] - '0');
744
745 for (int i = 1; i < number_pos; i++){
746 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
747 }
748 return integer_type_suffix (ul, c);
749 } else {
750 uint ui = (uint) (number_builder [0] - '0');
751
752 for (int i = 1; i < number_pos; i++){
753 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
754 }
755 return integer_type_suffix (ui, c);
756 }
757 } catch (OverflowException) {
758 error_details = "Integral constant is too large";
759 Report.Error (1021, Location, error_details);
760 val = 0ul;
761 return Token.LITERAL_INTEGER;
762 }
763 }
764
765 int adjust_real (int t)
766 {
767 string s = new String (number_builder, 0, number_pos);
768
769 switch (t){
770 case Token.LITERAL_DECIMAL:
771 try {
772 val = System.Decimal.Parse (s, styles, csharp_format_info);
773 } catch (OverflowException) {
774 val = 0m;
775 error_details = "Floating-point constant is outside the range of the type 'decimal'";
776 Report.Error (594, Location, error_details);
777 }
778 break;
779 case Token.LITERAL_FLOAT:
780 try {
781 val = (float) System.Double.Parse (s, styles, csharp_format_info);
782 } catch (OverflowException) {
783 val = 0.0f;
784 error_details = "Floating-point constant is outside the range of the type 'float'";
785 Report.Error (594, Location, error_details);
786 }
787 break;
788
789 case Token.LITERAL_DOUBLE:
790 case Token.NONE:
791 t = Token.LITERAL_DOUBLE;
792 try {
793 val = System.Double.Parse (s, styles, csharp_format_info);
794 } catch (OverflowException) {
795 val = 0.0;
796 error_details = "Floating-point constant is outside the range of the type 'double'";
797 Report.Error (594, Location, error_details);
798 }
799 break;
800 }
801 return t;
802 }
803
804 int handle_hex ()
805 {
806 int d;
807 ulong ul;
808
809 getChar ();
810 while ((d = peekChar ()) != -1){
811 if (is_hex (d)){
812 number_builder [number_pos++] = (char) d;
813 getChar ();
814 } else
815 break;
816 }
817
818 string s = new String (number_builder, 0, number_pos);
819 try {
820 if (number_pos <= 8)
821 ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
822 else
823 ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
824 } catch (OverflowException){
825 error_details = "Integral constant is too large";
826 Report.Error (1021, Location, error_details);
827 val = 0ul;
828 return Token.LITERAL_INTEGER;
829 }
830 catch (FormatException) {
831 Report.Error (1013, Location, "Invalid number");
832 val = 0ul;
833 return Token.LITERAL_INTEGER;
834 }
835
836 return integer_type_suffix (ul, peekChar ());
837 }
838
839 //
840 // Invoked if we know we have .digits or digits
841 //
842 int is_number (int c)
843 {
844 bool is_real = false;
845 int type;
846
847 number_pos = 0;
848
849 if (c >= '0' && c <= '9'){
850 if (c == '0'){
851 int peek = peekChar ();
852
853 if (peek == 'x' || peek == 'X')
854 return handle_hex ();
855 }
856 decimal_digits (c);
857 c = getChar ();
858 }
859
860 //
861 // We need to handle the case of
862 // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
863 //
864 if (c == '.'){
865 if (decimal_digits ('.')){
866 is_real = true;
867 c = getChar ();
868 } else {
869 putback ('.');
870 number_pos--;
871 return adjust_int (-1);
872 }
873 }
874
875 if (c == 'e' || c == 'E'){
876 is_real = true;
877 if (number_pos == max_number_size)
878 Error_NumericConstantTooLong ();
879 number_builder [number_pos++] = 'e';
880 c = getChar ();
881
882 if (c == '+'){
883 if (number_pos == max_number_size)
884 Error_NumericConstantTooLong ();
885 number_builder [number_pos++] = '+';
886 c = -1;
887 } else if (c == '-') {
888 if (number_pos == max_number_size)
889 Error_NumericConstantTooLong ();
890 number_builder [number_pos++] = '-';
891 c = -1;
892 } else {
893 if (number_pos == max_number_size)
894 Error_NumericConstantTooLong ();
895 number_builder [number_pos++] = '+';
896 }
897
898 decimal_digits (c);
899 c = getChar ();
900 }
901
902 type = real_type_suffix (c);
903 if (type == Token.NONE && !is_real){
904 putback (c);
905 return adjust_int (c);
906 } else
907 is_real = true;
908
909 if (type == Token.NONE){
910 putback (c);
911 }
912
913 if (is_real)
914 return adjust_real (type);
915
916 Console.WriteLine ("This should not be reached");
917 throw new Exception ("Is Number should never reach this point");
918 }
919
920 //
921 // Accepts exactly count (4 or 8) hex, no more no less
922 //
923 int getHex (int count, out bool error)
924 {
925 int i;
926 int total = 0;
927 int c;
928 int top = count != -1 ? count : 4;
929
930 getChar ();
931 error = false;
932 for (i = 0; i < top; i++){
933 c = getChar ();
934
935 if (c >= '0' && c <= '9')
936 c = (int) c - (int) '0';
937 else if (c >= 'A' && c <= 'F')
938 c = (int) c - (int) 'A' + 10;
939 else if (c >= 'a' && c <= 'f')
940 c = (int) c - (int) 'a' + 10;
941 else {
942 error = true;
943 return 0;
944 }
945
946 total = (total * 16) + c;
947 if (count == -1){
948 int p = peekChar ();
949 if (p == -1)
950 break;
951 if (!is_hex ((char)p))
952 break;
953 }
954 }
955 return total;
956 }
957
958 int escape (int c)
959 {
960 bool error;
961 int d;
962 int v;
963
964 d = peekChar ();
965 if (c != '\\')
966 return c;
967
968 switch (d){
969 case 'a':
970 v = '\a'; break;
971 case 'b':
972 v = '\b'; break;
973 case 'n':
974 v = '\n'; break;
975 case 't':
976 v = '\t'; break;
977 case 'v':
978 v = '\v'; break;
979 case 'r':
980 v = '\r'; break;
981 case '\\':
982 v = '\\'; break;
983 case 'f':
984 v = '\f'; break;
985 case '0':
986 v = 0; break;
987 case '"':
988 v = '"'; break;
989 case '\'':
990 v = '\''; break;
991 case 'x':
992 v = getHex (-1, out error);
993 if (error)
994 goto default;
995 return v;
996 case 'u':
997 v = getHex (4, out error);
998 if (error)
999 goto default;
1000 return v;
1001 case 'U':
1002 v = getHex (8, out error);
1003 if (error)
1004 goto default;
1005 return v;
1006 default:
1007 Report.Error (1009, Location, "Unrecognized escape sequence in " + (char)d);
1008 return d;
1009 }
1010 getChar ();
1011 return v;
1012 }
1013
1014 int getChar ()
1015 {
1016 if (putback_char != -1){
1017 int x = putback_char;
1018 putback_char = -1;
1019
1020 return x;
1021 }
1022 return reader.Read ();
1023 }
1024
1025 int peekChar ()
1026 {
1027 if (putback_char != -1)
1028 return putback_char;
1029 putback_char = reader.Read ();
1030 return putback_char;
1031 }
1032
1033 int peekChar2 ()
1034 {
1035 if (putback_char != -1)
1036 return putback_char;
1037 return reader.Peek ();
1038 }
1039
1040 void putback (int c)
1041 {
1042 if (putback_char != -1){
1043 Console.WriteLine ("Col: " + col);
1044 Console.WriteLine ("Row: " + line);
1045 Console.WriteLine ("Name: " + ref_name.Name);
1046 Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c);
1047 throw new Exception ("This should not happen putback on putback");
1048 }
1049 putback_char = c;
1050 }
1051
1052 public bool advance ()
1053 {
1054 return peekChar () != -1;
1055 }
1056
1057 public Object Value {
1058 get {
1059 return val;
1060 }
1061 }
1062
1063 public Object value ()
1064 {
1065 return val;
1066 }
1067
1068 bool IsCastToken (int token)
1069 {
1070 switch (token) {
1071 case Token.BANG:
1072 case Token.TILDE:
1073 case Token.IDENTIFIER:
1074 case Token.LITERAL_INTEGER:
1075 case Token.LITERAL_FLOAT:
1076 case Token.LITERAL_DOUBLE:
1077 case Token.LITERAL_DECIMAL:
1078 case Token.LITERAL_CHARACTER:
1079 case Token.LITERAL_STRING:
1080 case Token.BASE:
1081 case Token.CHECKED:
1082 case Token.FALSE:
1083 case Token.FIXED:
1084 case Token.NEW:
1085 case Token.NULL:
1086 case Token.SIZEOF:
1087 case Token.THIS:
1088 case Token.THROW:
1089 case Token.TRUE:
1090 case Token.TYPEOF:
1091 case Token.UNCHECKED:
1092 case Token.UNSAFE:
1093
1094 //
1095 // These can be part of a member access
1096 //
1097 case Token.INT:
1098 case Token.UINT:
1099 case Token.SHORT:
1100 case Token.USHORT:
1101 case Token.LONG:
1102 case Token.ULONG:
1103 case Token.DOUBLE:
1104 case Token.FLOAT:
1105 case Token.CHAR:
1106 return true;
1107
1108 default:
1109 return false;
1110 }
1111 }
1112
1113 public int token ()
1114 {
1115 current_token = xtoken ();
1116 return current_token;
1117 }
1118
1119 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
1120
1121 void get_cmd_arg (out string cmd, out string arg)
1122 {
1123 int c;
1124
1125 tokens_seen = false;
1126 arg = "";
1127 static_cmd_arg.Length = 0;
1128
1129 // skip over white space
1130 while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1131 ;
1132
1133
1134 while ((c != -1) && (c != '\n') && (c != ' ') && (c != '\t') && (c != '\r')){
1135 if (is_identifier_part_character ((char) c)){
1136 static_cmd_arg.Append ((char) c);
1137 c = getChar ();
1138 } else {
1139 putback (c);
1140 break;
1141 }
1142 }
1143
1144 cmd = static_cmd_arg.ToString ();
1145
1146 if (c == '\n'){
1147 line++;
1148 ref_line++;
1149 return;
1150 } else if (c == '\r')
1151 col = 0;
1152
1153 // skip over white space
1154 while ((c = getChar ()) != -1 && (c != '\n') && ((c == '\r') || (c == ' ') || (c == '\t')))
1155 ;
1156
1157 if (c == '\n'){
1158 line++;
1159 ref_line++;
1160 return;
1161 } else if (c == '\r'){
1162 col = 0;
1163 return;
1164 }
1165
1166 static_cmd_arg.Length = 0;
1167 static_cmd_arg.Append ((char) c);
1168
1169 while ((c = getChar ()) != -1 && (c != '\n') && (c != '\r')){
1170 static_cmd_arg.Append ((char) c);
1171 }
1172
1173 if (c == '\n'){
1174 line++;
1175 ref_line++;
1176 } else if (c == '\r')
1177 col = 0;
1178 arg = static_cmd_arg.ToString ().Trim ();
1179 }
1180
1181 //
1182 // Handles the #line directive
1183 //
1184 bool PreProcessLine (string arg)
1185 {
1186 if (arg == "")
1187 return false;
1188
1189 if (arg == "default"){
1190 ref_line = line;
1191 ref_name = file_name;
1192 Location.Push (ref_name);
1193 return true;
1194 } else if (arg == "hidden"){
1195 //
1196 // We ignore #line hidden
1197 //
1198 return true;
1199 }
1200
1201 try {
1202 int pos;
1203
1204 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1205 ref_line = System.Int32.Parse (arg.Substring (0, pos));
1206 pos++;
1207
1208 char [] quotes = { '\"' };
1209
1210 string name = arg.Substring (pos). Trim (quotes);
1211 ref_name = Location.LookupFile (name);
1212 file_name.HasLineDirective = true;
1213 ref_name.HasLineDirective = true;
1214 Location.Push (ref_name);
1215 } else {
1216 ref_line = System.Int32.Parse (arg);
1217 }
1218 } catch {
1219 return false;
1220 }
1221
1222 return true;
1223 }
1224
1225 //
1226 // Handles #define and #undef
1227 //
1228 void PreProcessDefinition (bool is_define, string arg)
1229 {
1230 if (arg == "" || arg == "true" || arg == "false"){
1231 Report.Error (1001, Location, "Missing identifer to pre-processor directive");
1232 return;
1233 }
1234
1235 char[] whitespace = { ' ', '\t' };
1236 if (arg.IndexOfAny (whitespace) != -1){
1237 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
1238 return;
1239 }
1240
1241 if (!is_identifier_start_character (arg [0]))
1242 Report.Error (1001, Location, "Identifier expected: " + arg);
1243
1244 foreach (char c in arg.Substring (1)){
1245 if (!is_identifier_part_character (c)){
1246 Report.Error (1001, Location, "Identifier expected: " + arg);
1247 return;
1248 }
1249 }
1250
1251 if (is_define){
1252 if (defines == null)
1253 defines = new Hashtable ();
1254 define (arg);
1255 } else {
1256 if (defines == null)
1257 return;
1258 if (defines.Contains (arg))
1259 defines.Remove (arg);
1260 }
1261 }
1262
1263 /// <summary>
1264 /// Handles #pragma directive
1265 /// </summary>
1266 void PreProcessPragma (string arg)
1267 {
1268 const string disable = "warning disable";
1269 const string restore = "warning restore";
1270
1271 if (arg == disable) {
1272 Report.RegisterWarningRegion (Location).WarningDisable (line);
1273 return;
1274 }
1275
1276 if (arg == restore) {
1277 Report.RegisterWarningRegion (Location).WarningEnable (line);
1278 return;
1279 }
1280
1281 if (arg.StartsWith (disable)) {
1282 int[] codes = ParseNumbers (arg.Substring (disable.Length));
1283 foreach (int code in codes) {
1284 if (code != 0)
1285 Report.RegisterWarningRegion (Location).WarningDisable (Location, code);
1286 }
1287 return;
1288 }
1289
1290 if (arg.StartsWith (restore)) {
1291 int[] codes = ParseNumbers (arg.Substring (restore.Length));
1292 foreach (int code in codes) {
1293 Report.RegisterWarningRegion (Location).WarningEnable (Location, code);
1294 }
1295 return;
1296 }
1297
1298 return;
1299 }
1300
1301 int[] ParseNumbers (string text)
1302 {
1303 string[] string_array = text.Split (',');
1304 int[] values = new int [string_array.Length];
1305 int index = 0;
1306 foreach (string string_code in string_array) {
1307 try {
1308 values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1309 }
1310 catch (FormatException) {
1311 Report.Warning (1692, Location, "Invalid number");
1312 }
1313 }
1314 return values;
1315 }
1316
1317 bool eval_val (string s)
1318 {
1319 if (s == "true")
1320 return true;
1321 if (s == "false")
1322 return false;
1323
1324 if (defines == null)
1325 return false;
1326 if (defines.Contains (s))
1327 return true;
1328
1329 return false;
1330 }
1331
1332 bool pp_primary (ref string s)
1333 {
1334 s = s.Trim ();
1335 int len = s.Length;
1336
1337 if (len > 0){
1338 char c = s [0];
1339
1340 if (c == '('){
1341 s = s.Substring (1);
1342 bool val = pp_expr (ref s);
1343 if (s.Length > 0 && s [0] == ')'){
1344 s = s.Substring (1);
1345 return val;
1346 }
1347 Error_InvalidDirective ();
1348 return false;
1349 }
1350
1351 if (is_identifier_start_character (c)){
1352 int j = 1;
1353
1354 while (j < len){
1355 c = s [j];
1356
1357 if (is_identifier_part_character (c)){
1358 j++;
1359 continue;
1360 }
1361 bool v = eval_val (s.Substring (0, j));
1362 s = s.Substring (j);
1363 return v;
1364 }
1365 bool vv = eval_val (s);
1366 s = "";
1367 return vv;
1368 }
1369 }
1370 Error_InvalidDirective ();
1371 return false;
1372 }
1373
1374 bool pp_unary (ref string s)
1375 {
1376 s = s.Trim ();
1377 int len = s.Length;
1378
1379 if (len > 0){
1380 if (s [0] == '!'){
1381 if (len > 1 && s [1] == '='){
1382 Error_InvalidDirective ();
1383 return false;
1384 }
1385 s = s.Substring (1);
1386 return ! pp_primary (ref s);
1387 } else
1388 return pp_primary (ref s);
1389 } else {
1390 Error_InvalidDirective ();
1391 return false;
1392 }
1393 }
1394
1395 bool pp_eq (ref string s)
1396 {
1397 bool va = pp_unary (ref s);
1398
1399 s = s.Trim ();
1400 int len = s.Length;
1401 if (len > 0){
1402 if (s [0] == '='){
1403 if (len > 2 && s [1] == '='){
1404 s = s.Substring (2);
1405 return va == pp_unary (ref s);
1406 } else {
1407 Error_InvalidDirective ();
1408 return false;
1409 }
1410 } else if (s [0] == '!' && len > 1 && s [1] == '='){
1411 s = s.Substring (2);
1412
1413 return va != pp_unary (ref s);
1414
1415 }
1416 }
1417
1418 return va;
1419
1420 }
1421
1422 bool pp_and (ref string s)
1423 {
1424 bool va = pp_eq (ref s);
1425
1426 s = s.Trim ();
1427 int len = s.Length;
1428 if (len > 0){
1429 if (s [0] == '&'){
1430 if (len > 2 && s [1] == '&'){
1431 s = s.Substring (2);
1432 return (va & pp_eq (ref s));
1433 } else {
1434 Error_InvalidDirective ();
1435 return false;
1436 }
1437 }
1438 }
1439 return va;
1440 }
1441
1442 //
1443 // Evaluates an expression for `#if' or `#elif'
1444 //
1445 bool pp_expr (ref string s)
1446 {
1447 bool va = pp_and (ref s);
1448 s = s.Trim ();
1449 int len = s.Length;
1450 if (len > 0){
1451 char c = s [0];
1452
1453 if (c == '|'){
1454 if (len > 2 && s [1] == '|'){
1455 s = s.Substring (2);
1456 return va | pp_expr (ref s);
1457 } else {
1458 Error_InvalidDirective ();
1459 return false;
1460 }
1461 }
1462 }
1463
1464 return va;
1465 }
1466
1467 bool eval (string s)
1468 {
1469 bool v = pp_expr (ref s);
1470 s = s.Trim ();
1471 if (s.Length != 0){
1472 Error_InvalidDirective ();
1473 return false;
1474 }
1475
1476 return v;
1477 }
1478
1479 void Error_InvalidDirective ()
1480 {
1481 Report.Error (1517, Location, "Invalid pre-processor directive");
1482 }
1483
1484 void Error_UnexpectedDirective (string extra)
1485 {
1486 Report.Error (
1487 1028, Location,
1488 "Unexpected processor directive (" + extra + ")");
1489 }
1490
1491 void Error_TokensSeen ()
1492 {
1493 Report.Error (
1494 1032, Location,
1495 "Cannot define or undefine pre-processor symbols after a token in the file");
1496 }
1497
1498 //
1499 // if true, then the code continues processing the code
1500 // if false, the code stays in a loop until another directive is
1501 // reached.
1502 //
1503 bool handle_preprocessing_directive (bool caller_is_taking)
1504 {
1505 string cmd, arg;
1506 bool region_directive = false;
1507
1508 get_cmd_arg (out cmd, out arg);
1509
1510 // Eat any trailing whitespaces and single-line comments
1511 if (arg.IndexOf ("//") != -1)
1512 arg = arg.Substring (0, arg.IndexOf ("//"));
1513 arg = arg.TrimEnd (' ', '\t');
1514
1515 //
1516 // The first group of pre-processing instructions is always processed
1517 //
1518 switch (cmd){
1519 case "pragma":
1520 if (RootContext.Version == LanguageVersion.ISO_1) {
1521 Report.FeatureIsNotStandardized (Location, "#pragma");
1522 return caller_is_taking;
1523 }
1524
1525 PreProcessPragma (arg);
1526 return caller_is_taking;
1527
1528 case "line":
1529 if (!PreProcessLine (arg))
1530 Report.Error (
1531 1576, Location,
1532 "Argument to #line directive is missing or invalid");
1533 return caller_is_taking;
1534
1535 case "region":
1536 region_directive = true;
1537 arg = "true";
1538 goto case "if";
1539
1540 case "endregion":
1541 region_directive = true;
1542 goto case "endif";
1543
1544 case "if":
1545 if (arg == ""){
1546 Error_InvalidDirective ();
1547 return true;
1548 }
1549 bool taking = false;
1550 if (ifstack == null)
1551 ifstack = new Stack ();
1552
1553 if (ifstack.Count == 0){
1554 taking = true;
1555 } else {
1556 int state = (int) ifstack.Peek ();
1557 if ((state & TAKING) != 0)
1558 taking = true;
1559 }
1560
1561 if (eval (arg) && taking){
1562 int push = TAKING | TAKEN_BEFORE | PARENT_TAKING;
1563 if (region_directive)
1564 push |= REGION;
1565 ifstack.Push (push);
1566 return true;
1567 } else {
1568 int push = (taking ? PARENT_TAKING : 0);
1569 if (region_directive)
1570 push |= REGION;
1571 ifstack.Push (push);
1572 return false;
1573 }
1574
1575 case "endif":
1576 if (ifstack == null || ifstack.Count == 0){
1577 Error_UnexpectedDirective ("no #if for this #endif");
1578 return true;
1579 } else {
1580 int pop = (int) ifstack.Pop ();
1581
1582 if (region_directive && ((pop & REGION) == 0))
1583 Report.Error (1027, Location, "#endif directive expected");
1584 else if (!region_directive && ((pop & REGION) != 0))
1585 Report.Error (1038, Location, "#endregion directive expected");
1586
1587 if (ifstack.Count == 0)
1588 return true;
1589 else {
1590 int state = (int) ifstack.Peek ();
1591
1592 if ((state & TAKING) != 0)
1593 return true;
1594 else
1595 return false;
1596 }
1597 }
1598
1599 case "elif":
1600 if (ifstack == null || ifstack.Count == 0){
1601 Error_UnexpectedDirective ("no #if for this #elif");
1602 return true;
1603 } else {
1604 int state = (int) ifstack.Peek ();
1605
1606 if ((state & REGION) != 0) {
1607 Report.Error (1038, Location, "#endregion directive expected");
1608 return true;
1609 }
1610
1611 if ((state & ELSE_SEEN) != 0){
1612 Error_UnexpectedDirective ("#elif not valid after #else");
1613 return true;
1614 }
1615
1616 if ((state & (TAKEN_BEFORE | TAKING)) != 0)
1617 return false;
1618
1619 if (eval (arg) && ((state & PARENT_TAKING) != 0)){
1620 state = (int) ifstack.Pop ();
1621 ifstack.Push (state | TAKING | TAKEN_BEFORE);
1622 return true;
1623 } else
1624 return false;
1625 }
1626
1627 case "else":
1628 if (ifstack == null || ifstack.Count == 0){
1629 Report.Error (
1630 1028, Location,
1631 "Unexpected processor directive (no #if for this #else)");
1632 return true;
1633 } else {
1634 int state = (int) ifstack.Peek ();
1635
1636 if ((state & REGION) != 0) {
1637 Report.Error (1038, Location, "#endregion directive expected");
1638 return true;
1639 }
1640
1641 if ((state & ELSE_SEEN) != 0){
1642 Error_UnexpectedDirective ("#else within #else");
1643 return true;
1644 }
1645
1646 ifstack.Pop ();
1647
1648 bool ret;
1649 if ((state & TAKEN_BEFORE) == 0){
1650 ret = ((state & PARENT_TAKING) != 0);
1651 } else
1652 ret = false;
1653
1654 if (ret)
1655 state |= TAKING;
1656 else
1657 state &= ~TAKING;
1658
1659 ifstack.Push (state | ELSE_SEEN);
1660
1661 return ret;
1662 }
1663 }
1664
1665 //
1666 // These are only processed if we are in a `taking' block
1667 //
1668 if (!caller_is_taking)
1669 return false;
1670
1671 switch (cmd){
1672 case "define":
1673 if (any_token_seen){
1674 Error_TokensSeen ();
1675 return true;
1676 }
1677 PreProcessDefinition (true, arg);
1678 return true;
1679
1680 case "undef":
1681 if (any_token_seen){
1682 Error_TokensSeen ();
1683 return true;
1684 }
1685 PreProcessDefinition (false, arg);
1686 return true;
1687
1688 case "error":
1689 Report.Error (1029, Location, "#error: '" + arg + "'");
1690 return true;
1691
1692 case "warning":
1693 Report.Warning (1030, Location, "#warning: '{0}'", arg);
1694 return true;
1695 }
1696
1697 Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")");
1698 return true;
1699
1700 }
1701
1702 private int consume_string (bool quoted)
1703 {
1704 int c;
1705 string_builder.Length = 0;
1706
1707 while ((c = getChar ()) != -1){
1708 if (c == '"'){
1709 if (quoted && peekChar () == '"'){
1710 string_builder.Append ((char) c);
1711 getChar ();
1712 continue;
1713 } else {
1714 val = string_builder.ToString ();
1715 return Token.LITERAL_STRING;
1716 }
1717 }
1718
1719 if (c == '\n'){
1720 if (!quoted)
1721 Report.Error (1010, Location, "Newline in constant");
1722 line++;
1723 ref_line++;
1724 col = 0;
1725 } else
1726 col++;
1727
1728 if (!quoted){
1729 c = escape (c);
1730 if (c == -1)
1731 return Token.ERROR;
1732 }
1733 string_builder.Append ((char) c);
1734 }
1735
1736 Report.Error (1039, Location, "Unterminated string literal");
1737 return Token.EOF;
1738 }
1739
1740 private int consume_identifier (int s)
1741 {
1742 int res = consume_identifier (s, false);
1743
1744 if (res == Token.PARTIAL) {
1745 // Save current position and parse next token.
1746 int old = reader.Position;
1747 int old_putback = putback_char;
1748
1749 putback_char = -1;
1750
1751 int next_token = token ();
1752 bool ok = (next_token == Token.CLASS) ||
1753 (next_token == Token.STRUCT) ||
1754 (next_token == Token.INTERFACE);
1755
1756 reader.Position = old;
1757 putback_char = old_putback;
1758
1759 if (ok)
1760 return res;
1761 else {
1762 val = "partial";
1763 return Token.IDENTIFIER;
1764 }
1765 }
1766
1767 return res;
1768 }
1769
1770 private int consume_identifier (int s, bool quoted)
1771 {
1772 int pos = 1;
1773 int c;
1774
1775 id_builder [0] = (char) s;
1776
1777 while ((c = reader.Read ()) != -1) {
1778 if (is_identifier_part_character ((char) c)){
1779 if (pos == max_id_size){
1780 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
1781 return Token.ERROR;
1782 }
1783
1784 id_builder [pos++] = (char) c;
1785 putback_char = -1;
1786 col++;
1787 } else {
1788 putback_char = c;
1789 break;
1790 }
1791 }
1792
1793 //
1794 // Optimization: avoids doing the keyword lookup
1795 // on uppercase letters and _
1796 //
1797 if (!quoted && (s >= 'a' || s == '_')){
1798 int keyword = GetKeyword (id_builder, pos);
1799 if (keyword != -1)
1800 return keyword;
1801 }
1802
1803 //
1804 // Keep identifiers in an array of hashtables to avoid needless
1805 // allocations
1806 //
1807
1808 if (identifiers [pos] != null) {
1809 val = identifiers [pos][id_builder];
1810 if (val != null) {
1811 return Token.IDENTIFIER;
1812 }
1813 }
1814 else
1815 identifiers [pos] = new CharArrayHashtable (pos);
1816
1817 val = new String (id_builder, 0, pos);
1818
1819 char [] chars = new char [pos];
1820 Array.Copy (id_builder, chars, pos);
1821
1822 identifiers [pos] [chars] = val;
1823
1824 return Token.IDENTIFIER;
1825 }
1826
1827 public int xtoken ()
1828 {
1829 int t;
1830 bool doread = false;
1831 int c;
1832
1833 val = null;
1834 // optimization: eliminate col and implement #directive semantic correctly.
1835 for (;(c = getChar ()) != -1; col++) {
1836 if (c == ' ')
1837 continue;
1838
1839 if (c == '\t') {
1840 col = (((col + 8) / 8) * 8) - 1;
1841 continue;
1842 }
1843
1844 if (c == ' ' || c == '\f' || c == '\v' || c == 0xa0)
1845 continue;
1846
1847 if (c == '\r') {
1848 if (peekChar () == '\n')
1849 getChar ();
1850
1851 line++;
1852 ref_line++;
1853 col = 0;
1854 any_token_seen |= tokens_seen;
1855 tokens_seen = false;
1856 continue;
1857 }
1858
1859 // Handle double-slash comments.
1860 if (c == '/'){
1861 int d = peekChar ();
1862
1863 if (d == '/'){
1864 getChar ();
1865 while ((d = getChar ()) != -1 && (d != '\n') && d != '\r')
1866 col++;
1867 if (d == '\n'){
1868 line++;
1869 ref_line++;
1870 col = 0;
1871 }
1872 any_token_seen |= tokens_seen;
1873 tokens_seen = false;
1874 continue;
1875 } else if (d == '*'){
1876 getChar ();
1877
1878 while ((d = getChar ()) != -1){
1879 if (d == '*' && peekChar () == '/'){
1880 getChar ();
1881 col++;
1882 break;
1883 }
1884 if (d == '\n'){
1885 line++;
1886 ref_line++;
1887 col = 0;
1888 any_token_seen |= tokens_seen;
1889 tokens_seen = false;
1890 }
1891 }
1892 continue;
1893 }
1894 goto is_punct_label;
1895 }
1896
1897
1898 if (is_identifier_start_character ((char)c)){
1899 tokens_seen = true;
1900 return consume_identifier (c);
1901 }
1902
1903 is_punct_label:
1904 if ((t = is_punct ((char)c, ref doread)) != Token.ERROR){
1905 tokens_seen = true;
1906 if (doread){
1907 getChar ();
1908 col++;
1909 }
1910 return t;
1911 }
1912
1913 // white space
1914 if (c == '\n'){
1915 line++;
1916 ref_line++;
1917 col = 0;
1918 any_token_seen |= tokens_seen;
1919 tokens_seen = false;
1920 continue;
1921 }
1922
1923 if (c >= '0' && c <= '9'){
1924 tokens_seen = true;
1925 return is_number (c);
1926 }
1927
1928 if (c == '.'){
1929 tokens_seen = true;
1930 int peek = peekChar ();
1931 if (peek >= '0' && peek <= '9')
1932 return is_number (c);
1933 return Token.DOT;
1934 }
1935
1936 /* For now, ignore pre-processor commands */
1937 // FIXME: In C# the '#' is not limited to appear
1938 // on the first column.
1939 if (c == '#' && !tokens_seen){
1940 bool cont = true;
1941
1942 start_again:
1943
1944 cont = handle_preprocessing_directive (cont);
1945
1946 if (cont){
1947 col = 0;
1948 continue;
1949 }
1950 col = 1;
1951
1952 bool skipping = false;
1953 for (;(c = getChar ()) != -1; col++){
1954 if (c == '\n'){
1955 col = 0;
1956 line++;
1957 ref_line++;
1958 skipping = false;
1959 } else if (c == ' ' || c == '\t' || c == '\v' || c == '\r' || c == 0xa0)
1960 continue;
1961 else if (c != '#')
1962 skipping = true;
1963 if (c == '#' && !skipping)
1964 goto start_again;
1965 }
1966 any_token_seen |= tokens_seen;
1967 tokens_seen = false;
1968 if (c == -1)
1969 Report.Error (1027, Location, "#endif/#endregion expected");
1970 continue;
1971 }
1972
1973 if (c == '"')
1974 return consume_string (false);
1975
1976 if (c == '\''){
1977 c = getChar ();
1978 tokens_seen = true;
1979 if (c == '\''){
1980 error_details = "Empty character literal";
1981 Report.Error (1011, Location, error_details);
1982 return Token.ERROR;
1983 }
1984 c = escape (c);
1985 if (c == -1)
1986 return Token.ERROR;
1987 val = new System.Char ();
1988 val = (char) c;
1989 c = getChar ();
1990
1991 if (c != '\''){
1992 error_details = "Too many characters in character literal";
1993 Report.Error (1012, Location, error_details);
1994
1995 // Try to recover, read until newline or next "'"
1996 while ((c = getChar ()) != -1){
1997 if (c == '\n' || c == '\''){
1998 line++;
1999 ref_line++;
2000 col = 0;
2001 break;
2002 } else
2003 col++;
2004
2005 }
2006 return Token.ERROR;
2007 }
2008 return Token.LITERAL_CHARACTER;
2009 }
2010
2011 if (c == '@') {
2012 c = getChar ();
2013 if (c == '"') {
2014 tokens_seen = true;
2015 return consume_string (true);
2016 } else if (is_identifier_start_character ((char) c)){
2017 return consume_identifier (c, true);
2018 } else {
2019 Report.Error (1033, Location, "'@' must be followed by string constant or identifier");
2020 }
2021 }
2022
2023 if (c == '#') {
2024 error_details = "Preprocessor directives must appear as the first non-whitespace " +
2025 "character on a line.";
2026
2027 Report.Error (1040, Location, error_details);
2028
2029 return Token.ERROR;
2030 }
2031
2032 error_details = ((char)c).ToString ();
2033
2034 return Token.ERROR;
2035 }
2036
2037 return Token.EOF;
2038 }
2039
2040 public void cleanup ()
2041 {
2042 if (ifstack != null && ifstack.Count >= 1) {
2043 int state = (int) ifstack.Pop ();
2044 if ((state & REGION) != 0)
2045 Report.Error (1038, Location, "#endregion directive expected");
2046 else
2047 Report.Error (1027, "#endif directive expected");
2048 }
2049
2050 }
2051 }
2052 }
Mirror of the official Mono project git repository