?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
2009-12-09 Jb Evain <jbevain@novell.com>
[mcs.git] / mcs / cs-tokenizer.cs
blob80a27336c78acb30c05f7f9e3004796c5bb1820f
1 //
2 // cs-tokenizer.cs: The Tokenizer for the C# compiler
3 // This also implements the preprocessor
4 //
5 // Author: Miguel de Icaza (miguel@gnu.org)
6 // Marek Safar (marek.safar@seznam.cz)
7 //
8 // Dual licensed under the terms of the MIT X11 or GNU GPL
9 //
10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
11 // Copyright 2004-2008 Novell, Inc
12 //
13 //
14
15 using System;
16 using System.Text;
17 using System.Collections.Generic;
18 using System.IO;
19 using System.Globalization;
20 using System.Reflection;
21
22 namespace Mono.CSharp
23 {
24 /// <summary>
25 /// Tokenizer for C# source code.
26 /// </summary>
27
28 public class Tokenizer : yyParser.yyInput
29 {
30 class KeywordEntry
31 {
32 public readonly int Token;
33 public KeywordEntry Next;
34 public readonly char[] Value;
35
36 public KeywordEntry (string value, int token)
37 {
38 this.Value = value.ToCharArray ();
39 this.Token = token;
40 }
41 }
42
43 sealed class IdentifiersComparer : IEqualityComparer<char[]>
44 {
45 readonly int length;
46
47 public IdentifiersComparer (int length)
48 {
49 this.length = length;
50 }
51
52 public bool Equals (char[] x, char[] y)
53 {
54 for (int i = 0; i < length; ++i)
55 if (x [i] != y [i])
56 return false;
57
58 return true;
59 }
60
61 public int GetHashCode (char[] obj)
62 {
63 int h = 0;
64 for (int i = 0; i < length; ++i)
65 h = (h << 5) - h + obj [i];
66
67 return h;
68 }
69 }
70
71 //
72 // This class has to be used in the parser only, it reuses token
73 // details after each parse
74 //
75 public class LocatedToken
76 {
77 int row, column;
78 string value;
79
80 static LocatedToken[] buffer;
81 static int pos;
82
83 private LocatedToken ()
84 {
85 }
86
87 public static LocatedToken Create (int row, int column)
88 {
89 return Create (null, row, column);
90 }
91
92 public static LocatedToken Create (string value, int row, int column)
93 {
94 //
95 // TODO: I am not very happy about the logic but it's the best
96 // what I could come up with for now.
97 // Ideally we should be using just tiny buffer (256 elements) which
98 // is enough to hold all details for currect stack and recycle elements
99 // poped from the stack but there is a trick needed to recycle
100 // them properly.
101 //
102 LocatedToken entry;
103 if (pos >= buffer.Length) {
104 entry = new LocatedToken ();
105 } else {
106 entry = buffer [pos];
107 if (entry == null) {
108 entry = new LocatedToken ();
109 buffer [pos] = entry;
110 }
111
112 ++pos;
113 }
114 entry.value = value;
115 entry.row = row;
116 entry.column = column;
117 return entry;
118 }
119
120 public static void Initialize ()
121 {
122 if (buffer == null)
123 buffer = new LocatedToken [10000];
124 pos = 0;
125 }
126
127 public Location Location {
128 get { return new Location (row, column); }
129 }
130
131 public string Value {
132 get { return value; }
133 }
134 }
135
136 SeekableStreamReader reader;
137 SourceFile ref_name;
138 CompilationUnit file_name;
139 CompilerContext context;
140 bool hidden = false;
141 int ref_line = 1;
142 int line = 1;
143 int col = 0;
144 int previous_col;
145 int current_token;
146 bool handle_get_set = false;
147 bool handle_remove_add = false;
148 bool handle_where = false;
149 bool handle_typeof = false;
150 bool lambda_arguments_parsing;
151 Location current_comment_location = Location.Null;
152 List<Location> escaped_identifiers;
153 int parsing_generic_less_than;
154
155 //
156 // Used mainly for parser optimizations. Some expressions for instance
157 // can appear only in block (including initializer, base initializer)
158 // scope only
159 //
160 public int parsing_block;
161 internal bool query_parsing;
162
163 //
164 // When parsing type only, useful for ambiguous nullable types
165 //
166 public int parsing_type;
167
168 //
169 // Set when parsing generic declaration (type or method header)
170 //
171 public bool parsing_generic_declaration;
172
173 //
174 // The value indicates that we have not reach any declaration or
175 // namespace yet
176 //
177 public int parsing_declaration;
178
179 //
180 // The special character to inject on streams to trigger the EXPRESSION_PARSE
181 // token to be returned. It just happens to be a Unicode character that
182 // would never be part of a program (can not be an identifier).
183 //
184 // This character is only tested just before the tokenizer is about to report
185 // an error; So on the regular operation mode, this addition will have no
186 // impact on the tokenizer's performance.
187 //
188
189 public const int EvalStatementParserCharacter = 0x2190; // Unicode Left Arrow
190 public const int EvalCompilationUnitParserCharacter = 0x2191; // Unicode Arrow
191 public const int EvalUsingDeclarationsParserCharacter = 0x2192; // Unicode Arrow
192
193 //
194 // XML documentation buffer. The save point is used to divide
195 // comments on types and comments on members.
196 //
197 StringBuilder xml_comment_buffer;
198
199 //
200 // See comment on XmlCommentState enumeration.
201 //
202 XmlCommentState xml_doc_state = XmlCommentState.Allowed;
203
204 //
205 // Whether tokens have been seen on this line
206 //
207 bool tokens_seen = false;
208
209 //
210 // Set to true once the GENERATE_COMPLETION token has bee
211 // returned. This helps produce one GENERATE_COMPLETION,
212 // as many COMPLETE_COMPLETION as necessary to complete the
213 // AST tree and one final EOF.
214 //
215 bool generated;
216
217 //
218 // Whether a token has been seen on the file
219 // This is needed because `define' is not allowed to be used
220 // after a token has been seen.
221 //
222 bool any_token_seen = false;
223
224 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
225
226 public bool PropertyParsing {
227 get { return handle_get_set; }
228 set { handle_get_set = value; }
229 }
230
231 public bool EventParsing {
232 get { return handle_remove_add; }
233 set { handle_remove_add = value; }
234 }
235
236 public bool ConstraintsParsing {
237 get { return handle_where; }
238 set { handle_where = value; }
239 }
240
241 public bool TypeOfParsing {
242 get { return handle_typeof; }
243 set { handle_typeof = value; }
244 }
245
246 public XmlCommentState doc_state {
247 get { return xml_doc_state; }
248 set {
249 if (value == XmlCommentState.Allowed) {
250 check_incorrect_doc_comment ();
251 reset_doc_comment ();
252 }
253 xml_doc_state = value;
254 }
255 }
256
257 //
258 // This is used to trigger completion generation on the parser
259 public bool CompleteOnEOF;
260
261 void AddEscapedIdentifier (Location loc)
262 {
263 if (escaped_identifiers == null)
264 escaped_identifiers = new List<Location> ();
265
266 escaped_identifiers.Add (loc);
267 }
268
269 public bool IsEscapedIdentifier (Location loc)
270 {
271 if (escaped_identifiers != null) {
272 foreach (Location lt in escaped_identifiers)
273 if (lt.Equals (loc))
274 return true;
275 }
276
277 return false;
278 }
279
280 //
281 // Class variables
282 //
283 static KeywordEntry[][] keywords;
284 static Dictionary<string, object> keyword_strings; // TODO: HashSet
285 static NumberStyles styles;
286 static NumberFormatInfo csharp_format_info;
287
288 //
289 // Values for the associated token returned
290 //
291 internal int putback_char; // Used by repl only
292 Object val;
293
294 //
295 // Pre-processor
296 //
297 const int TAKING = 1;
298 const int ELSE_SEEN = 4;
299 const int PARENT_TAKING = 8;
300 const int REGION = 16;
301
302 //
303 // pre-processor if stack state:
304 //
305 Stack<int> ifstack;
306
307 static System.Text.StringBuilder string_builder;
308
309 const int max_id_size = 512;
310 static char [] id_builder = new char [max_id_size];
311
312 public static Dictionary<char[], string>[] identifiers = new Dictionary<char[], string>[max_id_size + 1];
313
314 const int max_number_size = 512;
315 static char [] number_builder = new char [max_number_size];
316 static int number_pos;
317
318 static StringBuilder static_cmd_arg = new System.Text.StringBuilder ();
319
320 //
321 // Details about the error encoutered by the tokenizer
322 //
323 string error_details;
324
325 public string error {
326 get {
327 return error_details;
328 }
329 }
330
331 public int Line {
332 get {
333 return ref_line;
334 }
335 }
336
337 //
338 // This is used when the tokenizer needs to save
339 // the current position as it needs to do some parsing
340 // on its own to deamiguate a token in behalf of the
341 // parser.
342 //
343 Stack<Position> position_stack = new Stack<Position> (2);
344
345 class Position {
346 public int position;
347 public int line;
348 public int ref_line;
349 public int col;
350 public bool hidden;
351 public int putback_char;
352 public int previous_col;
353 public Stack<int> ifstack;
354 public int parsing_generic_less_than;
355 public int current_token;
356
357 public Position (Tokenizer t)
358 {
359 position = t.reader.Position;
360 line = t.line;
361 ref_line = t.ref_line;
362 col = t.col;
363 hidden = t.hidden;
364 putback_char = t.putback_char;
365 previous_col = t.previous_col;
366 if (t.ifstack != null && t.ifstack.Count != 0) {
367 // There is no simple way to clone Stack<T> all
368 // methods reverse the order
369 var clone = t.ifstack.ToArray ();
370 Array.Reverse (clone);
371 ifstack = new Stack<int> (clone);
372 }
373 parsing_generic_less_than = t.parsing_generic_less_than;
374 current_token = t.current_token;
375 }
376 }
377
378 public void PushPosition ()
379 {
380 position_stack.Push (new Position (this));
381 }
382
383 public void PopPosition ()
384 {
385 Position p = position_stack.Pop ();
386
387 reader.Position = p.position;
388 ref_line = p.ref_line;
389 line = p.line;
390 col = p.col;
391 hidden = p.hidden;
392 putback_char = p.putback_char;
393 previous_col = p.previous_col;
394 ifstack = p.ifstack;
395 parsing_generic_less_than = p.parsing_generic_less_than;
396 current_token = p.current_token;
397 }
398
399 // Do not reset the position, ignore it.
400 public void DiscardPosition ()
401 {
402 position_stack.Pop ();
403 }
404
405 static void AddKeyword (string kw, int token)
406 {
407 keyword_strings.Add (kw, null);
408
409 int length = kw.Length;
410 if (keywords [length] == null) {
411 keywords [length] = new KeywordEntry ['z' - '_' + 1];
412 }
413
414 int char_index = kw [0] - '_';
415 KeywordEntry kwe = keywords [length] [char_index];
416 if (kwe == null) {
417 keywords [length] [char_index] = new KeywordEntry (kw, token);
418 return;
419 }
420
421 while (kwe.Next != null) {
422 kwe = kwe.Next;
423 }
424
425 kwe.Next = new KeywordEntry (kw, token);
426 }
427
428 static void InitTokens ()
429 {
430 keyword_strings = new Dictionary<string, object> ();
431
432 // 11 is the length of the longest keyword for now
433 keywords = new KeywordEntry [11] [];
434
435 AddKeyword ("__arglist", Token.ARGLIST);
436 AddKeyword ("abstract", Token.ABSTRACT);
437 AddKeyword ("as", Token.AS);
438 AddKeyword ("add", Token.ADD);
439 AddKeyword ("base", Token.BASE);
440 AddKeyword ("bool", Token.BOOL);
441 AddKeyword ("break", Token.BREAK);
442 AddKeyword ("byte", Token.BYTE);
443 AddKeyword ("case", Token.CASE);
444 AddKeyword ("catch", Token.CATCH);
445 AddKeyword ("char", Token.CHAR);
446 AddKeyword ("checked", Token.CHECKED);
447 AddKeyword ("class", Token.CLASS);
448 AddKeyword ("const", Token.CONST);
449 AddKeyword ("continue", Token.CONTINUE);
450 AddKeyword ("decimal", Token.DECIMAL);
451 AddKeyword ("default", Token.DEFAULT);
452 AddKeyword ("delegate", Token.DELEGATE);
453 AddKeyword ("do", Token.DO);
454 AddKeyword ("double", Token.DOUBLE);
455 AddKeyword ("else", Token.ELSE);
456 AddKeyword ("enum", Token.ENUM);
457 AddKeyword ("event", Token.EVENT);
458 AddKeyword ("explicit", Token.EXPLICIT);
459 AddKeyword ("extern", Token.EXTERN);
460 AddKeyword ("false", Token.FALSE);
461 AddKeyword ("finally", Token.FINALLY);
462 AddKeyword ("fixed", Token.FIXED);
463 AddKeyword ("float", Token.FLOAT);
464 AddKeyword ("for", Token.FOR);
465 AddKeyword ("foreach", Token.FOREACH);
466 AddKeyword ("goto", Token.GOTO);
467 AddKeyword ("get", Token.GET);
468 AddKeyword ("if", Token.IF);
469 AddKeyword ("implicit", Token.IMPLICIT);
470 AddKeyword ("in", Token.IN);
471 AddKeyword ("int", Token.INT);
472 AddKeyword ("interface", Token.INTERFACE);
473 AddKeyword ("internal", Token.INTERNAL);
474 AddKeyword ("is", Token.IS);
475 AddKeyword ("lock", Token.LOCK);
476 AddKeyword ("long", Token.LONG);
477 AddKeyword ("namespace", Token.NAMESPACE);
478 AddKeyword ("new", Token.NEW);
479 AddKeyword ("null", Token.NULL);
480 AddKeyword ("object", Token.OBJECT);
481 AddKeyword ("operator", Token.OPERATOR);
482 AddKeyword ("out", Token.OUT);
483 AddKeyword ("override", Token.OVERRIDE);
484 AddKeyword ("params", Token.PARAMS);
485 AddKeyword ("private", Token.PRIVATE);
486 AddKeyword ("protected", Token.PROTECTED);
487 AddKeyword ("public", Token.PUBLIC);
488 AddKeyword ("readonly", Token.READONLY);
489 AddKeyword ("ref", Token.REF);
490 AddKeyword ("remove", Token.REMOVE);
491 AddKeyword ("return", Token.RETURN);
492 AddKeyword ("sbyte", Token.SBYTE);
493 AddKeyword ("sealed", Token.SEALED);
494 AddKeyword ("set", Token.SET);
495 AddKeyword ("short", Token.SHORT);
496 AddKeyword ("sizeof", Token.SIZEOF);
497 AddKeyword ("stackalloc", Token.STACKALLOC);
498 AddKeyword ("static", Token.STATIC);
499 AddKeyword ("string", Token.STRING);
500 AddKeyword ("struct", Token.STRUCT);
501 AddKeyword ("switch", Token.SWITCH);
502 AddKeyword ("this", Token.THIS);
503 AddKeyword ("throw", Token.THROW);
504 AddKeyword ("true", Token.TRUE);
505 AddKeyword ("try", Token.TRY);
506 AddKeyword ("typeof", Token.TYPEOF);
507 AddKeyword ("uint", Token.UINT);
508 AddKeyword ("ulong", Token.ULONG);
509 AddKeyword ("unchecked", Token.UNCHECKED);
510 AddKeyword ("unsafe", Token.UNSAFE);
511 AddKeyword ("ushort", Token.USHORT);
512 AddKeyword ("using", Token.USING);
513 AddKeyword ("virtual", Token.VIRTUAL);
514 AddKeyword ("void", Token.VOID);
515 AddKeyword ("volatile", Token.VOLATILE);
516 AddKeyword ("while", Token.WHILE);
517 AddKeyword ("partial", Token.PARTIAL);
518 AddKeyword ("where", Token.WHERE);
519
520 // LINQ keywords
521 AddKeyword ("from", Token.FROM);
522 AddKeyword ("join", Token.JOIN);
523 AddKeyword ("on", Token.ON);
524 AddKeyword ("equals", Token.EQUALS);
525 AddKeyword ("select", Token.SELECT);
526 AddKeyword ("group", Token.GROUP);
527 AddKeyword ("by", Token.BY);
528 AddKeyword ("let", Token.LET);
529 AddKeyword ("orderby", Token.ORDERBY);
530 AddKeyword ("ascending", Token.ASCENDING);
531 AddKeyword ("descending", Token.DESCENDING);
532 AddKeyword ("into", Token.INTO);
533 }
534
535 //
536 // Class initializer
537 //
538 static Tokenizer ()
539 {
540 InitTokens ();
541 csharp_format_info = NumberFormatInfo.InvariantInfo;
542 styles = NumberStyles.Float;
543
544 string_builder = new System.Text.StringBuilder ();
545 }
546
547 int GetKeyword (char[] id, int id_len)
548 {
549 //
550 // Keywords are stored in an array of arrays grouped by their
551 // length and then by the first character
552 //
553 if (id_len >= keywords.Length || keywords [id_len] == null)
554 return -1;
555
556 int first_index = id [0] - '_';
557 if (first_index > 'z')
558 return -1;
559
560 KeywordEntry kwe = keywords [id_len] [first_index];
561 if (kwe == null)
562 return -1;
563
564 int res;
565 do {
566 res = kwe.Token;
567 for (int i = 1; i < id_len; ++i) {
568 if (id [i] != kwe.Value [i]) {
569 res = 0;
570 kwe = kwe.Next;
571 break;
572 }
573 }
574 } while (res == 0 && kwe != null);
575
576 if (res == 0)
577 return -1;
578
579 int next_token;
580 switch (res) {
581 case Token.GET:
582 case Token.SET:
583 if (!handle_get_set)
584 res = -1;
585 break;
586 case Token.REMOVE:
587 case Token.ADD:
588 if (!handle_remove_add)
589 res = -1;
590 break;
591 case Token.EXTERN:
592 if (parsing_declaration == 0)
593 res = Token.EXTERN_ALIAS;
594 break;
595 case Token.DEFAULT:
596 if (peek_token () == Token.COLON) {
597 token ();
598 res = Token.DEFAULT_COLON;
599 }
600 break;
601 case Token.WHERE:
602 if (!handle_where && !query_parsing)
603 res = -1;
604 break;
605 case Token.FROM:
606 //
607 // A query expression is any expression that starts with `from identifier'
608 // followed by any token except ; , =
609 //
610 if (!query_parsing) {
611 if (lambda_arguments_parsing) {
612 res = -1;
613 break;
614 }
615
616 PushPosition ();
617 // HACK: to disable generics micro-parser, because PushPosition does not
618 // store identifiers array
619 parsing_generic_less_than = 1;
620 switch (xtoken ()) {
621 case Token.IDENTIFIER:
622 case Token.INT:
623 case Token.BOOL:
624 case Token.BYTE:
625 case Token.CHAR:
626 case Token.DECIMAL:
627 case Token.FLOAT:
628 case Token.LONG:
629 case Token.OBJECT:
630 case Token.STRING:
631 case Token.UINT:
632 case Token.ULONG:
633 next_token = xtoken ();
634 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS)
635 goto default;
636
637 res = Token.FROM_FIRST;
638 query_parsing = true;
639 if (RootContext.Version <= LanguageVersion.ISO_2)
640 Report.FeatureIsNotAvailable (Location, "query expressions");
641 break;
642 case Token.VOID:
643 Expression.Error_VoidInvalidInTheContext (Location, Report);
644 break;
645 default:
646 PopPosition ();
647 // HACK: A token is not a keyword so we need to restore identifiers buffer
648 // which has been overwritten before we grabbed the identifier
649 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
650 return -1;
651 }
652 PopPosition ();
653 }
654 break;
655 case Token.JOIN:
656 case Token.ON:
657 case Token.EQUALS:
658 case Token.SELECT:
659 case Token.GROUP:
660 case Token.BY:
661 case Token.LET:
662 case Token.ORDERBY:
663 case Token.ASCENDING:
664 case Token.DESCENDING:
665 case Token.INTO:
666 if (!query_parsing)
667 res = -1;
668 break;
669
670 case Token.USING:
671 case Token.NAMESPACE:
672 // TODO: some explanation needed
673 check_incorrect_doc_comment ();
674 break;
675
676 case Token.PARTIAL:
677 if (parsing_block > 0) {
678 res = -1;
679 break;
680 }
681
682 // Save current position and parse next token.
683 PushPosition ();
684
685 next_token = token ();
686 bool ok = (next_token == Token.CLASS) ||
687 (next_token == Token.STRUCT) ||
688 (next_token == Token.INTERFACE) ||
689 (next_token == Token.VOID);
690
691 PopPosition ();
692
693 if (ok) {
694 if (next_token == Token.VOID) {
695 if (RootContext.Version == LanguageVersion.ISO_1 ||
696 RootContext.Version == LanguageVersion.ISO_2)
697 Report.FeatureIsNotAvailable (Location, "partial methods");
698 } else if (RootContext.Version == LanguageVersion.ISO_1)
699 Report.FeatureIsNotAvailable (Location, "partial types");
700
701 return res;
702 }
703
704 if (next_token < Token.LAST_KEYWORD) {
705 Report.Error (267, Location,
706 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
707 return token ();
708 }
709
710 res = -1;
711 break;
712 }
713
714 return res;
715 }
716
717 public Location Location {
718 get {
719 return new Location (ref_line, hidden ? -1 : col);
720 }
721 }
722
723 public Tokenizer (SeekableStreamReader input, CompilationUnit file, CompilerContext ctx)
724 {
725 this.ref_name = file;
726 this.file_name = file;
727 this.context = ctx;
728 reader = input;
729
730 putback_char = -1;
731
732 xml_comment_buffer = new StringBuilder ();
733
734 //
735 // FIXME: This could be `Location.Push' but we have to
736 // find out why the MS compiler allows this
737 //
738 Mono.CSharp.Location.Push (file, file);
739 }
740
741 static bool is_identifier_start_character (int c)
742 {
743 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
744 }
745
746 static bool is_identifier_part_character (char c)
747 {
748 if (c >= 'a' && c <= 'z')
749 return true;
750
751 if (c >= 'A' && c <= 'Z')
752 return true;
753
754 if (c == '_' || (c >= '0' && c <= '9'))
755 return true;
756
757 if (c < 0x80)
758 return false;
759
760 return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
761 }
762
763 public static bool IsKeyword (string s)
764 {
765 return keyword_strings.ContainsKey (s);
766 }
767
768 //
769 // Open parens micro parser. Detects both lambda and cast ambiguity.
770 //
771
772 int TokenizeOpenParens ()
773 {
774 int ptoken;
775 current_token = -1;
776
777 int bracket_level = 0;
778 bool is_type = false;
779 bool can_be_type = false;
780
781 while (true) {
782 ptoken = current_token;
783 token ();
784
785 switch (current_token) {
786 case Token.CLOSE_PARENS:
787 token ();
788
789 //
790 // Expression inside parens is lambda, (int i) =>
791 //
792 if (current_token == Token.ARROW) {
793 if (RootContext.Version <= LanguageVersion.ISO_2)
794 Report.FeatureIsNotAvailable (Location, "lambda expressions");
795
796 return Token.OPEN_PARENS_LAMBDA;
797 }
798
799 //
800 // Expression inside parens is single type, (int[])
801 //
802 if (is_type)
803 return Token.OPEN_PARENS_CAST;
804
805 //
806 // Expression is possible cast, look at next token, (T)null
807 //
808 if (can_be_type) {
809 switch (current_token) {
810 case Token.OPEN_PARENS:
811 case Token.BANG:
812 case Token.TILDE:
813 case Token.IDENTIFIER:
814 case Token.LITERAL:
815 case Token.BASE:
816 case Token.CHECKED:
817 case Token.DELEGATE:
818 case Token.FALSE:
819 case Token.FIXED:
820 case Token.NEW:
821 case Token.NULL:
822 case Token.SIZEOF:
823 case Token.THIS:
824 case Token.THROW:
825 case Token.TRUE:
826 case Token.TYPEOF:
827 case Token.UNCHECKED:
828 case Token.UNSAFE:
829 case Token.DEFAULT:
830
831 //
832 // These can be part of a member access
833 //
834 case Token.INT:
835 case Token.UINT:
836 case Token.SHORT:
837 case Token.USHORT:
838 case Token.LONG:
839 case Token.ULONG:
840 case Token.DOUBLE:
841 case Token.FLOAT:
842 case Token.CHAR:
843 case Token.BYTE:
844 case Token.DECIMAL:
845 case Token.BOOL:
846 return Token.OPEN_PARENS_CAST;
847 }
848 }
849 return Token.OPEN_PARENS;
850
851 case Token.DOT:
852 case Token.DOUBLE_COLON:
853 if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
854 goto default;
855
856 continue;
857
858 case Token.IDENTIFIER:
859 switch (ptoken) {
860 case Token.DOT:
861 case Token.OP_GENERICS_LT:
862 case Token.COMMA:
863 case Token.DOUBLE_COLON:
864 case -1:
865 if (bracket_level == 0)
866 can_be_type = true;
867 continue;
868 default:
869 can_be_type = is_type = false;
870 continue;
871 }
872
873 case Token.OBJECT:
874 case Token.STRING:
875 case Token.BOOL:
876 case Token.DECIMAL:
877 case Token.FLOAT:
878 case Token.DOUBLE:
879 case Token.SBYTE:
880 case Token.BYTE:
881 case Token.SHORT:
882 case Token.USHORT:
883 case Token.INT:
884 case Token.UINT:
885 case Token.LONG:
886 case Token.ULONG:
887 case Token.CHAR:
888 case Token.VOID:
889 if (bracket_level == 0)
890 is_type = true;
891 continue;
892
893 case Token.COMMA:
894 if (bracket_level == 0) {
895 bracket_level = 100;
896 can_be_type = is_type = false;
897 }
898 continue;
899
900 case Token.OP_GENERICS_LT:
901 case Token.OPEN_BRACKET:
902 if (bracket_level++ == 0)
903 is_type = true;
904 continue;
905
906 case Token.OP_GENERICS_GT:
907 case Token.CLOSE_BRACKET:
908 --bracket_level;
909 continue;
910
911 case Token.INTERR_NULLABLE:
912 case Token.STAR:
913 if (bracket_level == 0)
914 is_type = true;
915 continue;
916
917 case Token.REF:
918 case Token.OUT:
919 can_be_type = is_type = false;
920 continue;
921
922 default:
923 return Token.OPEN_PARENS;
924 }
925 }
926 }
927
928 public static bool IsValidIdentifier (string s)
929 {
930 if (s == null || s.Length == 0)
931 return false;
932
933 if (!is_identifier_start_character (s [0]))
934 return false;
935
936 for (int i = 1; i < s.Length; i ++)
937 if (! is_identifier_part_character (s [i]))
938 return false;
939
940 return true;
941 }
942
943 bool parse_less_than ()
944 {
945 start:
946 int the_token = token ();
947 if (the_token == Token.OPEN_BRACKET) {
948 do {
949 the_token = token ();
950 } while (the_token != Token.CLOSE_BRACKET);
951 the_token = token ();
952 } else if (the_token == Token.IN || the_token == Token.OUT) {
953 the_token = token ();
954 }
955 switch (the_token) {
956 case Token.IDENTIFIER:
957 case Token.OBJECT:
958 case Token.STRING:
959 case Token.BOOL:
960 case Token.DECIMAL:
961 case Token.FLOAT:
962 case Token.DOUBLE:
963 case Token.SBYTE:
964 case Token.BYTE:
965 case Token.SHORT:
966 case Token.USHORT:
967 case Token.INT:
968 case Token.UINT:
969 case Token.LONG:
970 case Token.ULONG:
971 case Token.CHAR:
972 case Token.VOID:
973 break;
974 case Token.OP_GENERICS_GT:
975 return true;
976
977 default:
978 return false;
979 }
980 again:
981 the_token = token ();
982
983 if (the_token == Token.OP_GENERICS_GT)
984 return true;
985 else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
986 goto start;
987 else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
988 goto again;
989 else if (the_token == Token.OP_GENERICS_LT) {
990 if (!parse_less_than ())
991 return false;
992 goto again;
993 } else if (the_token == Token.OPEN_BRACKET) {
994 rank_specifiers:
995 the_token = token ();
996 if (the_token == Token.CLOSE_BRACKET)
997 goto again;
998 else if (the_token == Token.COMMA)
999 goto rank_specifiers;
1000 return false;
1001 }
1002
1003 return false;
1004 }
1005
1006 bool parse_generic_dimension (out int dimension)
1007 {
1008 dimension = 1;
1009
1010 again:
1011 int the_token = token ();
1012 if (the_token == Token.OP_GENERICS_GT)
1013 return true;
1014 else if (the_token == Token.COMMA) {
1015 dimension++;
1016 goto again;
1017 }
1018
1019 return false;
1020 }
1021
1022 public int peek_token ()
1023 {
1024 int the_token;
1025
1026 PushPosition ();
1027 the_token = token ();
1028 PopPosition ();
1029
1030 return the_token;
1031 }
1032
1033 //
1034 // Tonizes `?' using custom disambiguous rules to return one
1035 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1036 //
1037 // Tricky expression look like:
1038 //
1039 // Foo ? a = x ? b : c;
1040 //
1041 int TokenizePossibleNullableType ()
1042 {
1043 if (parsing_block == 0 || parsing_type > 0)
1044 return Token.INTERR_NULLABLE;
1045
1046 int d = peek_char ();
1047 if (d == '?') {
1048 get_char ();
1049 return Token.OP_COALESCING;
1050 }
1051
1052 switch (current_token) {
1053 case Token.CLOSE_PARENS:
1054 case Token.TRUE:
1055 case Token.FALSE:
1056 case Token.NULL:
1057 case Token.LITERAL:
1058 return Token.INTERR;
1059 }
1060
1061 if (d != ' ') {
1062 if (d == ',' || d == ';' || d == '>')
1063 return Token.INTERR_NULLABLE;
1064 if (d == '*' || (d >= '0' && d <= '9'))
1065 return Token.INTERR;
1066 }
1067
1068 PushPosition ();
1069 current_token = Token.NONE;
1070 int next_token;
1071 switch (xtoken ()) {
1072 case Token.LITERAL:
1073 case Token.TRUE:
1074 case Token.FALSE:
1075 case Token.NULL:
1076 case Token.THIS:
1077 case Token.NEW:
1078 next_token = Token.INTERR;
1079 break;
1080
1081 case Token.SEMICOLON:
1082 case Token.COMMA:
1083 case Token.CLOSE_PARENS:
1084 case Token.OPEN_BRACKET:
1085 case Token.OP_GENERICS_GT:
1086 next_token = Token.INTERR_NULLABLE;
1087 break;
1088
1089 default:
1090 next_token = -1;
1091 break;
1092 }
1093
1094 if (next_token == -1) {
1095 switch (xtoken ()) {
1096 case Token.COMMA:
1097 case Token.SEMICOLON:
1098 case Token.OPEN_BRACE:
1099 case Token.CLOSE_PARENS:
1100 case Token.IN:
1101 next_token = Token.INTERR_NULLABLE;
1102 break;
1103
1104 case Token.COLON:
1105 next_token = Token.INTERR;
1106 break;
1107
1108 default:
1109 int ntoken;
1110 int interrs = 1;
1111 int colons = 0;
1112 //
1113 // All shorcuts failed, do it hard way
1114 //
1115 while ((ntoken = xtoken ()) != Token.EOF) {
1116 if (ntoken == Token.SEMICOLON)
1117 break;
1118
1119 if (ntoken == Token.COLON) {
1120 if (++colons == interrs)
1121 break;
1122 continue;
1123 }
1124
1125 if (ntoken == Token.INTERR) {
1126 ++interrs;
1127 continue;
1128 }
1129 }
1130
1131 next_token = colons != interrs ? Token.INTERR_NULLABLE : Token.INTERR;
1132 break;
1133 }
1134 }
1135
1136 PopPosition ();
1137 return next_token;
1138 }
1139
1140 bool decimal_digits (int c)
1141 {
1142 int d;
1143 bool seen_digits = false;
1144
1145 if (c != -1){
1146 if (number_pos == max_number_size)
1147 Error_NumericConstantTooLong ();
1148 number_builder [number_pos++] = (char) c;
1149 }
1150
1151 //
1152 // We use peek_char2, because decimal_digits needs to do a
1153 // 2-character look-ahead (5.ToString for example).
1154 //
1155 while ((d = peek_char2 ()) != -1){
1156 if (d >= '0' && d <= '9'){
1157 if (number_pos == max_number_size)
1158 Error_NumericConstantTooLong ();
1159 number_builder [number_pos++] = (char) d;
1160 get_char ();
1161 seen_digits = true;
1162 } else
1163 break;
1164 }
1165
1166 return seen_digits;
1167 }
1168
1169 static bool is_hex (int e)
1170 {
1171 return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1172 }
1173
1174 static TypeCode real_type_suffix (int c)
1175 {
1176 switch (c){
1177 case 'F': case 'f':
1178 return TypeCode.Single;
1179 case 'D': case 'd':
1180 return TypeCode.Double;
1181 case 'M': case 'm':
1182 return TypeCode.Decimal;
1183 default:
1184 return TypeCode.Empty;
1185 }
1186 }
1187
1188 int integer_type_suffix (ulong ul, int c)
1189 {
1190 bool is_unsigned = false;
1191 bool is_long = false;
1192
1193 if (c != -1){
1194 bool scanning = true;
1195 do {
1196 switch (c){
1197 case 'U': case 'u':
1198 if (is_unsigned)
1199 scanning = false;
1200 is_unsigned = true;
1201 get_char ();
1202 break;
1203
1204 case 'l':
1205 if (!is_unsigned){
1206 //
1207 // if we have not seen anything in between
1208 // report this error
1209 //
1210 Report.Warning (78, 4, Location, "The 'l' suffix is easily confused with the digit '1' (use 'L' for clarity)");
1211 }
1212
1213 goto case 'L';
1214
1215 case 'L':
1216 if (is_long)
1217 scanning = false;
1218 is_long = true;
1219 get_char ();
1220 break;
1221
1222 default:
1223 scanning = false;
1224 break;
1225 }
1226 c = peek_char ();
1227 } while (scanning);
1228 }
1229
1230 if (is_long && is_unsigned){
1231 val = new ULongLiteral (ul, Location);
1232 return Token.LITERAL;
1233 }
1234
1235 if (is_unsigned){
1236 // uint if possible, or ulong else.
1237
1238 if ((ul & 0xffffffff00000000) == 0)
1239 val = new UIntLiteral ((uint) ul, Location);
1240 else
1241 val = new ULongLiteral (ul, Location);
1242 } else if (is_long){
1243 // long if possible, ulong otherwise
1244 if ((ul & 0x8000000000000000) != 0)
1245 val = new ULongLiteral (ul, Location);
1246 else
1247 val = new LongLiteral ((long) ul, Location);
1248 } else {
1249 // int, uint, long or ulong in that order
1250 if ((ul & 0xffffffff00000000) == 0){
1251 uint ui = (uint) ul;
1252
1253 if ((ui & 0x80000000) != 0)
1254 val = new UIntLiteral (ui, Location);
1255 else
1256 val = new IntLiteral ((int) ui, Location);
1257 } else {
1258 if ((ul & 0x8000000000000000) != 0)
1259 val = new ULongLiteral (ul, Location);
1260 else
1261 val = new LongLiteral ((long) ul, Location);
1262 }
1263 }
1264 return Token.LITERAL;
1265 }
1266
1267 //
1268 // given `c' as the next char in the input decide whether
1269 // we need to convert to a special type, and then choose
1270 // the best representation for the integer
1271 //
1272 int adjust_int (int c)
1273 {
1274 try {
1275 if (number_pos > 9){
1276 ulong ul = (uint) (number_builder [0] - '0');
1277
1278 for (int i = 1; i < number_pos; i++){
1279 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1280 }
1281 return integer_type_suffix (ul, c);
1282 } else {
1283 uint ui = (uint) (number_builder [0] - '0');
1284
1285 for (int i = 1; i < number_pos; i++){
1286 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1287 }
1288 return integer_type_suffix (ui, c);
1289 }
1290 } catch (OverflowException) {
1291 error_details = "Integral constant is too large";
1292 Report.Error (1021, Location, error_details);
1293 val = new IntLiteral (0, Location);
1294 return Token.LITERAL;
1295 }
1296 catch (FormatException) {
1297 Report.Error (1013, Location, "Invalid number");
1298 val = new IntLiteral (0, Location);
1299 return Token.LITERAL;
1300 }
1301 }
1302
1303 int adjust_real (TypeCode t)
1304 {
1305 string s = new String (number_builder, 0, number_pos);
1306 const string error_details = "Floating-point constant is outside the range of type `{0}'";
1307
1308 switch (t){
1309 case TypeCode.Decimal:
1310 try {
1311 val = new DecimalLiteral (decimal.Parse (s, styles, csharp_format_info), Location);
1312 } catch (OverflowException) {
1313 val = new DecimalLiteral (0, Location);
1314 Report.Error (594, Location, error_details, "decimal");
1315 }
1316 break;
1317 case TypeCode.Single:
1318 try {
1319 val = new FloatLiteral (float.Parse (s, styles, csharp_format_info), Location);
1320 } catch (OverflowException) {
1321 val = new FloatLiteral (0, Location);
1322 Report.Error (594, Location, error_details, "float");
1323 }
1324 break;
1325 default:
1326 try {
1327 val = new DoubleLiteral (double.Parse (s, styles, csharp_format_info), Location);
1328 } catch (OverflowException) {
1329 val = new DoubleLiteral (0, Location);
1330 Report.Error (594, Location, error_details, "double");
1331 }
1332 break;
1333 }
1334
1335 return Token.LITERAL;
1336 }
1337
1338 int handle_hex ()
1339 {
1340 int d;
1341 ulong ul;
1342
1343 get_char ();
1344 while ((d = peek_char ()) != -1){
1345 if (is_hex (d)){
1346 number_builder [number_pos++] = (char) d;
1347 get_char ();
1348 } else
1349 break;
1350 }
1351
1352 string s = new String (number_builder, 0, number_pos);
1353 try {
1354 if (number_pos <= 8)
1355 ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1356 else
1357 ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1358 } catch (OverflowException){
1359 error_details = "Integral constant is too large";
1360 Report.Error (1021, Location, error_details);
1361 val = new IntLiteral (0, Location);
1362 return Token.LITERAL;
1363 }
1364 catch (FormatException) {
1365 Report.Error (1013, Location, "Invalid number");
1366 val = new IntLiteral (0, Location);
1367 return Token.LITERAL;
1368 }
1369
1370 return integer_type_suffix (ul, peek_char ());
1371 }
1372
1373 //
1374 // Invoked if we know we have .digits or digits
1375 //
1376 int is_number (int c)
1377 {
1378 bool is_real = false;
1379
1380 number_pos = 0;
1381
1382 if (c >= '0' && c <= '9'){
1383 if (c == '0'){
1384 int peek = peek_char ();
1385
1386 if (peek == 'x' || peek == 'X')
1387 return handle_hex ();
1388 }
1389 decimal_digits (c);
1390 c = get_char ();
1391 }
1392
1393 //
1394 // We need to handle the case of
1395 // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1396 //
1397 if (c == '.'){
1398 if (decimal_digits ('.')){
1399 is_real = true;
1400 c = get_char ();
1401 } else {
1402 putback ('.');
1403 number_pos--;
1404 return adjust_int (-1);
1405 }
1406 }
1407
1408 if (c == 'e' || c == 'E'){
1409 is_real = true;
1410 if (number_pos == max_number_size)
1411 Error_NumericConstantTooLong ();
1412 number_builder [number_pos++] = 'e';
1413 c = get_char ();
1414
1415 if (c == '+'){
1416 if (number_pos == max_number_size)
1417 Error_NumericConstantTooLong ();
1418 number_builder [number_pos++] = '+';
1419 c = -1;
1420 } else if (c == '-') {
1421 if (number_pos == max_number_size)
1422 Error_NumericConstantTooLong ();
1423 number_builder [number_pos++] = '-';
1424 c = -1;
1425 } else {
1426 if (number_pos == max_number_size)
1427 Error_NumericConstantTooLong ();
1428 number_builder [number_pos++] = '+';
1429 }
1430
1431 decimal_digits (c);
1432 c = get_char ();
1433 }
1434
1435 var type = real_type_suffix (c);
1436 if (type == TypeCode.Empty && !is_real){
1437 putback (c);
1438 return adjust_int (c);
1439 }
1440
1441 is_real = true;
1442
1443 if (type == TypeCode.Empty){
1444 putback (c);
1445 }
1446
1447 if (is_real)
1448 return adjust_real (type);
1449
1450 throw new Exception ("Is Number should never reach this point");
1451 }
1452
1453 //
1454 // Accepts exactly count (4 or 8) hex, no more no less
1455 //
1456 int getHex (int count, out int surrogate, out bool error)
1457 {
1458 int i;
1459 int total = 0;
1460 int c;
1461 int top = count != -1 ? count : 4;
1462
1463 get_char ();
1464 error = false;
1465 surrogate = 0;
1466 for (i = 0; i < top; i++){
1467 c = get_char ();
1468
1469 if (c >= '0' && c <= '9')
1470 c = (int) c - (int) '0';
1471 else if (c >= 'A' && c <= 'F')
1472 c = (int) c - (int) 'A' + 10;
1473 else if (c >= 'a' && c <= 'f')
1474 c = (int) c - (int) 'a' + 10;
1475 else {
1476 error = true;
1477 return 0;
1478 }
1479
1480 total = (total * 16) + c;
1481 if (count == -1){
1482 int p = peek_char ();
1483 if (p == -1)
1484 break;
1485 if (!is_hex ((char)p))
1486 break;
1487 }
1488 }
1489
1490 if (top == 8) {
1491 if (total > 0x0010FFFF) {
1492 error = true;
1493 return 0;
1494 }
1495
1496 if (total >= 0x00010000) {
1497 surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);
1498 total = ((total - 0x00010000) / 0x0400 + 0xD800);
1499 }
1500 }
1501
1502 return total;
1503 }
1504
1505 int escape (int c, out int surrogate)
1506 {
1507 bool error;
1508 int d;
1509 int v;
1510
1511 d = peek_char ();
1512 if (c != '\\') {
1513 surrogate = 0;
1514 return c;
1515 }
1516
1517 switch (d){
1518 case 'a':
1519 v = '\a'; break;
1520 case 'b':
1521 v = '\b'; break;
1522 case 'n':
1523 v = '\n'; break;
1524 case 't':
1525 v = '\t'; break;
1526 case 'v':
1527 v = '\v'; break;
1528 case 'r':
1529 v = '\r'; break;
1530 case '\\':
1531 v = '\\'; break;
1532 case 'f':
1533 v = '\f'; break;
1534 case '0':
1535 v = 0; break;
1536 case '"':
1537 v = '"'; break;
1538 case '\'':
1539 v = '\''; break;
1540 case 'x':
1541 v = getHex (-1, out surrogate, out error);
1542 if (error)
1543 goto default;
1544 return v;
1545 case 'u':
1546 case 'U':
1547 return EscapeUnicode (d, out surrogate);
1548 default:
1549 surrogate = 0;
1550 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1551 return d;
1552 }
1553
1554 get_char ();
1555 surrogate = 0;
1556 return v;
1557 }
1558
1559 int EscapeUnicode (int ch, out int surrogate)
1560 {
1561 bool error;
1562 if (ch == 'U') {
1563 ch = getHex (8, out surrogate, out error);
1564 } else {
1565 ch = getHex (4, out surrogate, out error);
1566 }
1567
1568 if (error)
1569 Report.Error (1009, Location, "Unrecognized escape sequence");
1570
1571 return ch;
1572 }
1573
1574 int get_char ()
1575 {
1576 int x;
1577 if (putback_char != -1) {
1578 x = putback_char;
1579 putback_char = -1;
1580 } else
1581 x = reader.Read ();
1582 if (x == '\n') {
1583 advance_line ();
1584 } else {
1585 col++;
1586 }
1587 return x;
1588 }
1589
1590 void advance_line ()
1591 {
1592 line++;
1593 ref_line++;
1594 previous_col = col;
1595 col = 0;
1596 }
1597
1598 int peek_char ()
1599 {
1600 if (putback_char == -1)
1601 putback_char = reader.Read ();
1602 return putback_char;
1603 }
1604
1605 int peek_char2 ()
1606 {
1607 if (putback_char != -1)
1608 return putback_char;
1609 return reader.Peek ();
1610 }
1611
1612 void putback (int c)
1613 {
1614 if (putback_char != -1){
1615 Console.WriteLine ("Col: " + col);
1616 Console.WriteLine ("Row: " + line);
1617 Console.WriteLine ("Name: " + ref_name.Name);
1618 Console.WriteLine ("Current [{0}] putting back [{1}] ", putback_char, c);
1619 throw new Exception ("This should not happen putback on putback");
1620 }
1621 if (c == '\n' || col == 0) {
1622 // It won't happen though.
1623 line--;
1624 ref_line--;
1625 col = previous_col;
1626 }
1627 else
1628 col--;
1629 putback_char = c;
1630 }
1631
1632 public bool advance ()
1633 {
1634 return peek_char () != -1 || CompleteOnEOF;
1635 }
1636
1637 public Object Value {
1638 get {
1639 return val;
1640 }
1641 }
1642
1643 public Object value ()
1644 {
1645 return val;
1646 }
1647
1648 public int token ()
1649 {
1650 current_token = xtoken ();
1651 return current_token;
1652 }
1653
1654 void get_cmd_arg (out string cmd, out string arg)
1655 {
1656 int c;
1657
1658 tokens_seen = false;
1659 arg = "";
1660
1661 // skip over white space
1662 do {
1663 c = get_char ();
1664 } while (c == '\r' || c == ' ' || c == '\t');
1665
1666 static_cmd_arg.Length = 0;
1667 while (c != -1 && is_identifier_part_character ((char)c)) {
1668 static_cmd_arg.Append ((char)c);
1669 c = get_char ();
1670 if (c == '\\') {
1671 int peek = peek_char ();
1672 if (peek == 'U' || peek == 'u') {
1673 int surrogate;
1674 c = EscapeUnicode (c, out surrogate);
1675 if (surrogate != 0) {
1676 if (is_identifier_part_character ((char) c))
1677 static_cmd_arg.Append ((char) c);
1678 c = surrogate;
1679 }
1680 }
1681 }
1682 }
1683
1684 cmd = static_cmd_arg.ToString ();
1685
1686 // skip over white space
1687 while (c == '\r' || c == ' ' || c == '\t')
1688 c = get_char ();
1689
1690 static_cmd_arg.Length = 0;
1691 int has_identifier_argument = 0;
1692
1693 while (c != -1 && c != '\n' && c != '\r') {
1694 if (c == '\\' && has_identifier_argument >= 0) {
1695 if (has_identifier_argument != 0 || (cmd == "define" || cmd == "if" || cmd == "elif" || cmd == "undef")) {
1696 has_identifier_argument = 1;
1697
1698 int peek = peek_char ();
1699 if (peek == 'U' || peek == 'u') {
1700 int surrogate;
1701 c = EscapeUnicode (c, out surrogate);
1702 if (surrogate != 0) {
1703 if (is_identifier_part_character ((char) c))
1704 static_cmd_arg.Append ((char) c);
1705 c = surrogate;
1706 }
1707 }
1708 } else {
1709 has_identifier_argument = -1;
1710 }
1711 }
1712 static_cmd_arg.Append ((char) c);
1713 c = get_char ();
1714 }
1715
1716 if (static_cmd_arg.Length != 0)
1717 arg = static_cmd_arg.ToString ();
1718 }
1719
1720 //
1721 // Handles the #line directive
1722 //
1723 bool PreProcessLine (string arg)
1724 {
1725 if (arg.Length == 0)
1726 return false;
1727
1728 if (arg == "default"){
1729 ref_line = line;
1730 ref_name = file_name;
1731 hidden = false;
1732 Location.Push (file_name, ref_name);
1733 return true;
1734 } else if (arg == "hidden"){
1735 hidden = true;
1736 return true;
1737 }
1738
1739 try {
1740 int pos;
1741
1742 if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){
1743 ref_line = System.Int32.Parse (arg.Substring (0, pos));
1744 pos++;
1745
1746 char [] quotes = { '\"' };
1747
1748 string name = arg.Substring (pos). Trim (quotes);
1749 ref_name = Location.LookupFile (file_name, name);
1750 file_name.AddFile (ref_name);
1751 hidden = false;
1752 Location.Push (file_name, ref_name);
1753 } else {
1754 ref_line = System.Int32.Parse (arg);
1755 hidden = false;
1756 }
1757 } catch {
1758 return false;
1759 }
1760
1761 return true;
1762 }
1763
1764 //
1765 // Handles #define and #undef
1766 //
1767 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking)
1768 {
1769 if (ident.Length == 0 || ident == "true" || ident == "false"){
1770 Report.Error (1001, Location, "Missing identifier to pre-processor directive");
1771 return;
1772 }
1773
1774 if (ident.IndexOfAny (simple_whitespaces) != -1){
1775 Error_EndLineExpected ();
1776 return;
1777 }
1778
1779 if (!is_identifier_start_character (ident [0]))
1780 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1781
1782 foreach (char c in ident.Substring (1)){
1783 if (!is_identifier_part_character (c)){
1784 Report.Error (1001, Location, "Identifier expected: {0}", ident);
1785 return;
1786 }
1787 }
1788
1789 if (!caller_is_taking)
1790 return;
1791
1792 if (is_define) {
1793 //
1794 // #define ident
1795 //
1796 if (RootContext.IsConditionalDefined (ident))
1797 return;
1798
1799 file_name.AddDefine (ident);
1800 } else {
1801 //
1802 // #undef ident
1803 //
1804 file_name.AddUndefine (ident);
1805 }
1806 }
1807
1808 static byte read_hex (string arg, int pos, out bool error)
1809 {
1810 error = false;
1811
1812 int total;
1813 char c = arg [pos];
1814
1815 if ((c >= '0') && (c <= '9'))
1816 total = (int) c - (int) '0';
1817 else if ((c >= 'A') && (c <= 'F'))
1818 total = (int) c - (int) 'A' + 10;
1819 else if ((c >= 'a') && (c <= 'f'))
1820 total = (int) c - (int) 'a' + 10;
1821 else {
1822 error = true;
1823 return 0;
1824 }
1825
1826 total *= 16;
1827 c = arg [pos+1];
1828
1829 if ((c >= '0') && (c <= '9'))
1830 total += (int) c - (int) '0';
1831 else if ((c >= 'A') && (c <= 'F'))
1832 total += (int) c - (int) 'A' + 10;
1833 else if ((c >= 'a') && (c <= 'f'))
1834 total += (int) c - (int) 'a' + 10;
1835 else {
1836 error = true;
1837 return 0;
1838 }
1839
1840 return (byte) total;
1841 }
1842
1843 /// <summary>
1844 /// Handles #pragma checksum
1845 /// </summary>
1846 bool PreProcessPragmaChecksum (string arg)
1847 {
1848 if ((arg [0] != ' ') && (arg [0] != '\t'))
1849 return false;
1850
1851 arg = arg.Trim (simple_whitespaces);
1852 if ((arg.Length < 2) || (arg [0] != '"'))
1853 return false;
1854
1855 StringBuilder file_sb = new StringBuilder ();
1856
1857 int pos = 1;
1858 char ch;
1859 while ((ch = arg [pos++]) != '"') {
1860 if (pos >= arg.Length)
1861 return false;
1862
1863 if (ch == '\\') {
1864 if (pos+1 >= arg.Length)
1865 return false;
1866 ch = arg [pos++];
1867 }
1868
1869 file_sb.Append (ch);
1870 }
1871
1872 if ((pos+2 >= arg.Length) || ((arg [pos] != ' ') && (arg [pos] != '\t')))
1873 return false;
1874
1875 arg = arg.Substring (pos).Trim (simple_whitespaces);
1876 if ((arg.Length < 42) || (arg [0] != '"') || (arg [1] != '{') ||
1877 (arg [10] != '-') || (arg [15] != '-') || (arg [20] != '-') ||
1878 (arg [25] != '-') || (arg [38] != '}') || (arg [39] != '"'))
1879 return false;
1880
1881 bool error;
1882 byte[] guid_bytes = new byte [16];
1883
1884 for (int i = 0; i < 4; i++) {
1885 guid_bytes [i] = read_hex (arg, 2+2*i, out error);
1886 if (error)
1887 return false;
1888 }
1889 for (int i = 0; i < 2; i++) {
1890 guid_bytes [i+4] = read_hex (arg, 11+2*i, out error);
1891 if (error)
1892 return false;
1893 guid_bytes [i+6] = read_hex (arg, 16+2*i, out error);
1894 if (error)
1895 return false;
1896 guid_bytes [i+8] = read_hex (arg, 21+2*i, out error);
1897 if (error)
1898 return false;
1899 }
1900
1901 for (int i = 0; i < 6; i++) {
1902 guid_bytes [i+10] = read_hex (arg, 26+2*i, out error);
1903 if (error)
1904 return false;
1905 }
1906
1907 arg = arg.Substring (40).Trim (simple_whitespaces);
1908 if ((arg.Length < 34) || (arg [0] != '"') || (arg [33] != '"'))
1909 return false;
1910
1911 byte[] checksum_bytes = new byte [16];
1912 for (int i = 0; i < 16; i++) {
1913 checksum_bytes [i] = read_hex (arg, 1+2*i, out error);
1914 if (error)
1915 return false;
1916 }
1917
1918 arg = arg.Substring (34).Trim (simple_whitespaces);
1919 if (arg.Length > 0)
1920 return false;
1921
1922 SourceFile file = Location.LookupFile (file_name, file_sb.ToString ());
1923 file.SetChecksum (guid_bytes, checksum_bytes);
1924 ref_name.AutoGenerated = true;
1925 return true;
1926 }
1927
1928 /// <summary>
1929 /// Handles #pragma directive
1930 /// </summary>
1931 void PreProcessPragma (string arg)
1932 {
1933 const string warning = "warning";
1934 const string w_disable = "warning disable";
1935 const string w_restore = "warning restore";
1936 const string checksum = "checksum";
1937
1938 if (arg == w_disable) {
1939 Report.RegisterWarningRegion (Location).WarningDisable (Location.Row);
1940 return;
1941 }
1942
1943 if (arg == w_restore) {
1944 Report.RegisterWarningRegion (Location).WarningEnable (Location.Row);
1945 return;
1946 }
1947
1948 if (arg.StartsWith (w_disable)) {
1949 int[] codes = ParseNumbers (arg.Substring (w_disable.Length));
1950 foreach (int code in codes) {
1951 if (code != 0)
1952 Report.RegisterWarningRegion (Location).WarningDisable (Location, code, Report);
1953 }
1954 return;
1955 }
1956
1957 if (arg.StartsWith (w_restore)) {
1958 int[] codes = ParseNumbers (arg.Substring (w_restore.Length));
1959 var w_table = Report.warning_ignore_table;
1960 foreach (int code in codes) {
1961 if (w_table != null && w_table.ContainsKey (code))
1962 Report.Warning (1635, 1, Location, "Cannot restore warning `CS{0:0000}' because it was disabled globally", code);
1963 Report.RegisterWarningRegion (Location).WarningEnable (Location, code, Report);
1964 }
1965 return;
1966 }
1967
1968 if (arg.StartsWith (warning)) {
1969 Report.Warning (1634, 1, Location, "Expected disable or restore");
1970 return;
1971 }
1972
1973 if (arg.StartsWith (checksum)) {
1974 if (!PreProcessPragmaChecksum (arg.Substring (checksum.Length)))
1975 Warning_InvalidPragmaChecksum ();
1976 return;
1977 }
1978
1979 Report.Warning (1633, 1, Location, "Unrecognized #pragma directive");
1980 }
1981
1982 int[] ParseNumbers (string text)
1983 {
1984 string[] string_array = text.Split (',');
1985 int[] values = new int [string_array.Length];
1986 int index = 0;
1987 foreach (string string_code in string_array) {
1988 try {
1989 values[index++] = int.Parse (string_code, System.Globalization.CultureInfo.InvariantCulture);
1990 }
1991 catch (FormatException) {
1992 Report.Warning (1692, 1, Location, "Invalid number");
1993 }
1994 }
1995 return values;
1996 }
1997
1998 bool eval_val (string s)
1999 {
2000 if (s == "true")
2001 return true;
2002 if (s == "false")
2003 return false;
2004
2005 return file_name.IsConditionalDefined (s);
2006 }
2007
2008 bool pp_primary (ref string s)
2009 {
2010 s = s.Trim ();
2011 int len = s.Length;
2012
2013 if (len > 0){
2014 char c = s [0];
2015
2016 if (c == '('){
2017 s = s.Substring (1);
2018 bool val = pp_expr (ref s, false);
2019 if (s.Length > 0 && s [0] == ')'){
2020 s = s.Substring (1);
2021 return val;
2022 }
2023 Error_InvalidDirective ();
2024 return false;
2025 }
2026
2027 if (is_identifier_start_character (c)){
2028 int j = 1;
2029
2030 while (j < len){
2031 c = s [j];
2032
2033 if (is_identifier_part_character (c)){
2034 j++;
2035 continue;
2036 }
2037 bool v = eval_val (s.Substring (0, j));
2038 s = s.Substring (j);
2039 return v;
2040 }
2041 bool vv = eval_val (s);
2042 s = "";
2043 return vv;
2044 }
2045 }
2046 Error_InvalidDirective ();
2047 return false;
2048 }
2049
2050 bool pp_unary (ref string s)
2051 {
2052 s = s.Trim ();
2053 int len = s.Length;
2054
2055 if (len > 0){
2056 if (s [0] == '!'){
2057 if (len > 1 && s [1] == '='){
2058 Error_InvalidDirective ();
2059 return false;
2060 }
2061 s = s.Substring (1);
2062 return ! pp_primary (ref s);
2063 } else
2064 return pp_primary (ref s);
2065 } else {
2066 Error_InvalidDirective ();
2067 return false;
2068 }
2069 }
2070
2071 bool pp_eq (ref string s)
2072 {
2073 bool va = pp_unary (ref s);
2074
2075 s = s.Trim ();
2076 int len = s.Length;
2077 if (len > 0){
2078 if (s [0] == '='){
2079 if (len > 2 && s [1] == '='){
2080 s = s.Substring (2);
2081 return va == pp_unary (ref s);
2082 } else {
2083 Error_InvalidDirective ();
2084 return false;
2085 }
2086 } else if (s [0] == '!' && len > 1 && s [1] == '='){
2087 s = s.Substring (2);
2088
2089 return va != pp_unary (ref s);
2090
2091 }
2092 }
2093
2094 return va;
2095
2096 }
2097
2098 bool pp_and (ref string s)
2099 {
2100 bool va = pp_eq (ref s);
2101
2102 s = s.Trim ();
2103 int len = s.Length;
2104 if (len > 0){
2105 if (s [0] == '&'){
2106 if (len > 2 && s [1] == '&'){
2107 s = s.Substring (2);
2108 return (va & pp_and (ref s));
2109 } else {
2110 Error_InvalidDirective ();
2111 return false;
2112 }
2113 }
2114 }
2115 return va;
2116 }
2117
2118 //
2119 // Evaluates an expression for `#if' or `#elif'
2120 //
2121 bool pp_expr (ref string s, bool isTerm)
2122 {
2123 bool va = pp_and (ref s);
2124 s = s.Trim ();
2125 int len = s.Length;
2126 if (len > 0){
2127 char c = s [0];
2128
2129 if (c == '|'){
2130 if (len > 2 && s [1] == '|'){
2131 s = s.Substring (2);
2132 return va | pp_expr (ref s, isTerm);
2133 } else {
2134 Error_InvalidDirective ();
2135 return false;
2136 }
2137 }
2138 if (isTerm) {
2139 Error_EndLineExpected ();
2140 return false;
2141 }
2142 }
2143
2144 return va;
2145 }
2146
2147 bool eval (string s)
2148 {
2149 bool v = pp_expr (ref s, true);
2150 s = s.Trim ();
2151 if (s.Length != 0){
2152 return false;
2153 }
2154
2155 return v;
2156 }
2157
2158 void Error_NumericConstantTooLong ()
2159 {
2160 Report.Error (1021, Location, "Numeric constant too long");
2161 }
2162
2163 void Error_InvalidDirective ()
2164 {
2165 Report.Error (1517, Location, "Invalid preprocessor directive");
2166 }
2167
2168 void Error_UnexpectedDirective (string extra)
2169 {
2170 Report.Error (
2171 1028, Location,
2172 "Unexpected processor directive ({0})", extra);
2173 }
2174
2175 void Error_TokensSeen ()
2176 {
2177 Report.Error (1032, Location,
2178 "Cannot define or undefine preprocessor symbols after first token in file");
2179 }
2180
2181 void Eror_WrongPreprocessorLocation ()
2182 {
2183 Report.Error (1040, Location,
2184 "Preprocessor directives must appear as the first non-whitespace character on a line");
2185 }
2186
2187 void Error_EndLineExpected ()
2188 {
2189 Report.Error (1025, Location, "Single-line comment or end-of-line expected");
2190 }
2191
2192 void Warning_InvalidPragmaChecksum ()
2193 {
2194 Report.Warning (1695, 1, Location,
2195 "Invalid #pragma checksum syntax; should be " +
2196 "#pragma checksum \"filename\" " +
2197 "\"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\"");
2198 }
2199 //
2200 // if true, then the code continues processing the code
2201 // if false, the code stays in a loop until another directive is
2202 // reached.
2203 // When caller_is_taking is false we ignore all directives except the ones
2204 // which can help us to identify where the #if block ends
2205 bool handle_preprocessing_directive (bool caller_is_taking)
2206 {
2207 string cmd, arg;
2208 bool region_directive = false;
2209
2210 get_cmd_arg (out cmd, out arg);
2211
2212 // Eat any trailing whitespaces and single-line comments
2213 if (arg.IndexOf ("//") != -1)
2214 arg = arg.Substring (0, arg.IndexOf ("//"));
2215 arg = arg.Trim (simple_whitespaces);
2216
2217 //
2218 // The first group of pre-processing instructions is always processed
2219 //
2220 switch (cmd){
2221 case "region":
2222 region_directive = true;
2223 arg = "true";
2224 goto case "if";
2225
2226 case "endregion":
2227 if (ifstack == null || ifstack.Count == 0){
2228 Error_UnexpectedDirective ("no #region for this #endregion");
2229 return true;
2230 }
2231 int pop = ifstack.Pop ();
2232
2233 if ((pop & REGION) == 0)
2234 Report.Error (1027, Location, "Expected `#endif' directive");
2235
2236 return caller_is_taking;
2237
2238 case "if":
2239 if (ifstack == null)
2240 ifstack = new Stack<int> (2);
2241
2242 int flags = region_directive ? REGION : 0;
2243 if (ifstack.Count == 0){
2244 flags |= PARENT_TAKING;
2245 } else {
2246 int state = ifstack.Peek ();
2247 if ((state & TAKING) != 0) {
2248 flags |= PARENT_TAKING;
2249 }
2250 }
2251
2252 if (caller_is_taking && eval (arg)) {
2253 ifstack.Push (flags | TAKING);
2254 return true;
2255 }
2256 ifstack.Push (flags);
2257 return false;
2258
2259 case "endif":
2260 if (ifstack == null || ifstack.Count == 0){
2261 Error_UnexpectedDirective ("no #if for this #endif");
2262 return true;
2263 } else {
2264 pop = ifstack.Pop ();
2265
2266 if ((pop & REGION) != 0)
2267 Report.Error (1038, Location, "#endregion directive expected");
2268
2269 if (arg.Length != 0) {
2270 Error_EndLineExpected ();
2271 }
2272
2273 if (ifstack.Count == 0)
2274 return true;
2275
2276 int state = ifstack.Peek ();
2277 return (state & TAKING) != 0;
2278 }
2279
2280 case "elif":
2281 if (ifstack == null || ifstack.Count == 0){
2282 Error_UnexpectedDirective ("no #if for this #elif");
2283 return true;
2284 } else {
2285 int state = ifstack.Pop ();
2286
2287 if ((state & REGION) != 0) {
2288 Report.Error (1038, Location, "#endregion directive expected");
2289 return true;
2290 }
2291
2292 if ((state & ELSE_SEEN) != 0){
2293 Error_UnexpectedDirective ("#elif not valid after #else");
2294 return true;
2295 }
2296
2297 if ((state & TAKING) != 0) {
2298 ifstack.Push (0);
2299 return false;
2300 }
2301
2302 if (eval (arg) && ((state & PARENT_TAKING) != 0)){
2303 ifstack.Push (state | TAKING);
2304 return true;
2305 }
2306
2307 ifstack.Push (state);
2308 return false;
2309 }
2310
2311 case "else":
2312 if (ifstack == null || ifstack.Count == 0){
2313 Error_UnexpectedDirective ("no #if for this #else");
2314 return true;
2315 } else {
2316 int state = ifstack.Peek ();
2317
2318 if ((state & REGION) != 0) {
2319 Report.Error (1038, Location, "#endregion directive expected");
2320 return true;
2321 }
2322
2323 if ((state & ELSE_SEEN) != 0){
2324 Error_UnexpectedDirective ("#else within #else");
2325 return true;
2326 }
2327
2328 ifstack.Pop ();
2329
2330 if (arg.Length != 0) {
2331 Error_EndLineExpected ();
2332 return true;
2333 }
2334
2335 bool ret = false;
2336 if ((state & PARENT_TAKING) != 0) {
2337 ret = (state & TAKING) == 0;
2338
2339 if (ret)
2340 state |= TAKING;
2341 else
2342 state &= ~TAKING;
2343 }
2344
2345 ifstack.Push (state | ELSE_SEEN);
2346
2347 return ret;
2348 }
2349 case "define":
2350 if (any_token_seen){
2351 Error_TokensSeen ();
2352 return caller_is_taking;
2353 }
2354 PreProcessDefinition (true, arg, caller_is_taking);
2355 return caller_is_taking;
2356
2357 case "undef":
2358 if (any_token_seen){
2359 Error_TokensSeen ();
2360 return caller_is_taking;
2361 }
2362 PreProcessDefinition (false, arg, caller_is_taking);
2363 return caller_is_taking;
2364 }
2365
2366 //
2367 // These are only processed if we are in a `taking' block
2368 //
2369 if (!caller_is_taking)
2370 return false;
2371
2372 switch (cmd){
2373 case "error":
2374 Report.Error (1029, Location, "#error: '{0}'", arg);
2375 return true;
2376
2377 case "warning":
2378 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg);
2379 return true;
2380
2381 case "pragma":
2382 if (RootContext.Version == LanguageVersion.ISO_1) {
2383 Report.FeatureIsNotAvailable (Location, "#pragma");
2384 return true;
2385 }
2386
2387 PreProcessPragma (arg);
2388 return true;
2389
2390 case "line":
2391 if (!PreProcessLine (arg))
2392 Report.Error (
2393 1576, Location,
2394 "The line number specified for #line directive is missing or invalid");
2395 return caller_is_taking;
2396 }
2397
2398 Report.Error (1024, Location, "Wrong preprocessor directive");
2399 return true;
2400
2401 }
2402
2403 private int consume_string (bool quoted)
2404 {
2405 int c;
2406 string_builder.Length = 0;
2407
2408 while ((c = get_char ()) != -1){
2409 if (c == '"'){
2410 if (quoted && peek_char () == '"'){
2411 string_builder.Append ((char) c);
2412 get_char ();
2413 continue;
2414 } else {
2415 val = new StringLiteral (string_builder.ToString (), Location);
2416 return Token.LITERAL;
2417 }
2418 }
2419
2420 if (c == '\n'){
2421 if (!quoted)
2422 Report.Error (1010, Location, "Newline in constant");
2423 }
2424
2425 if (!quoted){
2426 int surrogate;
2427 c = escape (c, out surrogate);
2428 if (c == -1)
2429 return Token.ERROR;
2430 if (surrogate != 0) {
2431 string_builder.Append ((char) c);
2432 c = surrogate;
2433 }
2434 }
2435 string_builder.Append ((char) c);
2436 }
2437
2438 Report.Error (1039, Location, "Unterminated string literal");
2439 return Token.EOF;
2440 }
2441
2442 private int consume_identifier (int s)
2443 {
2444 int res = consume_identifier (s, false);
2445
2446 if (doc_state == XmlCommentState.Allowed)
2447 doc_state = XmlCommentState.NotAllowed;
2448
2449 return res;
2450 }
2451
2452 int consume_identifier (int c, bool quoted)
2453 {
2454 //
2455 // This method is very performance sensitive. It accounts
2456 // for approximately 25% of all parser time
2457 //
2458
2459 int pos = 0;
2460 int column = col;
2461
2462 if (c == '\\') {
2463 int surrogate;
2464 c = escape (c, out surrogate);
2465 if (surrogate != 0) {
2466 id_builder [pos++] = (char) c;
2467 c = surrogate;
2468 }
2469 }
2470
2471 id_builder [pos++] = (char) c;
2472
2473 try {
2474 while (true) {
2475 c = reader.Read ();
2476
2477 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) {
2478 id_builder [pos++] = (char) c;
2479 continue;
2480 }
2481
2482 if (c < 0x80) {
2483 if (c == '\\') {
2484 int surrogate;
2485 c = escape (c, out surrogate);
2486 if (surrogate != 0) {
2487 if (is_identifier_part_character ((char) c))
2488 id_builder[pos++] = (char) c;
2489 c = surrogate;
2490 }
2491
2492 continue;
2493 }
2494 } else if (Char.IsLetter ((char) c) || Char.GetUnicodeCategory ((char) c) == UnicodeCategory.ConnectorPunctuation) {
2495 id_builder [pos++] = (char) c;
2496 continue;
2497 }
2498
2499 putback_char = c;
2500 break;
2501 }
2502 } catch (IndexOutOfRangeException) {
2503 Report.Error (645, Location, "Identifier too long (limit is 512 chars)");
2504 col += pos - 1;
2505 return Token.ERROR;
2506 }
2507
2508 col += pos - 1;
2509
2510 //
2511 // Optimization: avoids doing the keyword lookup
2512 // on uppercase letters
2513 //
2514 if (id_builder [0] >= '_' && !quoted) {
2515 int keyword = GetKeyword (id_builder, pos);
2516 if (keyword != -1) {
2517 val = LocatedToken.Create (null, ref_line, column);
2518 return keyword;
2519 }
2520 }
2521
2522 //
2523 // Keep identifiers in an array of hashtables to avoid needless
2524 // allocations
2525 //
2526 var identifiers_group = identifiers [pos];
2527 string s;
2528 if (identifiers_group != null) {
2529 if (identifiers_group.TryGetValue (id_builder, out s)) {
2530 val = LocatedToken.Create (s, ref_line, column);
2531 if (quoted)
2532 AddEscapedIdentifier (((LocatedToken) val).Location);
2533 return Token.IDENTIFIER;
2534 }
2535 } else {
2536 // TODO: this should be number of files dependant
2537 // corlib compilation peaks at 1000 and System.Core at 150
2538 int capacity = pos > 20 ? 10 : 100;
2539 identifiers_group = new Dictionary<char[],string> (capacity, new IdentifiersComparer (pos));
2540 identifiers [pos] = identifiers_group;
2541 }
2542
2543 char [] chars = new char [pos];
2544 Array.Copy (id_builder, chars, pos);
2545
2546 s = new string (id_builder, 0, pos);
2547 identifiers_group.Add (chars, s);
2548
2549 val = LocatedToken.Create (s, ref_line, column);
2550 if (quoted)
2551 AddEscapedIdentifier (((LocatedToken) val).Location);
2552
2553 return Token.IDENTIFIER;
2554 }
2555
2556 public int xtoken ()
2557 {
2558 int d, c;
2559
2560 // Whether we have seen comments on the current line
2561 bool comments_seen = false;
2562 while ((c = get_char ()) != -1) {
2563 switch (c) {
2564 case '\t':
2565 col = ((col + 8) / 8) * 8;
2566 continue;
2567
2568 case ' ':
2569 case '\f':
2570 case '\v':
2571 case 0xa0:
2572 case 0:
2573 case 0xFEFF: // Ignore BOM anywhere in the file
2574 continue;
2575
2576 /* This is required for compatibility with .NET
2577 case 0xEF:
2578 if (peek_char () == 0xBB) {
2579 PushPosition ();
2580 get_char ();
2581 if (get_char () == 0xBF)
2582 continue;
2583 PopPosition ();
2584 }
2585 break;
2586 */
2587 case '\r':
2588 if (peek_char () != '\n')
2589 advance_line ();
2590 else
2591 get_char ();
2592
2593 any_token_seen |= tokens_seen;
2594 tokens_seen = false;
2595 comments_seen = false;
2596 continue;
2597
2598 case '\\':
2599 tokens_seen = true;
2600 return consume_identifier (c);
2601
2602 case '{':
2603 val = LocatedToken.Create (ref_line, col);
2604 return Token.OPEN_BRACE;
2605 case '}':
2606 val = LocatedToken.Create (ref_line, col);
2607 return Token.CLOSE_BRACE;
2608 case '[':
2609 // To block doccomment inside attribute declaration.
2610 if (doc_state == XmlCommentState.Allowed)
2611 doc_state = XmlCommentState.NotAllowed;
2612 return Token.OPEN_BRACKET;
2613 case ']':
2614 return Token.CLOSE_BRACKET;
2615 case '(':
2616 val = LocatedToken.Create (ref_line, col);
2617 //
2618 // An expression versions of parens can appear in block context only
2619 //
2620 if (parsing_block != 0 && !lambda_arguments_parsing) {
2621
2622 //
2623 // Optmize most common case where we know that parens
2624 // is not special
2625 //
2626 switch (current_token) {
2627 case Token.IDENTIFIER:
2628 case Token.IF:
2629 case Token.FOR:
2630 case Token.FOREACH:
2631 case Token.TYPEOF:
2632 case Token.WHILE:
2633 case Token.USING:
2634 case Token.DEFAULT:
2635 case Token.DELEGATE:
2636 case Token.OP_GENERICS_GT:
2637 return Token.OPEN_PARENS;
2638 }
2639
2640 // Optimize using peek
2641 int xx = peek_char ();
2642 switch (xx) {
2643 case '(':
2644 case '\'':
2645 case '"':
2646 case '0':
2647 case '1':
2648 return Token.OPEN_PARENS;
2649 }
2650
2651 lambda_arguments_parsing = true;
2652 PushPosition ();
2653 d = TokenizeOpenParens ();
2654 PopPosition ();
2655 lambda_arguments_parsing = false;
2656 return d;
2657 }
2658
2659 return Token.OPEN_PARENS;
2660 case ')':
2661 return Token.CLOSE_PARENS;
2662 case ',':
2663 return Token.COMMA;
2664 case ';':
2665 return Token.SEMICOLON;
2666 case '~':
2667 return Token.TILDE;
2668 case '?':
2669 return TokenizePossibleNullableType ();
2670 case '<':
2671 if (parsing_generic_less_than++ > 0)
2672 return Token.OP_GENERICS_LT;
2673
2674 return TokenizeLessThan ();
2675
2676 case '>':
2677 d = peek_char ();
2678
2679 if (d == '='){
2680 get_char ();
2681 return Token.OP_GE;
2682 }
2683
2684 if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) {
2685 parsing_generic_less_than--;
2686 return Token.OP_GENERICS_GT;
2687 }
2688
2689 if (d == '>') {
2690 get_char ();
2691 d = peek_char ();
2692
2693 if (d == '=') {
2694 get_char ();
2695 return Token.OP_SHIFT_RIGHT_ASSIGN;
2696 }
2697 return Token.OP_SHIFT_RIGHT;
2698 }
2699
2700 return Token.OP_GT;
2701
2702 case '+':
2703 d = peek_char ();
2704 if (d == '+') {
2705 d = Token.OP_INC;
2706 } else if (d == '=') {
2707 d = Token.OP_ADD_ASSIGN;
2708 } else {
2709 return Token.PLUS;
2710 }
2711 get_char ();
2712 return d;
2713
2714 case '-':
2715 d = peek_char ();
2716 if (d == '-') {
2717 d = Token.OP_DEC;
2718 } else if (d == '=')
2719 d = Token.OP_SUB_ASSIGN;
2720 else if (d == '>')
2721 d = Token.OP_PTR;
2722 else {
2723 return Token.MINUS;
2724 }
2725 get_char ();
2726 return d;
2727
2728 case '!':
2729 if (peek_char () == '='){
2730 get_char ();
2731 return Token.OP_NE;
2732 }
2733 return Token.BANG;
2734
2735 case '=':
2736 d = peek_char ();
2737 if (d == '='){
2738 get_char ();
2739 return Token.OP_EQ;
2740 }
2741 if (d == '>'){
2742 get_char ();
2743 return Token.ARROW;
2744 }
2745
2746 return Token.ASSIGN;
2747
2748 case '&':
2749 d = peek_char ();
2750 if (d == '&'){
2751 get_char ();
2752 return Token.OP_AND;
2753 }
2754 if (d == '='){
2755 get_char ();
2756 return Token.OP_AND_ASSIGN;
2757 }
2758 return Token.BITWISE_AND;
2759
2760 case '|':
2761 d = peek_char ();
2762 if (d == '|'){
2763 get_char ();
2764 return Token.OP_OR;
2765 }
2766 if (d == '='){
2767 get_char ();
2768 return Token.OP_OR_ASSIGN;
2769 }
2770 return Token.BITWISE_OR;
2771
2772 case '*':
2773 if (peek_char () == '='){
2774 get_char ();
2775 return Token.OP_MULT_ASSIGN;
2776 }
2777 val = LocatedToken.Create (ref_line, col);
2778 return Token.STAR;
2779
2780 case '/':
2781 d = peek_char ();
2782 if (d == '='){
2783 get_char ();
2784 return Token.OP_DIV_ASSIGN;
2785 }
2786
2787 // Handle double-slash comments.
2788 if (d == '/'){
2789 get_char ();
2790 if (RootContext.Documentation != null && peek_char () == '/') {
2791 get_char ();
2792 // Don't allow ////.
2793 if ((d = peek_char ()) != '/') {
2794 update_comment_location ();
2795 if (doc_state == XmlCommentState.Allowed)
2796 handle_one_line_xml_comment ();
2797 else if (doc_state == XmlCommentState.NotAllowed)
2798 warn_incorrect_doc_comment ();
2799 }
2800 }
2801 while ((d = get_char ()) != -1 && (d != '\n') && d != '\r');
2802
2803 any_token_seen |= tokens_seen;
2804 tokens_seen = false;
2805 comments_seen = false;
2806 continue;
2807 } else if (d == '*'){
2808 get_char ();
2809 bool docAppend = false;
2810 if (RootContext.Documentation != null && peek_char () == '*') {
2811 get_char ();
2812 update_comment_location ();
2813 // But when it is /**/, just do nothing.
2814 if (peek_char () == '/') {
2815 get_char ();
2816 continue;
2817 }
2818 if (doc_state == XmlCommentState.Allowed)
2819 docAppend = true;
2820 else if (doc_state == XmlCommentState.NotAllowed)
2821 warn_incorrect_doc_comment ();
2822 }
2823
2824 int current_comment_start = 0;
2825 if (docAppend) {
2826 current_comment_start = xml_comment_buffer.Length;
2827 xml_comment_buffer.Append (Environment.NewLine);
2828 }
2829
2830 while ((d = get_char ()) != -1){
2831 if (d == '*' && peek_char () == '/'){
2832 get_char ();
2833 comments_seen = true;
2834 break;
2835 }
2836 if (docAppend)
2837 xml_comment_buffer.Append ((char) d);
2838
2839 if (d == '\n'){
2840 any_token_seen |= tokens_seen;
2841 tokens_seen = false;
2842 //
2843 // Reset 'comments_seen' just to be consistent.
2844 // It doesn't matter either way, here.
2845 //
2846 comments_seen = false;
2847 }
2848 }
2849 if (!comments_seen)
2850 Report.Error (1035, Location, "End-of-file found, '*/' expected");
2851
2852 if (docAppend)
2853 update_formatted_doc_comment (current_comment_start);
2854 continue;
2855 }
2856 return Token.DIV;
2857
2858 case '%':
2859 if (peek_char () == '='){
2860 get_char ();
2861 return Token.OP_MOD_ASSIGN;
2862 }
2863 return Token.PERCENT;
2864
2865 case '^':
2866 if (peek_char () == '='){
2867 get_char ();
2868 return Token.OP_XOR_ASSIGN;
2869 }
2870 return Token.CARRET;
2871
2872 case ':':
2873 if (peek_char () == ':') {
2874 get_char ();
2875 return Token.DOUBLE_COLON;
2876 }
2877 return Token.COLON;
2878
2879 case '0': case '1': case '2': case '3': case '4':
2880 case '5': case '6': case '7': case '8': case '9':
2881 tokens_seen = true;
2882 return is_number (c);
2883
2884 case '\n': // white space
2885 any_token_seen |= tokens_seen;
2886 tokens_seen = false;
2887 comments_seen = false;
2888 continue;
2889
2890 case '.':
2891 tokens_seen = true;
2892 d = peek_char ();
2893 if (d >= '0' && d <= '9')
2894 return is_number (c);
2895 return Token.DOT;
2896
2897 case '#':
2898 if (tokens_seen || comments_seen) {
2899 Eror_WrongPreprocessorLocation ();
2900 return Token.ERROR;
2901 }
2902
2903 if (handle_preprocessing_directive (true))
2904 continue;
2905
2906 bool directive_expected = false;
2907 while ((c = get_char ()) != -1) {
2908 if (col == 1) {
2909 directive_expected = true;
2910 } else if (!directive_expected) {
2911 // TODO: Implement comment support for disabled code and uncomment this code
2912 // if (c == '#') {
2913 // Eror_WrongPreprocessorLocation ();
2914 // return Token.ERROR;
2915 // }
2916 continue;
2917 }
2918
2919 if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v' )
2920 continue;
2921
2922 if (c == '#') {
2923 if (handle_preprocessing_directive (false))
2924 break;
2925 }
2926 directive_expected = false;
2927 }
2928
2929 if (c != -1) {
2930 tokens_seen = false;
2931 continue;
2932 }
2933
2934 return Token.EOF;
2935
2936 case '"':
2937 return consume_string (false);
2938
2939 case '\'':
2940 return TokenizeBackslash ();
2941
2942 case '@':
2943 c = get_char ();
2944 if (c == '"') {
2945 tokens_seen = true;
2946 return consume_string (true);
2947 }
2948
2949 if (is_identifier_start_character (c)){
2950 return consume_identifier (c, true);
2951 }
2952
2953 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @");
2954 return Token.ERROR;
2955
2956 case EvalStatementParserCharacter:
2957 return Token.EVAL_STATEMENT_PARSER;
2958 case EvalCompilationUnitParserCharacter:
2959 return Token.EVAL_COMPILATION_UNIT_PARSER;
2960 case EvalUsingDeclarationsParserCharacter:
2961 return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER;
2962 }
2963
2964 if (is_identifier_start_character (c)) {
2965 tokens_seen = true;
2966 return consume_identifier (c);
2967 }
2968
2969 error_details = ((char)c).ToString ();
2970 return Token.ERROR;
2971 }
2972
2973 if (CompleteOnEOF){
2974 if (generated)
2975 return Token.COMPLETE_COMPLETION;
2976
2977 generated = true;
2978 return Token.GENERATE_COMPLETION;
2979 }
2980
2981
2982 return Token.EOF;
2983 }
2984
2985 int TokenizeBackslash ()
2986 {
2987 int c = get_char ();
2988 tokens_seen = true;
2989 if (c == '\'') {
2990 error_details = "Empty character literal";
2991 Report.Error (1011, Location, error_details);
2992 return Token.ERROR;
2993 }
2994 if (c == '\r' || c == '\n') {
2995 Report.Error (1010, Location, "Newline in constant");
2996 return Token.ERROR;
2997 }
2998
2999 int d;
3000 c = escape (c, out d);
3001 if (c == -1)
3002 return Token.ERROR;
3003 if (d != 0)
3004 throw new NotImplementedException ();
3005
3006 val = new CharLiteral ((char) c, Location);
3007 c = get_char ();
3008
3009 if (c != '\'') {
3010 Report.Error (1012, Location, "Too many characters in character literal");
3011
3012 // Try to recover, read until newline or next "'"
3013 while ((c = get_char ()) != -1) {
3014 if (c == '\n' || c == '\'')
3015 break;
3016 }
3017 return Token.ERROR;
3018 }
3019
3020 return Token.LITERAL;
3021 }
3022
3023 int TokenizeLessThan ()
3024 {
3025 int d;
3026 if (handle_typeof) {
3027 PushPosition ();
3028 if (parse_generic_dimension (out d)) {
3029 val = d;
3030 DiscardPosition ();
3031 return Token.GENERIC_DIMENSION;
3032 }
3033 PopPosition ();
3034 }
3035
3036 // Save current position and parse next token.
3037 PushPosition ();
3038 if (parse_less_than ()) {
3039 if (parsing_generic_declaration && token () != Token.DOT) {
3040 d = Token.OP_GENERICS_LT_DECL;
3041 } else {
3042 d = Token.OP_GENERICS_LT;
3043 }
3044 PopPosition ();
3045 return d;
3046 }
3047
3048 PopPosition ();
3049 parsing_generic_less_than = 0;
3050
3051 d = peek_char ();
3052 if (d == '<') {
3053 get_char ();
3054 d = peek_char ();
3055
3056 if (d == '=') {
3057 get_char ();
3058 return Token.OP_SHIFT_LEFT_ASSIGN;
3059 }
3060 return Token.OP_SHIFT_LEFT;
3061 }
3062
3063 if (d == '=') {
3064 get_char ();
3065 return Token.OP_LE;
3066 }
3067 return Token.OP_LT;
3068 }
3069
3070 //
3071 // Handles one line xml comment
3072 //
3073 private void handle_one_line_xml_comment ()
3074 {
3075 int c;
3076 while ((c = peek_char ()) == ' ')
3077 get_char (); // skip heading whitespaces.
3078 while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') {
3079 xml_comment_buffer.Append ((char) get_char ());
3080 }
3081 if (c == '\r' || c == '\n')
3082 xml_comment_buffer.Append (Environment.NewLine);
3083 }
3084
3085 //
3086 // Remove heading "*" in Javadoc-like xml documentation.
3087 //
3088 private void update_formatted_doc_comment (int current_comment_start)
3089 {
3090 int length = xml_comment_buffer.Length - current_comment_start;
3091 string [] lines = xml_comment_buffer.ToString (
3092 current_comment_start,
3093 length).Replace ("\r", "").Split ('\n');
3094
3095 // The first line starts with /**, thus it is not target
3096 // for the format check.
3097 for (int i = 1; i < lines.Length; i++) {
3098 string s = lines [i];
3099 int idx = s.IndexOf ('*');
3100 string head = null;
3101 if (idx < 0) {
3102 if (i < lines.Length - 1)
3103 return;
3104 head = s;
3105 } else
3106 head = s.Substring (0, idx);
3107 foreach (char c in head)
3108 if (c != ' ')
3109 return;
3110 lines [i] = s.Substring (idx + 1);
3111 }
3112 xml_comment_buffer.Remove (current_comment_start, length);
3113 xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines));
3114 }
3115
3116 //
3117 // Updates current comment location.
3118 //
3119 private void update_comment_location ()
3120 {
3121 if (current_comment_location.IsNull) {
3122 // "-2" is for heading "//" or "/*"
3123 current_comment_location =
3124 new Location (ref_line, hidden ? -1 : col - 2);
3125 }
3126 }
3127
3128 //
3129 // Checks if there was incorrect doc comments and raise
3130 // warnings.
3131 //
3132 public void check_incorrect_doc_comment ()
3133 {
3134 if (xml_comment_buffer.Length > 0)
3135 warn_incorrect_doc_comment ();
3136 }
3137
3138 //
3139 // Raises a warning when tokenizer found incorrect doccomment
3140 // markup.
3141 //
3142 private void warn_incorrect_doc_comment ()
3143 {
3144 if (doc_state != XmlCommentState.Error) {
3145 doc_state = XmlCommentState.Error;
3146 // in csc, it is 'XML comment is not placed on
3147 // a valid language element'. But that does not
3148 // make sense.
3149 Report.Warning (1587, 2, Location, "XML comment is not placed on a valid language element");
3150 }
3151 }
3152
3153 //
3154 // Consumes the saved xml comment lines (if any)
3155 // as for current target member or type.
3156 //
3157 public string consume_doc_comment ()
3158 {
3159 if (xml_comment_buffer.Length > 0) {
3160 string ret = xml_comment_buffer.ToString ();
3161 reset_doc_comment ();
3162 return ret;
3163 }
3164 return null;
3165 }
3166
3167 Report Report {
3168 get { return context.Report; }
3169 }
3170
3171 void reset_doc_comment ()
3172 {
3173 xml_comment_buffer.Length = 0;
3174 current_comment_location = Location.Null;
3175 }
3176
3177 public void cleanup ()
3178 {
3179 if (ifstack != null && ifstack.Count >= 1) {
3180 int state = ifstack.Pop ();
3181 if ((state & REGION) != 0)
3182 Report.Error (1038, Location, "#endregion directive expected");
3183 else
3184 Report.Error (1027, Location, "Expected `#endif' directive");
3185 }
3186 }
3187 }
3188
3189 //
3190 // Indicates whether it accepts XML documentation or not.
3191 //
3192 public enum XmlCommentState {
3193 // comment is allowed in this state.
3194 Allowed,
3195 // comment is not allowed in this state.
3196 NotAllowed,
3197 // once comments appeared when it is NotAllowed, then the
3198 // state is changed to it, until the state is changed to
3199 // .Allowed.
3200 Error
3201 }
3202 }
3203
git-svn mirror of mcs