1 /* valageniescanner.vala
3 * Copyright (C) 2008-2012 Jamie McCracken, Jürg Billeter
4 * Based on code by Jürg Billeter
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 * Jamie McCracken jamiemcc gnome org
27 * Lexical scanner for Genie source files.
29 public class Vala
.Genie
.Scanner
{
30 public SourceFile source_file
{ get; private set; }
32 public int indent_spaces
{ get; set;}
41 int current_indent_level
;
45 /* track open parens and braces for automatic line continuations */
46 int open_parens_count
;
54 Conditional
[] conditional_stack
;
58 public bool else_found
;
59 public bool skip_section
;
73 public Scanner (SourceFile source_file
) {
74 this
.source_file
= source_file
;
76 begin
= source_file
.get_mapped_contents ();
77 end
= begin
+ source_file
.get_mapped_length ();
84 current_indent_level
= 0;
88 open_parens_count
= 0;
91 parse_started
= false;
92 last_token
= TokenType
.NONE
;
97 return (state_stack
.length
> 0 && state_stack
[state_stack
.length
- 1] == State
.TEMPLATE
);
100 bool in_template_part () {
101 return (state_stack
.length
> 0 && state_stack
[state_stack
.length
- 1] == State
.TEMPLATE_PART
);
104 bool is_ident_char (char c
) {
105 return (c
.isalnum () || c
== '_');
108 bool in_regex_literal () {
109 return (state_stack
.length
> 0 && state_stack
[state_stack
.length
- 1] == State
.REGEX_LITERAL
);
112 SourceReference
get_source_reference (int offset
, int length
= 0) {
113 return new
SourceReference (source_file
, SourceLocation (current
, line
, column
+ offset
), SourceLocation (current
+ length
, line
, column
+ offset
+ length
));
116 public TokenType
read_regex_token (out SourceLocation token_begin
, out SourceLocation token_end
) {
118 char* begin
= current
;
119 token_begin
= SourceLocation (begin
, line
, column
);
121 int token_length_in_chars
= -1;
123 if (current
>= end
) {
124 type
= TokenType
.EOF
;
126 switch (current
[0]) {
128 type
= TokenType
.CLOSE_REGEX_LITERAL
;
130 state_stack
.length
--;
135 while (current
[0] == 'i' || current
[0] == 's' || current
[0] == 'm' || current
[0] == 'x') {
136 switch (current
[0]) {
139 Report
.error (get_source_reference (token_length_in_chars
), "modifier 'i' used more than once");
145 Report
.error (get_source_reference (token_length_in_chars
), "modifier 's' used more than once");
151 Report
.error (get_source_reference (token_length_in_chars
), "modifier 'm' used more than once");
157 Report
.error (get_source_reference (token_length_in_chars
), "modifier 'x' used more than once");
163 token_length_in_chars
++;
167 type
= TokenType
.REGEX_LITERAL
;
168 token_length_in_chars
= 0;
169 while (current
< end
&& current
[0] != '/') {
170 if (current
[0] == '\\') {
172 token_length_in_chars
++;
173 if (current
>= end
) {
177 switch (current
[0]) {
228 token_length_in_chars
++;
231 // hexadecimal escape character
233 token_length_in_chars
++;
234 while (current
< end
&& current
[0].isxdigit ()) {
236 token_length_in_chars
++;
240 Report
.error (get_source_reference (token_length_in_chars
), "invalid escape sequence");
243 } else if (current
[0] == '\n') {
246 unichar u
= ((string) current
).get_char_validated ((long) (end
- current
));
247 if (u
!= (unichar
) (-1)) {
248 current
+= u
.to_utf8 (null);
249 token_length_in_chars
++;
252 Report
.error (get_source_reference (token_length_in_chars
), "invalid UTF-8 character");
256 if (current
>= end
|| current
[0] == '\n') {
257 Report
.error (get_source_reference (token_length_in_chars
), "syntax error, expected \"");
258 state_stack
.length
--;
259 return read_token (out token_begin
, out token_end
);
265 if (token_length_in_chars
< 0) {
266 column
+= (int) (current
- begin
);
268 column
+= token_length_in_chars
;
271 token_end
= SourceLocation (current
, line
, column
- 1);
277 public void seek (SourceLocation location
) {
278 current
= location
.pos
;
279 line
= location
.line
;
280 column
= location
.column
;
282 conditional_stack
= null;
286 public static TokenType
get_identifier_or_keyword (char* begin
, int len
) {
291 if (matches (begin
, "as")) return TokenType
.AS
;
294 if (matches (begin
, "do")) return TokenType
.DO
;
307 if (matches (begin
, "of")) return TokenType
.OF
;
309 if (matches (begin
, "or")) return TokenType
.OP_OR
;
312 if (matches (begin
, "to")) return TokenType
.TO
;
319 if (matches (begin
, "and")) return TokenType
.OP_AND
;
322 if (matches (begin
, "def")) return TokenType
.DEF
;
325 if (matches (begin
, "for")) return TokenType
.FOR
;
328 if (matches (begin
, "get")) return TokenType
.GET
;
331 if (matches (begin
, "isa")) return TokenType
.ISA
;
336 if (matches (begin
, "new")) return TokenType
.NEW
;
339 if (matches (begin
, "not")) return TokenType
.OP_NEG
;
344 if (matches (begin
, "out")) return TokenType
.OUT
;
347 if (matches (begin
, "ref")) return TokenType
.REF
;
350 if (matches (begin
, "set")) return TokenType
.SET
;
353 if (matches (begin
, "try")) return TokenType
.TRY
;
356 if (matches (begin
, "var")) return TokenType
.VAR
;
363 if (matches (begin
, "case")) return TokenType
.CASE
;
366 if (matches (begin
, "dict")) return TokenType
.DICT
;
371 if (matches (begin
, "else")) return TokenType
.ELSE
;
374 if (matches (begin
, "enum")) return TokenType
.ENUM
;
379 if (matches (begin
, "init")) return TokenType
.INIT
;
384 if (matches (begin
, "list")) return TokenType
.LIST
;
387 if (matches (begin
, "lock")) return TokenType
.LOCK
;
393 if (matches (begin
, "null")) return TokenType
.NULL
;
398 if (matches (begin
, "pass")) return TokenType
.PASS
;
401 if (matches (begin
, "prop")) return TokenType
.PROP
;
406 if (matches (begin
, "self")) return TokenType
.THIS
;
409 if (matches (begin
, "true")) return TokenType
.TRUE
;
412 if (matches (begin
, "uses")) return TokenType
.USES
;
415 if (matches (begin
, "void")) return TokenType
.VOID
;
420 if (matches (begin
, "weak")) return TokenType
.WEAK
;
423 if (matches (begin
, "when")) return TokenType
.WHEN
;
434 if (matches (begin
, "array")) return TokenType
.ARRAY
;
437 if (matches (begin
, "async")) return TokenType
.ASYNC
;
442 if (matches (begin
, "break")) return TokenType
.BREAK
;
447 if (matches (begin
, "class")) return TokenType
.CLASS
;
450 if (matches (begin
, "const")) return TokenType
.CONST
;
455 if (matches (begin
, "event")) return TokenType
.EVENT
;
460 if (matches (begin
, "false")) return TokenType
.FALSE
;
463 if (matches (begin
, "final")) return TokenType
.FINAL
;
468 if (matches (begin
, "owned")) return TokenType
.OWNED
;
471 if (matches (begin
, "print")) return TokenType
.PRINT
;
474 if (matches (begin
, "super")) return TokenType
.SUPER
;
477 if (matches (begin
, "raise")) return TokenType
.RAISE
;
480 if (matches (begin
, "while")) return TokenType
.WHILE
;
483 if (matches (begin
, "yield")) return TokenType
.YIELD
;
490 if (matches (begin
, "assert")) return TokenType
.ASSERT
;
495 if (matches (begin
, "delete")) return TokenType
.DELETE
;
498 if (matches (begin
, "downto")) return TokenType
.DOWNTO
;
507 if (matches (begin
, "except")) return TokenType
.EXCEPT
;
510 if (matches (begin
, "extern")) return TokenType
.EXTERN
;
517 if (matches (begin
, "inline")) return TokenType
.INLINE
;
522 if (matches (begin
, "params")) return TokenType
.PARAMS
;
525 if (matches (begin
, "public")) return TokenType
.PUBLIC
;
532 if (matches (begin
, "raises")) return TokenType
.RAISES
;
535 if (matches (begin
, "return")) return TokenType
.RETURN
;
542 if (matches (begin
, "sealed")) return TokenType
.SEALED
;
545 if (matches (begin
, "sizeof")) return TokenType
.SIZEOF
;
550 if (matches (begin
, "static")) return TokenType
.STATIC
;
553 if (matches (begin
, "struct")) return TokenType
.STRUCT
;
560 if (matches (begin
, "typeof")) return TokenType
.TYPEOF
;
569 if (matches (begin
, "default")) return TokenType
.DEFAULT
;
572 if (matches (begin
, "dynamic")) return TokenType
.DYNAMIC
;
577 if (matches (begin
, "ensures")) return TokenType
.ENSURES
;
582 if (matches (begin
, "finally")) return TokenType
.FINALLY
;
587 if (matches (begin
, "private")) return TokenType
.PRIVATE
;
590 if (matches (begin
, "unowned")) return TokenType
.UNOWNED
;
593 if (matches (begin
, "virtual")) return TokenType
.VIRTUAL
;
600 if (matches (begin
, "abstract")) return TokenType
.ABSTRACT
;
603 if (matches (begin
, "continue")) return TokenType
.CONTINUE
;
606 if (matches (begin
, "delegate")) return TokenType
.DELEGATE
;
609 if (matches (begin
, "internal")) return TokenType
.INTERNAL
;
612 if (matches (begin
, "override")) return TokenType
.OVERRIDE
;
617 if (matches (begin
, "readonly")) return TokenType
.READONLY
;
620 if (matches (begin
, "requires")) return TokenType
.REQUIRES
;
625 if (matches (begin
, "volatile")) return TokenType
.VOLATILE
;
632 if (matches (begin
, "construct")) return TokenType
.CONSTRUCT
;
635 if (matches (begin
, "exception")) return TokenType
.ERRORDOMAIN
;
638 if (matches (begin
, "interface")) return TokenType
.INTERFACE
;
641 if (matches (begin
, "namespace")) return TokenType
.NAMESPACE
;
644 if (matches (begin
, "protected")) return TokenType
.PROTECTED
;
647 if (matches (begin
, "writeonly")) return TokenType
.WRITEONLY
;
654 if (matches (begin
, "implements")) return TokenType
.IMPLEMENTS
;
659 return TokenType
.IDENTIFIER
;
663 public TokenType
read_template_token (out SourceLocation token_begin
, out SourceLocation token_end
) {
665 char* begin
= current
;
666 token_begin
= SourceLocation (begin
, line
, column
);
668 int token_length_in_chars
= -1;
670 if (current
>= end
) {
671 type
= TokenType
.EOF
;
673 switch (current
[0]) {
675 type
= TokenType
.CLOSE_TEMPLATE
;
677 state_stack
.length
--;
680 token_begin
.pos
++; // $ is not part of following token
682 if (current
[0].isalpha () || current
[0] == '_') {
684 while (current
< end
&& is_ident_char (current
[0])) {
688 type
= TokenType
.IDENTIFIER
;
689 state_stack
+= State
.TEMPLATE_PART
;
690 } else if (current
[0] == '(') {
693 state_stack
+= State
.PARENS
;
694 return read_token (out token_begin
, out token_end
);
695 } else if (current
[0] == '$') {
696 type
= TokenType
.TEMPLATE_STRING_LITERAL
;
698 state_stack
+= State
.TEMPLATE_PART
;
700 Report
.error (get_source_reference (1), "unexpected character");
701 return read_template_token (out token_begin
, out token_end
);
705 type
= TokenType
.TEMPLATE_STRING_LITERAL
;
706 token_length_in_chars
= 0;
707 while (current
< end
&& current
[0] != '"' && current
[0] != '$') {
708 if (current
[0] == '\\') {
710 token_length_in_chars
++;
711 if (current
>= end
) {
715 switch (current
[0]) {
726 token_length_in_chars
++;
729 // hexadecimal escape character
731 token_length_in_chars
++;
732 while (current
< end
&& current
[0].isxdigit ()) {
734 token_length_in_chars
++;
738 Report
.error (get_source_reference (token_length_in_chars
), "invalid escape sequence");
741 } else if (current
[0] == '\n') {
745 token_length_in_chars
= 1;
747 unichar u
= ((string) current
).get_char_validated ((long) (end
- current
));
748 if (u
!= (unichar
) (-1)) {
749 current
+= u
.to_utf8 (null);
750 token_length_in_chars
++;
753 Report
.error (get_source_reference (token_length_in_chars
), "invalid UTF-8 character");
757 if (current
>= end
) {
758 Report
.error (get_source_reference (token_length_in_chars
), "syntax error, expected \"");
759 state_stack
.length
--;
760 return read_token (out token_begin
, out token_end
);
762 state_stack
+= State
.TEMPLATE_PART
;
767 if (token_length_in_chars
< 0) {
768 column
+= (int) (current
- begin
);
770 column
+= token_length_in_chars
;
773 token_end
= SourceLocation (current
, line
, column
- 1);
779 public TokenType
read_token (out SourceLocation token_begin
, out SourceLocation token_end
) {
780 if (current
== null) {
781 token_begin
= SourceLocation (current
, line
, column
);
782 token_end
= SourceLocation (current
, line
, column
);
783 return TokenType
.EOF
;
786 if (in_template ()) {
787 return read_template_token (out token_begin
, out token_end
);
788 } else if (in_template_part ()) {
789 state_stack
.length
--;
791 token_begin
= SourceLocation (current
, line
, column
);
792 token_end
= SourceLocation (current
, line
, column
- 1);
794 return TokenType
.COMMA
;
795 } else if (in_regex_literal ()) {
796 return read_regex_token (out token_begin
, out token_end
);
801 /* emit dedents if outstanding before checking any other chars */
803 if (pending_dedents
> 0) {
807 token_begin
= SourceLocation (current
, line
, column
);
808 token_end
= SourceLocation (current
, line
, column
);
810 last_token
= TokenType
.DEDENT
;
812 return TokenType
.DEDENT
;
815 if ((_indent_spaces
== 0 ) || (last_token
!= TokenType
.EOL
)) {
816 /* scrub whitespace (excluding newlines) and comments */
821 /* handle explicit line continuation (lines ending with "\") */
822 while (current
< end
&& current
[0] == '\\' && current
[1] == '\n') {
828 /* handle automatic line continuations (when inside parens or braces) */
829 while (current
< end
&& current
[0] == '\n' && (open_parens_count
> 0 || open_brace_count
> 0)) {
836 /* handle non-consecutive new line once parsing is underway - EOL */
837 if (newline () && parse_started
&& last_token
!= TokenType
.EOL
&& last_token
!= TokenType
.SEMICOLON
) {
838 token_begin
= SourceLocation (current
, line
, column
);
839 token_end
= SourceLocation (current
, line
, column
);
841 last_token
= TokenType
.EOL
;
843 return TokenType
.EOL
;
847 while (skip_newlines ()) {
848 token_begin
= SourceLocation (current
, line
, column
);
850 current_indent_level
= count_tabs ();
852 /* if its an empty new line then ignore */
853 if (current_indent_level
== -1) {
857 if (current_indent_level
> indent_level
) {
858 indent_level
= current_indent_level
;
860 token_end
= SourceLocation (current
, line
, column
);
862 last_token
= TokenType
.INDENT
;
864 return TokenType
.INDENT
;
865 } else if (current_indent_level
< indent_level
) {
868 pending_dedents
= (indent_level
- current_indent_level
);
869 token_end
= SourceLocation (current
, line
, column
);
871 last_token
= TokenType
.DEDENT
;
873 return TokenType
.DEDENT
;
878 char* begin
= current
;
879 token_begin
= SourceLocation (begin
, line
, column
);
881 int token_length_in_chars
= -1;
883 parse_started
= true;
885 if (current
>= end
) {
886 if (indent_level
> 0) {
889 pending_dedents
= indent_level
;
891 type
= TokenType
.DEDENT
;
893 type
= TokenType
.EOF
;
895 } else if (current
[0].isalpha () || current
[0] == '_') {
897 while (current
< end
&& is_ident_char (current
[0])) {
901 type
= get_identifier_or_keyword (begin
, len
);
902 } else if (current
[0] == '@') {
903 if (current
< end
- 1 && current
[1] == '"') {
904 type
= TokenType
.OPEN_TEMPLATE
;
906 state_stack
+= State
.TEMPLATE
;
908 token_begin
.pos
++; // @ is not part of the identifier
911 while (current
< end
&& is_ident_char (current
[0])) {
915 type
= TokenType
.IDENTIFIER
;
917 } else if (current
[0].isdigit ()) {
918 while (current
< end
&& current
[0].isdigit ()) {
921 type
= TokenType
.INTEGER_LITERAL
;
922 if (current
< end
&& current
[0].tolower () == 'l') {
924 if (current
< end
&& current
[0].tolower () == 'l') {
927 } else if (current
< end
&& current
[0].tolower () == 'u') {
929 if (current
< end
&& current
[0].tolower () == 'l') {
931 if (current
< end
&& current
[0].tolower () == 'l') {
935 } else if (current
< end
- 1 && current
[0] == '.' && current
[1].isdigit ()) {
937 while (current
< end
&& current
[0].isdigit ()) {
940 if (current
< end
&& current
[0].tolower () == 'e') {
942 if (current
< end
&& (current
[0] == '+' || current
[0] == '-')) {
945 while (current
< end
&& current
[0].isdigit ()) {
949 if (current
< end
&& current
[0].tolower () == 'f') {
952 type
= TokenType
.REAL_LITERAL
;
953 } else if (current
< end
&& current
== begin
+ 1
954 && begin
[0] == '0' && begin
[1] == 'x' && begin
[2].isxdigit ()) {
955 // hexadecimal integer literal
957 while (current
< end
&& current
[0].isxdigit ()) {
960 } else if (current
< end
&& is_ident_char (current
[0])) {
961 // allow identifiers to start with a digit
962 // as long as they contain at least one char
963 while (current
< end
&& is_ident_char (current
[0])) {
966 type
= TokenType
.IDENTIFIER
;
969 switch (current
[0]) {
971 type
= TokenType
.OPEN_BRACE
;
973 state_stack
+= State
.BRACE
;
977 type
= TokenType
.CLOSE_BRACE
;
979 if (state_stack
.length
> 0) {
980 state_stack
.length
--;
985 type
= TokenType
.OPEN_PARENS
;
987 state_stack
+= State
.PARENS
;
991 type
= TokenType
.CLOSE_PARENS
;
994 if (state_stack
.length
> 0) {
995 state_stack
.length
--;
997 if (in_template ()) {
998 type
= TokenType
.COMMA
;
1002 type
= TokenType
.OPEN_BRACKET
;
1003 state_stack
+= State
.BRACKET
;
1007 type
= TokenType
.CLOSE_BRACKET
;
1008 if (state_stack
.length
> 0) {
1009 state_stack
.length
--;
1014 type
= TokenType
.DOT
;
1016 if (current
< end
- 1) {
1017 if (current
[0] == '.' && current
[1] == '.') {
1018 type
= TokenType
.ELLIPSIS
;
1024 type
= TokenType
.COLON
;
1028 type
= TokenType
.COMMA
;
1032 type
= TokenType
.SEMICOLON
;
1036 type
= TokenType
.HASH
;
1040 type
= TokenType
.INTERR
;
1044 type
= TokenType
.BITWISE_OR
;
1046 if (current
< end
) {
1047 switch (current
[0]) {
1049 type
= TokenType
.ASSIGN_BITWISE_OR
;
1053 type
= TokenType
.OP_OR
;
1060 type
= TokenType
.BITWISE_AND
;
1062 if (current
< end
) {
1063 switch (current
[0]) {
1065 type
= TokenType
.ASSIGN_BITWISE_AND
;
1069 type
= TokenType
.OP_AND
;
1076 type
= TokenType
.CARRET
;
1078 if (current
< end
&& current
[0] == '=') {
1079 type
= TokenType
.ASSIGN_BITWISE_XOR
;
1084 type
= TokenType
.TILDE
;
1088 type
= TokenType
.ASSIGN
;
1090 if (current
< end
) {
1091 switch (current
[0]) {
1093 type
= TokenType
.OP_EQ
;
1097 type
= TokenType
.LAMBDA
;
1104 type
= TokenType
.OP_LT
;
1106 if (current
< end
) {
1107 switch (current
[0]) {
1109 type
= TokenType
.OP_LE
;
1113 type
= TokenType
.OP_SHIFT_LEFT
;
1115 if (current
< end
&& current
[0] == '=') {
1116 type
= TokenType
.ASSIGN_SHIFT_LEFT
;
1124 type
= TokenType
.OP_GT
;
1126 if (current
< end
&& current
[0] == '=') {
1127 type
= TokenType
.OP_GE
;
1132 type
= TokenType
.OP_NEG
;
1134 if (current
< end
&& current
[0] == '=') {
1135 type
= TokenType
.OP_NE
;
1140 type
= TokenType
.PLUS
;
1142 if (current
< end
) {
1143 switch (current
[0]) {
1145 type
= TokenType
.ASSIGN_ADD
;
1149 type
= TokenType
.OP_INC
;
1156 type
= TokenType
.MINUS
;
1158 if (current
< end
) {
1159 switch (current
[0]) {
1161 type
= TokenType
.ASSIGN_SUB
;
1165 type
= TokenType
.OP_DEC
;
1169 type
= TokenType
.OP_PTR
;
1176 type
= TokenType
.STAR
;
1178 if (current
< end
&& current
[0] == '=') {
1179 type
= TokenType
.ASSIGN_MUL
;
1184 switch (last_token
) {
1185 case TokenType
.ASSIGN
:
1186 case TokenType
.COMMA
:
1187 case TokenType
.MINUS
:
1188 case TokenType
.OP_AND
:
1189 case TokenType
.OP_EQ
:
1190 case TokenType
.OP_GE
:
1191 case TokenType
.OP_GT
:
1192 case TokenType
.OP_INC
:
1193 case TokenType
.OP_LE
:
1194 case TokenType
.OP_LT
:
1195 case TokenType
.OP_NE
:
1196 case TokenType
.OP_NEG
:
1197 case TokenType
.OP_OR
:
1198 case TokenType
.OPEN_BRACE
:
1199 case TokenType
.OPEN_PARENS
:
1200 case TokenType
.PLUS
:
1201 case TokenType
.RETURN
:
1202 type
= TokenType
.OPEN_REGEX_LITERAL
;
1203 state_stack
+= State
.REGEX_LITERAL
;
1207 type
= TokenType
.DIV
;
1209 if (current
< end
&& current
[0] == '=') {
1210 type
= TokenType
.ASSIGN_DIV
;
1218 type
= TokenType
.PERCENT
;
1220 if (current
< end
&& current
[0] == '=') {
1221 type
= TokenType
.ASSIGN_PERCENT
;
1227 if (begin
[0] == '\'') {
1228 type
= TokenType
.CHARACTER_LITERAL
;
1229 } else if (current
< end
- 6 && begin
[1] == '"' && begin
[2] == '"') {
1230 type
= TokenType
.VERBATIM_STRING_LITERAL
;
1231 token_length_in_chars
= 6;
1233 while (current
< end
- 4) {
1234 if (current
[0] == '"' && current
[1] == '"' && current
[2] == '"' && current
[3] != '"') {
1236 } else if (current
[0] == '\n') {
1240 token_length_in_chars
= 3;
1242 unichar u
= ((string) current
).get_char_validated ((long) (end
- current
));
1243 if (u
!= (unichar
) (-1)) {
1244 current
+= u
.to_utf8 (null);
1245 token_length_in_chars
++;
1247 Report
.error (get_source_reference (token_length_in_chars
), "invalid UTF-8 character");
1251 if (current
[0] == '"' && current
[1] == '"' && current
[2] == '"') {
1254 Report
.error (get_source_reference (token_length_in_chars
), "syntax error, expected \"\"\"");
1258 type
= TokenType
.STRING_LITERAL
;
1260 token_length_in_chars
= 2;
1262 while (current
< end
&& current
[0] != begin
[0]) {
1263 if (current
[0] == '\\') {
1265 token_length_in_chars
++;
1266 if (current
>= end
) {
1270 switch (current
[0]) {
1281 token_length_in_chars
++;
1284 // hexadecimal escape character
1286 token_length_in_chars
++;
1287 while (current
< end
&& current
[0].isxdigit ()) {
1289 token_length_in_chars
++;
1293 Report
.error (get_source_reference (token_length_in_chars
), "invalid escape sequence");
1296 } else if (current
[0] == '\n') {
1300 token_length_in_chars
= 1;
1302 unichar u
= ((string) current
).get_char_validated ((long) (end
- current
));
1303 if (u
!= (unichar
) (-1)) {
1304 current
+= u
.to_utf8 (null);
1305 token_length_in_chars
++;
1308 Report
.error (get_source_reference (token_length_in_chars
), "invalid UTF-8 character");
1311 if (current
< end
&& begin
[0] == '\'' && current
[0] != '\'') {
1312 // multiple characters in single character literal
1313 Report
.error (get_source_reference (token_length_in_chars
), "invalid character literal");
1316 if (current
< end
) {
1319 Report
.error (get_source_reference (token_length_in_chars
), "syntax error, expected %c".printf (begin
[0]));
1323 unichar u
= ((string) current
).get_char_validated ((long) (end
- current
));
1324 if (u
!= (unichar
) (-1)) {
1325 current
+= u
.to_utf8 (null);
1326 Report
.error (get_source_reference (0), "syntax error, unexpected character");
1329 Report
.error (get_source_reference (0), "invalid UTF-8 character");
1332 return read_token (out token_begin
, out token_end
);
1336 if (token_length_in_chars
< 0) {
1337 column
+= (int) (current
- begin
);
1339 column
+= token_length_in_chars
;
1342 token_end
= SourceLocation (current
, line
, column
- 1);
1354 if (_indent_spaces
== 0) {
1355 while (current
< end
&& current
[0] == '\t') {
1361 int space_count
= 0;
1362 while (current
< end
&& current
[0] == ' ') {
1368 tab_count
= space_count
/ _indent_spaces
;
1372 /* ignore comments and whitspace and other lines that contain no code */
1376 if ((current
< end
) && (current
[0] == '\n')) return -1;
1381 static bool matches (char* begin
, string keyword
) {
1382 char* keyword_array
= (char *) keyword
;
1383 long len
= keyword
.length
;
1384 for (int i
= 0; i
< len
; i
++) {
1385 if (begin
[i
] != keyword_array
[i
]) {
1392 bool whitespace () {
1394 while (current
< end
&& current
[0].isspace () && current
[0] != '\n' ) {
1401 if ((column
== 1) && (current
< end
) && (current
[0] == '#')) {
1409 inline
bool newline () {
1410 if (current
[0] == '\n') {
1417 bool skip_newlines () {
1418 bool new_lines
= false;
1420 while (newline ()) {
1425 current_indent_level
= 0;
1433 bool comment (bool file_comment
= false) {
1435 || current
> end
- 2
1436 || current
[0] != '/'
1437 || (current
[1] != '/' && current
[1] != '*')) {
1442 if (current
[1] == '/') {
1443 // single-line comment
1445 SourceReference source_reference
= null;
1447 source_reference
= get_source_reference (0);
1452 // skip until end of line or end of file
1453 while (current
< end
&& current
[0] != '\n') {
1457 /* do not ignore EOL if comment does not exclusively occupy the line */
1458 if (current
[0] == '\n' && last_token
== TokenType
.EOL
) {
1462 current_indent_level
= 0;
1465 if (source_reference
!= null) {
1466 push_comment (((string) begin
).substring (0, (long) (current
- begin
)), source_reference
, file_comment
);
1470 // delimited comment
1471 SourceReference source_reference
= null;
1472 if (file_comment
&& current
[2] == '*') {
1476 if (current
[2] == '*' || file_comment
) {
1477 source_reference
= get_source_reference (0);
1481 char* begin
= current
;
1483 while (current
< end
- 1
1484 && (current
[0] != '*' || current
[1] != '/')) {
1485 if (current
[0] == '\n') {
1492 if (current
== end
- 1) {
1493 Report
.error (get_source_reference (0), "syntax error, expected */");
1497 if (source_reference
!= null) {
1498 string comment
= ((string) begin
).substring (0, (long) (current
- begin
));
1499 push_comment (comment
, source_reference
, file_comment
);
1511 while (current
< end
&& current
[0] == '\t' ) {
1520 void skip_space_tabs () {
1521 while (whitespace () || skip_tabs () || comment () ) {
1527 while (whitespace () || comment ()) {
1531 public void parse_file_comments () {
1532 while (whitespace () || comment (true)) {
1537 void push_comment (string comment_item
, SourceReference source_reference
, bool file_comment
) {
1538 if (comment_item
[0] == '*') {
1539 if (_comment
!= null) {
1540 // extra doc comment, add it to source file comments
1541 source_file
.add_comment (_comment
);
1543 _comment
= new
Comment (comment_item
, source_reference
);
1547 source_file
.add_comment (new
Comment (comment_item
, source_reference
));
1553 * Clears and returns the content of the comment stack.
1555 * @return saved comment
1557 public Comment?
pop_comment () {
1558 if (_comment
== null) {
1562 var comment
= _comment
;
1567 bool pp_whitespace () {
1569 while (current
< end
&& current
[0].isspace () && current
[0] != '\n') {
1578 while (pp_whitespace () || comment ()) {
1582 void pp_directive () {
1589 char* begin
= current
;
1591 while (current
< end
&& current
[0].isalnum ()) {
1597 if (len
== 2 && matches (begin
, "if")) {
1599 } else if (len
== 4 && matches (begin
, "elif")) {
1601 } else if (len
== 4 && matches (begin
, "else")) {
1603 } else if (len
== 5 && matches (begin
, "endif")) {
1606 Report
.error (get_source_reference (-len
, len
), "syntax error, invalid preprocessing directive");
1609 if (conditional_stack
.length
> 0
1610 && conditional_stack
[conditional_stack
.length
- 1].skip_section
) {
1611 // skip lines until next preprocessing directive
1613 while (current
< end
) {
1614 if (bol
&& current
< end
&& current
[0] == '#') {
1615 // go back to begin of line
1616 current
-= (column
- 1);
1620 if (current
[0] == '\n') {
1624 } else if (!current
[0].isspace ()) {
1635 if (current
>= end
|| current
[0] != '\n') {
1636 Report
.error (get_source_reference (0), "syntax error, expected newline");
1640 void parse_pp_if () {
1643 bool condition
= parse_pp_expression ();
1647 conditional_stack
+= Conditional ();
1649 if (condition
&& (conditional_stack
.length
== 1 || !conditional_stack
[conditional_stack
.length
- 2].skip_section
)) {
1650 // condition true => process code within if
1651 conditional_stack
[conditional_stack
.length
- 1].matched
= true;
1653 // skip lines until next preprocessing directive
1654 conditional_stack
[conditional_stack
.length
- 1].skip_section
= true;
1658 void parse_pp_elif () {
1661 bool condition
= parse_pp_expression ();
1665 if (conditional_stack
.length
== 0 || conditional_stack
[conditional_stack
.length
- 1].else_found
) {
1666 Report
.error (get_source_reference (0), "syntax error, unexpected #elif");
1670 if (condition
&& !conditional_stack
[conditional_stack
.length
- 1].matched
1671 && (conditional_stack
.length
== 1 || !conditional_stack
[conditional_stack
.length
- 2].skip_section
)) {
1672 // condition true => process code within if
1673 conditional_stack
[conditional_stack
.length
- 1].matched
= true;
1674 conditional_stack
[conditional_stack
.length
- 1].skip_section
= false;
1676 // skip lines until next preprocessing directive
1677 conditional_stack
[conditional_stack
.length
- 1].skip_section
= true;
1681 void parse_pp_else () {
1684 if (conditional_stack
.length
== 0 || conditional_stack
[conditional_stack
.length
- 1].else_found
) {
1685 Report
.error (get_source_reference (0), "syntax error, unexpected #else");
1689 if (!conditional_stack
[conditional_stack
.length
- 1].matched
1690 && (conditional_stack
.length
== 1 || !conditional_stack
[conditional_stack
.length
- 2].skip_section
)) {
1691 // condition true => process code within if
1692 conditional_stack
[conditional_stack
.length
- 1].matched
= true;
1693 conditional_stack
[conditional_stack
.length
- 1].skip_section
= false;
1695 // skip lines until next preprocessing directive
1696 conditional_stack
[conditional_stack
.length
- 1].skip_section
= true;
1700 void parse_pp_endif () {
1703 if (conditional_stack
.length
== 0) {
1704 Report
.error (get_source_reference (0), "syntax error, unexpected #endif");
1708 conditional_stack
.length
--;
1711 bool parse_pp_symbol () {
1713 while (current
< end
&& is_ident_char (current
[0])) {
1720 Report
.error (get_source_reference (0), "syntax error, expected identifier");
1724 string identifier
= ((string) (current
- len
)).substring (0, len
);
1726 if (identifier
== "true") {
1728 } else if (identifier
== "false") {
1731 defined
= source_file
.context
.is_defined (identifier
);
1737 bool parse_pp_primary_expression () {
1738 if (current
>= end
) {
1739 Report
.error (get_source_reference (0), "syntax error, expected identifier");
1740 } else if (is_ident_char (current
[0])) {
1741 return parse_pp_symbol ();
1742 } else if (current
[0] == '(') {
1746 bool result
= parse_pp_expression ();
1748 if (current
< end
&& current
[0] == ')') {
1752 Report
.error (get_source_reference (0), "syntax error, expected `)'");
1756 Report
.error (get_source_reference (0), "syntax error, expected identifier");
1761 bool parse_pp_unary_expression () {
1762 if (current
< end
&& current
[0] == '!') {
1766 return !parse_pp_unary_expression ();
1769 return parse_pp_primary_expression ();
1772 bool parse_pp_equality_expression () {
1773 bool left
= parse_pp_unary_expression ();
1776 if (current
< end
- 1 && current
[0] == '=' && current
[1] == '=') {
1780 bool right
= parse_pp_unary_expression ();
1781 left
= (left
== right
);
1782 } else if (current
< end
- 1 && current
[0] == '!' && current
[1] == '=') {
1786 bool right
= parse_pp_unary_expression ();
1787 left
= (left
!= right
);
1795 bool parse_pp_and_expression () {
1796 bool left
= parse_pp_equality_expression ();
1798 while (current
< end
- 1 && current
[0] == '&' && current
[1] == '&') {
1802 bool right
= parse_pp_equality_expression ();
1803 left
= left
&& right
;
1808 bool parse_pp_or_expression () {
1809 bool left
= parse_pp_and_expression ();
1811 while (current
< end
- 1 && current
[0] == '|' && current
[1] == '|') {
1815 bool right
= parse_pp_and_expression ();
1816 left
= left
|| right
;
1821 bool parse_pp_expression () {
1822 return parse_pp_or_expression ();