1 // lex.cc -- Go frontend lexer.
3 // Copyright 2009 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
8 #include "go-diagnostics.h"
12 // Manage mapping from keywords to the Keyword codes.
17 // The structure which maps keywords to codes.
21 const char* keystring
;
26 // Return the parsecode corresponding to KEYSTRING, or
27 // KEYWORD_INVALID if it is not a keyword.
29 keyword_to_code(const char* keyword
, size_t len
) const;
31 // Return the string for a keyword.
33 keyword_to_string(Keyword
) const;
36 static const Mapping mapping_
[];
37 static const int count_
;
40 // Mapping from keyword string to keyword code. This array must be
41 // kept in sorted order, and the order must match the Keyword enum.
42 // Strings are looked up using bsearch.
44 const Keywords::Mapping
45 Keywords::mapping_
[] =
47 { NULL
, KEYWORD_INVALID
},
48 { "__asm__", KEYWORD_ASM
},
49 { "break", KEYWORD_BREAK
},
50 { "case", KEYWORD_CASE
},
51 { "chan", KEYWORD_CHAN
},
52 { "const", KEYWORD_CONST
},
53 { "continue", KEYWORD_CONTINUE
},
54 { "default", KEYWORD_DEFAULT
},
55 { "defer", KEYWORD_DEFER
},
56 { "else", KEYWORD_ELSE
},
57 { "fallthrough", KEYWORD_FALLTHROUGH
},
58 { "for", KEYWORD_FOR
},
59 { "func", KEYWORD_FUNC
},
61 { "goto", KEYWORD_GOTO
},
63 { "import", KEYWORD_IMPORT
},
64 { "interface", KEYWORD_INTERFACE
},
65 { "map", KEYWORD_MAP
},
66 { "package", KEYWORD_PACKAGE
},
67 { "range", KEYWORD_RANGE
},
68 { "return", KEYWORD_RETURN
},
69 { "select", KEYWORD_SELECT
},
70 { "struct", KEYWORD_STRUCT
},
71 { "switch", KEYWORD_SWITCH
},
72 { "type", KEYWORD_TYPE
},
73 { "var", KEYWORD_VAR
}
76 // Number of entries in the map.
78 const int Keywords::count_
=
79 sizeof(Keywords::mapping_
) / sizeof(Keywords::mapping_
[0]);
81 // Comparison function passed to bsearch.
86 struct Keywords_search_key
93 keyword_compare(const void* keyv
, const void* mapv
)
95 const Keywords_search_key
* key
=
96 static_cast<const Keywords_search_key
*>(keyv
);
97 const Keywords::Mapping
* map
=
98 static_cast<const Keywords::Mapping
*>(mapv
);
99 if (map
->keystring
== NULL
)
101 int i
= strncmp(key
->str
, map
->keystring
, key
->len
);
104 if (map
->keystring
[key
->len
] != '\0')
111 // Convert a string to a keyword code. Return KEYWORD_INVALID if the
112 // string is not a keyword.
115 Keywords::keyword_to_code(const char* keyword
, size_t len
) const
117 Keywords_search_key key
;
120 void* mapv
= bsearch(&key
,
123 sizeof(this->mapping_
[0]),
126 return KEYWORD_INVALID
;
127 Mapping
* map
= static_cast<Mapping
*>(mapv
);
131 // Convert a keyword code to a string.
134 Keywords::keyword_to_string(Keyword code
) const
136 go_assert(code
> KEYWORD_INVALID
&& code
< this->count_
);
137 const Mapping
* map
= &this->mapping_
[code
];
138 go_assert(map
->keycode
== code
);
139 return map
->keystring
;
142 // There is one instance of the Keywords class.
144 static Keywords keywords
;
148 // Make a general token.
150 Token::Token(Classification classification
, Location location
)
151 : classification_(classification
), location_(location
)
162 // Clear a token--release memory.
167 if (this->classification_
== TOKEN_INTEGER
168 || this->classification_
== TOKEN_CHARACTER
)
169 mpz_clear(this->u_
.integer_value
);
170 else if (this->classification_
== TOKEN_FLOAT
171 || this->classification_
== TOKEN_IMAGINARY
)
172 mpfr_clear(this->u_
.float_value
);
175 // Construct a token.
177 Token::Token(const Token
& tok
)
178 : classification_(tok
.classification_
), location_(tok
.location_
)
180 switch (this->classification_
)
186 this->u_
.keyword
= tok
.u_
.keyword
;
188 case TOKEN_IDENTIFIER
:
190 this->u_
.string_value
= tok
.u_
.string_value
;
193 this->u_
.op
= tok
.u_
.op
;
195 case TOKEN_CHARACTER
:
197 mpz_init_set(this->u_
.integer_value
, tok
.u_
.integer_value
);
200 case TOKEN_IMAGINARY
:
201 mpfr_init_set(this->u_
.float_value
, tok
.u_
.float_value
, GMP_RNDN
);
208 // Assign to a token.
211 Token::operator=(const Token
& tok
)
214 this->classification_
= tok
.classification_
;
215 this->location_
= tok
.location_
;
216 switch (tok
.classification_
)
222 this->u_
.keyword
= tok
.u_
.keyword
;
224 case TOKEN_IDENTIFIER
:
225 this->u_
.identifier_value
.name
= tok
.u_
.identifier_value
.name
;
226 this->u_
.identifier_value
.is_exported
=
227 tok
.u_
.identifier_value
.is_exported
;
230 this->u_
.string_value
= tok
.u_
.string_value
;
233 this->u_
.op
= tok
.u_
.op
;
235 case TOKEN_CHARACTER
:
237 mpz_init_set(this->u_
.integer_value
, tok
.u_
.integer_value
);
240 case TOKEN_IMAGINARY
:
241 mpfr_init_set(this->u_
.float_value
, tok
.u_
.float_value
, GMP_RNDN
);
249 // Print the token for debugging.
252 Token::print(FILE* file
) const
254 switch (this->classification_
)
257 fprintf(file
, "invalid");
260 fprintf(file
, "EOF");
263 fprintf(file
, "keyword %s", keywords
.keyword_to_string(this->u_
.keyword
));
265 case TOKEN_IDENTIFIER
:
266 fprintf(file
, "identifier \"%s\"", this->u_
.string_value
->c_str());
269 fprintf(file
, "quoted string \"%s\"", this->u_
.string_value
->c_str());
271 case TOKEN_CHARACTER
:
272 fprintf(file
, "character ");
273 mpz_out_str(file
, 10, this->u_
.integer_value
);
276 fprintf(file
, "integer ");
277 mpz_out_str(file
, 10, this->u_
.integer_value
);
280 fprintf(file
, "float ");
281 mpfr_out_str(file
, 10, 0, this->u_
.float_value
, GMP_RNDN
);
283 case TOKEN_IMAGINARY
:
284 fprintf(file
, "imaginary ");
285 mpfr_out_str(file
, 10, 0, this->u_
.float_value
, GMP_RNDN
);
288 fprintf(file
, "operator ");
291 case OPERATOR_INVALID
:
292 fprintf(file
, "invalid");
297 case OPERATOR_ANDAND
:
339 case OPERATOR_LSHIFT
:
342 case OPERATOR_RSHIFT
:
348 case OPERATOR_BITCLEAR
:
354 case OPERATOR_CHANOP
:
360 case OPERATOR_PLUSEQ
:
363 case OPERATOR_MINUSEQ
:
372 case OPERATOR_MULTEQ
:
379 fprintf(file
, "%%=");
381 case OPERATOR_LSHIFTEQ
:
382 fprintf(file
, "<<=");
384 case OPERATOR_RSHIFTEQ
:
385 fprintf(file
, ">>=");
390 case OPERATOR_BITCLEAREQ
:
391 fprintf(file
, "&^=");
393 case OPERATOR_PLUSPLUS
:
396 case OPERATOR_MINUSMINUS
:
402 case OPERATOR_COLONEQ
:
405 case OPERATOR_SEMICOLON
:
414 case OPERATOR_LPAREN
:
417 case OPERATOR_RPAREN
:
420 case OPERATOR_LCURLY
:
423 case OPERATOR_RCURLY
:
426 case OPERATOR_LSQUARE
:
429 case OPERATOR_RSQUARE
:
443 Lex::Lex(const char* input_file_name
, FILE* input_file
, Linemap
* linemap
)
444 : input_file_name_(input_file_name
), input_file_(input_file
),
445 linemap_(linemap
), linebuf_(NULL
), linebufsize_(120), linesize_(0),
446 lineoff_(0), lineno_(0), add_semi_at_eol_(false), pragmas_(0),
447 extern_(), linknames_(NULL
)
449 this->linebuf_
= new char[this->linebufsize_
];
450 this->linemap_
->start_file(input_file_name
, 0);
455 delete[] this->linebuf_
;
458 // Read a new line from the file.
463 char* buf
= this->linebuf_
;
464 size_t size
= this->linebufsize_
;
466 FILE* file
= this->input_file_
;
479 size_t ns
= 2 * size
+ 1;
480 if (ns
< size
|| static_cast<ssize_t
>(ns
) < 0)
481 go_error_at(this->location(), "out of memory");
482 char* nb
= new char[ns
];
483 memcpy(nb
, buf
, cur
);
497 this->linebuf_
= buf
;
498 this->linebufsize_
= size
;
503 // See if we need to read a new line. Return true if there is a new
504 // line, false if we are at EOF.
509 if (this->lineoff_
< this->linesize_
)
512 ssize_t got
= this->get_line();
516 this->linesize_
= got
;
519 this->linemap_
->start_line(this->lineno_
, this->linesize_
);
524 // Get the current location.
527 Lex::location() const
529 return this->linemap_
->get_location(this->lineoff_
+ 1);
532 // Get a location slightly before the current one. This is used for
533 // slightly more efficient handling of operator tokens.
536 Lex::earlier_location(int chars
) const
538 return this->linemap_
->get_location(this->lineoff_
+ 1 - chars
);
541 // Get the next token.
546 bool saw_cpp_comment
= false;
549 if (!this->require_line())
551 bool add_semi_at_eol
= this->add_semi_at_eol_
;
552 this->add_semi_at_eol_
= false;
554 return this->make_operator(OPERATOR_SEMICOLON
, 1);
555 return this->make_eof_token();
558 if (!saw_cpp_comment
)
559 this->extern_
.clear();
560 saw_cpp_comment
= false;
562 const char* p
= this->linebuf_
+ this->lineoff_
;
563 const char* pend
= this->linebuf_
+ this->linesize_
;
567 unsigned char cc
= *p
;
570 case ' ': case '\t': case '\r':
572 // Skip whitespace quickly.
573 while (*p
== ' ' || *p
== '\t' || *p
== '\r')
580 bool add_semi_at_eol
= this->add_semi_at_eol_
;
581 this->add_semi_at_eol_
= false;
584 this->lineoff_
= p
- this->linebuf_
;
585 return this->make_operator(OPERATOR_SEMICOLON
, 1);
593 this->lineoff_
= p
+ 2 - this->linebuf_
;
594 this->skip_cpp_comment();
596 if (p
[-1] == '\n' && this->add_semi_at_eol_
)
598 saw_cpp_comment
= true;
600 else if (p
[1] == '*')
602 this->lineoff_
= p
+ 2 - this->linebuf_
;
603 Location location
= this->location();
604 bool found_newline
= false;
605 if (!this->skip_c_comment(&found_newline
))
606 return Token::make_invalid_token(location
);
607 if (found_newline
&& this->add_semi_at_eol_
)
609 this->add_semi_at_eol_
= false;
610 return this->make_operator(OPERATOR_SEMICOLON
, 1);
612 p
= this->linebuf_
+ this->lineoff_
;
613 pend
= this->linebuf_
+ this->linesize_
;
615 else if (p
[1] == '=')
617 this->add_semi_at_eol_
= false;
618 this->lineoff_
= p
+ 2 - this->linebuf_
;
619 return this->make_operator(OPERATOR_DIVEQ
, 2);
623 this->add_semi_at_eol_
= false;
624 this->lineoff_
= p
+ 1 - this->linebuf_
;
625 return this->make_operator(OPERATOR_DIV
, 1);
629 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
630 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
631 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
632 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
634 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
635 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
636 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
637 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
640 this->lineoff_
= p
- this->linebuf_
;
641 return this->gather_identifier();
643 case '0': case '1': case '2': case '3': case '4':
644 case '5': case '6': case '7': case '8': case '9':
645 this->add_semi_at_eol_
= true;
646 this->lineoff_
= p
- this->linebuf_
;
647 return this->gather_number();
650 this->add_semi_at_eol_
= true;
651 this->lineoff_
= p
- this->linebuf_
;
652 return this->gather_character();
655 this->add_semi_at_eol_
= true;
656 this->lineoff_
= p
- this->linebuf_
;
657 return this->gather_string();
660 this->add_semi_at_eol_
= true;
661 this->lineoff_
= p
- this->linebuf_
;
662 return this->gather_raw_string();
669 this->add_semi_at_eol_
= false;
670 Operator op
= this->three_character_operator(cc
, p
[1], p
[2]);
671 if (op
!= OPERATOR_INVALID
)
673 this->lineoff_
= p
+ 3 - this->linebuf_
;
674 return this->make_operator(op
, 3);
685 // '/' handled above.
694 this->add_semi_at_eol_
= false;
695 Operator op
= this->two_character_operator(cc
, p
[1]);
697 if (op
!= OPERATOR_INVALID
)
704 op
= this->one_character_operator(cc
);
707 this->lineoff_
= p
+ 1 - this->linebuf_
;
708 return this->make_operator(op
, chars
);
712 if (p
[1] >= '0' && p
[1] <= '9')
714 this->add_semi_at_eol_
= true;
715 this->lineoff_
= p
- this->linebuf_
;
716 return this->gather_number();
718 if (p
[1] == '.' && p
[2] == '.')
720 this->add_semi_at_eol_
= false;
721 this->lineoff_
= p
+ 3 - this->linebuf_
;
722 return this->make_operator(OPERATOR_ELLIPSIS
, 3);
724 this->add_semi_at_eol_
= false;
725 this->lineoff_
= p
+ 1 - this->linebuf_
;
726 return this->make_operator(OPERATOR_DOT
, 1);
732 this->lineoff_
= p
- this->linebuf_
;
733 const char *pnext
= this->advance_one_utf8_char(p
, &ci
,
736 // Ignore byte order mark at start of file.
743 if (Lex::is_unicode_letter(ci
))
744 return this->gather_identifier();
747 go_error_at(this->location(),
748 "invalid character 0x%x in input file",
758 this->lineoff_
= p
- this->linebuf_
;
762 // Fetch one UTF-8 character from a string. Set *VALUE to the value.
763 // Return the number of bytes read from the string. Returns 0 if the
764 // string does not point to a valid UTF-8 character.
767 Lex::fetch_char(const char* p
, unsigned int* value
)
769 unsigned char c
= *p
;
775 else if ((c
& 0xe0) == 0xc0
776 && (p
[1] & 0xc0) == 0x80)
778 *value
= (((c
& 0x1f) << 6)
787 else if ((c
& 0xf0) == 0xe0
788 && (p
[1] & 0xc0) == 0x80
789 && (p
[2] & 0xc0) == 0x80)
791 *value
= (((c
& 0xf) << 12)
792 + ((p
[1] & 0x3f) << 6)
801 else if ((c
& 0xf8) == 0xf0
802 && (p
[1] & 0xc0) == 0x80
803 && (p
[2] & 0xc0) == 0x80
804 && (p
[3] & 0xc0) == 0x80)
806 *value
= (((c
& 0x7) << 18)
807 + ((p
[1] & 0x3f) << 12)
808 + ((p
[2] & 0x3f) << 6)
810 if (*value
<= 0xffff)
819 /* Invalid encoding. Return the Unicode replacement
826 // Advance one UTF-8 character. Return the pointer beyond the
827 // character. Set *VALUE to the value. Set *ISSUED_ERROR if an error
831 Lex::advance_one_utf8_char(const char* p
, unsigned int* value
,
834 *issued_error
= false;
838 go_error_at(this->location(), "invalid NUL byte");
839 *issued_error
= true;
844 int adv
= Lex::fetch_char(p
, value
);
847 go_error_at(this->location(), "invalid UTF-8 encoding");
848 *issued_error
= true;
852 // Warn about byte order mark, except at start of file.
853 if (*value
== 0xfeff && (this->lineno_
!= 1 || this->lineoff_
!= 0))
855 go_error_at(this->location(), "Unicode (UTF-8) BOM in middle of file");
856 *issued_error
= true;
862 // Pick up an identifier.
865 Lex::gather_identifier()
867 const char* pstart
= this->linebuf_
+ this->lineoff_
;
868 const char* p
= pstart
;
869 const char* pend
= this->linebuf_
+ this->linesize_
;
870 bool is_first
= true;
871 bool is_exported
= false;
872 bool has_non_ascii_char
= false;
876 unsigned char cc
= *p
;
879 if ((cc
< 'A' || cc
> 'Z')
880 && (cc
< 'a' || cc
> 'z')
882 && (cc
< '0' || cc
> '9'))
884 // Check for an invalid character here, as we get better
885 // error behaviour if we swallow them as part of the
886 // identifier we are building.
887 if ((cc
>= ' ' && cc
< 0x7f)
893 this->lineoff_
= p
- this->linebuf_
;
894 go_error_at(this->location(),
895 "invalid character 0x%x in identifier",
897 if (!has_non_ascii_char
)
899 buf
.assign(pstart
, p
- pstart
);
900 has_non_ascii_char
= true;
902 if (!Lex::is_invalid_identifier(buf
))
903 buf
.append("$INVALID$");
908 is_exported
= cc
>= 'A' && cc
<= 'Z';
911 if (has_non_ascii_char
)
918 this->lineoff_
= p
- this->linebuf_
;
919 const char* pnext
= this->advance_one_utf8_char(p
, &ci
,
921 bool is_invalid
= false;
922 if (!Lex::is_unicode_letter(ci
) && !Lex::is_unicode_digit(ci
))
924 // There is no valid place for a non-ASCII character
925 // other than an identifier, so we get better error
926 // handling behaviour if we swallow this character after
929 go_error_at(this->location(),
930 "invalid character 0x%x in identifier",
936 is_exported
= Lex::is_unicode_uppercase(ci
);
939 if (!has_non_ascii_char
)
941 buf
.assign(pstart
, p
- pstart
);
942 has_non_ascii_char
= true;
944 if (is_invalid
&& !Lex::is_invalid_identifier(buf
))
945 buf
.append("$INVALID$");
946 buf
.append(p
, pnext
- p
);
950 Location location
= this->location();
951 this->add_semi_at_eol_
= true;
952 this->lineoff_
= p
- this->linebuf_
;
953 if (has_non_ascii_char
)
954 return Token::make_identifier_token(buf
, is_exported
, location
);
957 Keyword code
= keywords
.keyword_to_code(pstart
, p
- pstart
);
958 if (code
== KEYWORD_INVALID
)
959 return Token::make_identifier_token(std::string(pstart
, p
- pstart
),
960 is_exported
, location
);
966 case KEYWORD_CONTINUE
:
967 case KEYWORD_FALLTHROUGH
:
971 this->add_semi_at_eol_
= false;
974 return Token::make_keyword_token(code
, location
);
979 // Return whether C is a hex digit.
982 Lex::is_hex_digit(char c
)
984 return ((c
>= '0' && c
<= '9')
985 || (c
>= 'A' && c
<= 'F')
986 || (c
>= 'a' && c
<= 'f'));
992 // for use by Lex::hex_val
993 static const unsigned char hex_value_lookup_table
[256] =
995 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // NUL SOH STX ETX EOT ENQ ACK BEL
996 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // BS HT LF VT FF CR SO SI
997 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // DLE DC1 DC2 DC3 DC4 NAK SYN ETB
998 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // CAN EM SUB ESC FS GS RS US
999 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // SP ! " # $ % & '
1000 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // ( ) * + , - . /
1001 0, 1, 2, 3, 4, 5, 6, 7, // 0 1 2 3 4 5 6 7
1002 8, 9, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // 8 9 : ; < = > ?
1003 NHV
, 10, 11, 12, 13, 14, 15, NHV
, // @ A B C D E F G
1004 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // H I J K L M N O
1005 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // P Q R S T U V W
1006 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // X Y Z [ \ ] ^ _
1007 NHV
, 10, 11, 12, 13, 14, 15, NHV
, // ` a b c d e f g
1008 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // h i j k l m n o
1009 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // p q r s t u v w
1010 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, // x y z { | } ~
1011 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1012 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1013 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1014 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1015 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1016 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1017 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1018 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1019 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1020 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1021 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1022 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1023 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1024 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1025 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, //
1026 NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
, NHV
//
1030 Lex::hex_val(char c
)
1032 return hex_value_lookup_table
[static_cast<unsigned char>(c
)];
1035 // Return whether an exponent could start at P.
1038 Lex::could_be_exponent(const char* p
, const char* pend
)
1040 if (*p
!= 'e' && *p
!= 'E')
1045 if (*p
== '+' || *p
== '-')
1051 return *p
>= '0' && *p
<= '9';
1054 // Pick up a number.
1057 Lex::gather_number()
1059 const char* pstart
= this->linebuf_
+ this->lineoff_
;
1060 const char* p
= pstart
;
1061 const char* pend
= this->linebuf_
+ this->linesize_
;
1063 Location location
= this->location();
1074 const char* pnum
= p
;
1078 if ((p
[1] == 'x' || p
[1] == 'X')
1079 && Lex::is_hex_digit(p
[2]))
1086 if (!Lex::is_hex_digit(*p
))
1097 if (*p
< '0' || *p
> '9')
1103 // A partial token that looks like an octal literal might actually be the
1104 // beginning of a floating-point or imaginary literal.
1105 if (base
== 16 || (*p
!= '.' && *p
!= 'i' && !Lex::could_be_exponent(p
, pend
)))
1107 std::string
s(pnum
, p
- pnum
);
1109 int r
= mpz_init_set_str(val
, s
.c_str(), base
);
1113 go_error_at(this->location(), "invalid octal literal");
1115 go_error_at(this->location(), "invalid hex literal");
1121 this->lineoff_
= p
- this->linebuf_
;
1122 Token ret
= Token::make_integer_token(val
, location
);
1130 if (*p
< '0' || *p
> '9')
1135 if (*p
!= '.' && *p
!= 'i' && !Lex::could_be_exponent(p
, pend
))
1137 std::string
s(pnum
, p
- pnum
);
1139 int r
= mpz_init_set_str(val
, s
.c_str(), 10);
1145 this->lineoff_
= p
- this->linebuf_
;
1146 Token ret
= Token::make_integer_token(val
, location
);
1153 bool dot
= *p
== '.';
1159 if (*p
== '+' || *p
== '-')
1165 if (*p
< '0' || *p
> '9')
1170 if (dot
&& Lex::could_be_exponent(p
, pend
))
1173 if (*p
== '+' || *p
== '-')
1177 if (*p
< '0' || *p
> '9')
1184 std::string
s(pnum
, p
- pnum
);
1186 int r
= mpfr_init_set_str(val
, s
.c_str(), 10, GMP_RNDN
);
1190 mpfr_neg(val
, val
, GMP_RNDN
);
1192 bool is_imaginary
= *p
== 'i';
1196 this->lineoff_
= p
- this->linebuf_
;
1199 Token ret
= Token::make_imaginary_token(val
, location
);
1205 Token ret
= Token::make_float_token(val
, location
);
1211 // Advance one character, possibly escaped. Return the pointer beyond
1212 // the character. Set *VALUE to the character. Set *IS_CHARACTER if
1213 // this is a character (e.g., 'a' or '\u1234') rather than a byte
1214 // value (e.g., '\001').
1217 Lex::advance_one_char(const char* p
, bool is_single_quote
, unsigned int* value
,
1221 *is_character
= true;
1225 const char* ret
= this->advance_one_utf8_char(p
, value
, &issued_error
);
1227 && (*value
== '\'' || *value
== '\n')
1229 go_error_at(this->location(), "invalid character literal");
1237 case '0': case '1': case '2': case '3':
1238 case '4': case '5': case '6': case '7':
1239 *is_character
= false;
1240 if (p
[1] >= '0' && p
[1] <= '7'
1241 && p
[2] >= '0' && p
[2] <= '7')
1243 *value
= ((Lex::octal_value(p
[0]) << 6)
1244 + (Lex::octal_value(p
[1]) << 3)
1245 + Lex::octal_value(p
[2]));
1248 go_error_at(this->location(), "invalid octal constant");
1253 go_error_at(this->location(), "invalid octal character");
1254 return (p
[1] >= '0' && p
[1] <= '7'
1259 *is_character
= false;
1260 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2]))
1262 *value
= (Lex::hex_val(p
[1]) << 4) + Lex::hex_val(p
[2]);
1265 go_error_at(this->location(), "invalid hex character");
1266 return (Lex::is_hex_digit(p
[1])
1295 if (!is_single_quote
)
1296 go_error_at(this->location(), "invalid quoted character");
1300 if (is_single_quote
)
1301 go_error_at(this->location(), "invalid quoted character");
1306 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2])
1307 && Lex::is_hex_digit(p
[3]) && Lex::is_hex_digit(p
[4]))
1309 *value
= ((Lex::hex_val(p
[1]) << 12)
1310 + (Lex::hex_val(p
[2]) << 8)
1311 + (Lex::hex_val(p
[3]) << 4)
1312 + Lex::hex_val(p
[4]));
1313 if (*value
>= 0xd800 && *value
< 0xe000)
1315 go_error_at(this->location(),
1316 "invalid unicode code point 0x%x",
1318 // Use the replacement character.
1323 go_error_at(this->location(), "invalid little unicode code point");
1327 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2])
1328 && Lex::is_hex_digit(p
[3]) && Lex::is_hex_digit(p
[4])
1329 && Lex::is_hex_digit(p
[5]) && Lex::is_hex_digit(p
[6])
1330 && Lex::is_hex_digit(p
[7]) && Lex::is_hex_digit(p
[8]))
1332 *value
= ((Lex::hex_val(p
[1]) << 28)
1333 + (Lex::hex_val(p
[2]) << 24)
1334 + (Lex::hex_val(p
[3]) << 20)
1335 + (Lex::hex_val(p
[4]) << 16)
1336 + (Lex::hex_val(p
[5]) << 12)
1337 + (Lex::hex_val(p
[6]) << 8)
1338 + (Lex::hex_val(p
[7]) << 4)
1339 + Lex::hex_val(p
[8]));
1340 if (*value
> 0x10ffff
1341 || (*value
>= 0xd800 && *value
< 0xe000))
1343 go_error_at(this->location(),
1344 "invalid unicode code point 0x%x",
1346 // Use the replacement character.
1351 go_error_at(this->location(), "invalid big unicode code point");
1355 go_error_at(this->location(), "invalid character after %<\\%>");
1362 // Append V to STR. IS_CHARACTER is true for a character which should
1363 // be stored in UTF-8, false for a general byte value which should be
1367 Lex::append_char(unsigned int v
, bool is_character
, std::string
* str
,
1372 if (v
<= 0x7f || !is_character
)
1377 else if (v
<= 0x7ff)
1379 buf
[0] = 0xc0 + (v
>> 6);
1380 buf
[1] = 0x80 + (v
& 0x3f);
1387 go_warning_at(location
, 0,
1388 "unicode code point 0x%x out of range in string", v
);
1389 // Turn it into the "replacement character".
1392 if (v
>= 0xd800 && v
< 0xe000)
1394 go_warning_at(location
, 0,
1395 "unicode code point 0x%x is invalid surrogate pair", v
);
1400 buf
[0] = 0xe0 + (v
>> 12);
1401 buf
[1] = 0x80 + ((v
>> 6) & 0x3f);
1402 buf
[2] = 0x80 + (v
& 0x3f);
1407 buf
[0] = 0xf0 + (v
>> 18);
1408 buf
[1] = 0x80 + ((v
>> 12) & 0x3f);
1409 buf
[2] = 0x80 + ((v
>> 6) & 0x3f);
1410 buf
[3] = 0x80 + (v
& 0x3f);
1414 str
->append(buf
, len
);
1417 // Pick up a character literal.
1420 Lex::gather_character()
1423 const char* pstart
= this->linebuf_
+ this->lineoff_
;
1424 const char* p
= pstart
;
1428 p
= this->advance_one_char(p
, true, &value
, &is_character
);
1432 go_error_at(this->location(), "unterminated character constant");
1433 this->lineoff_
= p
- this->linebuf_
;
1434 return this->make_invalid_token();
1438 mpz_init_set_ui(val
, value
);
1440 Location location
= this->location();
1441 this->lineoff_
= p
+ 1 - this->linebuf_
;
1442 Token ret
= Token::make_character_token(val
, location
);
1447 // Pick up a quoted string.
1450 Lex::gather_string()
1452 const char* pstart
= this->linebuf_
+ this->lineoff_
+ 1;
1453 const char* p
= pstart
;
1454 const char* pend
= this->linebuf_
+ this->linesize_
;
1459 Location loc
= this->location();
1462 this->lineoff_
= p
- this->linebuf_
;
1463 p
= this->advance_one_char(p
, false, &c
, &is_character
);
1466 go_error_at(this->location(), "unterminated string");
1470 Lex::append_char(c
, is_character
, &value
, loc
);
1473 Location location
= this->location();
1474 this->lineoff_
= p
+ 1 - this->linebuf_
;
1475 return Token::make_string_token(value
, location
);
1478 // Pick up a raw string.
1481 Lex::gather_raw_string()
1483 const char* p
= this->linebuf_
+ this->lineoff_
+ 1;
1484 const char* pend
= this->linebuf_
+ this->linesize_
;
1485 Location location
= this->location();
1494 this->lineoff_
= p
+ 1 - this->linebuf_
;
1495 return Token::make_string_token(value
, location
);
1497 Location loc
= this->location();
1500 this->lineoff_
= p
- this->linebuf_
;
1501 p
= this->advance_one_utf8_char(p
, &c
, &issued_error
);
1502 // "Carriage return characters ('\r') inside raw string literals
1503 // are discarded from the raw string value."
1505 Lex::append_char(c
, true, &value
, loc
);
1507 this->lineoff_
= p
- this->linebuf_
;
1508 if (!this->require_line())
1510 go_error_at(location
, "unterminated raw string");
1511 return Token::make_string_token(value
, location
);
1513 p
= this->linebuf_
+ this->lineoff_
;
1514 pend
= this->linebuf_
+ this->linesize_
;
1518 // If C1 C2 C3 are a three character operator, return the code.
1521 Lex::three_character_operator(char c1
, char c2
, char c3
)
1525 if (c1
== '<' && c2
== '<')
1526 return OPERATOR_LSHIFTEQ
;
1527 else if (c1
== '>' && c2
== '>')
1528 return OPERATOR_RSHIFTEQ
;
1529 else if (c1
== '&' && c2
== '^')
1530 return OPERATOR_BITCLEAREQ
;
1532 return OPERATOR_INVALID
;
1535 // If C1 C2 are a two character operator, return the code.
1538 Lex::two_character_operator(char c1
, char c2
)
1544 return OPERATOR_OROR
;
1546 return OPERATOR_OREQ
;
1550 return OPERATOR_ANDAND
;
1552 return OPERATOR_BITCLEAR
;
1554 return OPERATOR_ANDEQ
;
1558 return OPERATOR_XOREQ
;
1562 return OPERATOR_EQEQ
;
1566 return OPERATOR_NOTEQ
;
1572 return OPERATOR_LSHIFT
;
1574 return OPERATOR_CHANOP
;
1580 return OPERATOR_RSHIFT
;
1584 return OPERATOR_MULTEQ
;
1588 return OPERATOR_DIVEQ
;
1592 return OPERATOR_MODEQ
;
1597 this->add_semi_at_eol_
= true;
1598 return OPERATOR_PLUSPLUS
;
1601 return OPERATOR_PLUSEQ
;
1606 this->add_semi_at_eol_
= true;
1607 return OPERATOR_MINUSMINUS
;
1610 return OPERATOR_MINUSEQ
;
1614 return OPERATOR_COLONEQ
;
1619 return OPERATOR_INVALID
;
1622 // If character C is an operator, return the code.
1625 Lex::one_character_operator(char c
)
1634 return OPERATOR_PLUS
;
1636 return OPERATOR_MINUS
;
1640 return OPERATOR_XOR
;
1642 return OPERATOR_MULT
;
1644 return OPERATOR_DIV
;
1646 return OPERATOR_MOD
;
1648 return OPERATOR_AND
;
1650 return OPERATOR_NOT
;
1654 return OPERATOR_COLON
;
1656 return OPERATOR_SEMICOLON
;
1658 return OPERATOR_DOT
;
1660 return OPERATOR_COMMA
;
1662 return OPERATOR_LPAREN
;
1664 this->add_semi_at_eol_
= true;
1665 return OPERATOR_RPAREN
;
1667 return OPERATOR_LCURLY
;
1669 this->add_semi_at_eol_
= true;
1670 return OPERATOR_RCURLY
;
1672 return OPERATOR_LSQUARE
;
1674 this->add_semi_at_eol_
= true;
1675 return OPERATOR_RSQUARE
;
1677 return OPERATOR_INVALID
;
1681 // Skip a C-style comment.
1684 Lex::skip_c_comment(bool* found_newline
)
1688 if (!this->require_line())
1690 go_error_at(this->location(), "unterminated comment");
1694 const char* p
= this->linebuf_
+ this->lineoff_
;
1695 const char* pend
= this->linebuf_
+ this->linesize_
;
1699 if (p
[0] == '*' && p
+ 1 < pend
&& p
[1] == '/')
1701 this->lineoff_
= p
+ 2 - this->linebuf_
;
1706 *found_newline
= true;
1708 this->lineoff_
= p
- this->linebuf_
;
1711 p
= this->advance_one_utf8_char(p
, &c
, &issued_error
);
1714 this->lineoff_
= p
- this->linebuf_
;
1718 // Skip a C++-style comment.
1721 Lex::skip_cpp_comment()
1723 // Ensure that if EXTERN_ is set, it means that we just saw a
1724 // //extern comment.
1725 this->extern_
.clear();
1727 Location loc
= this->location();
1728 size_t lineoff
= this->lineoff_
;
1730 const char* p
= this->linebuf_
+ lineoff
;
1731 const char* pend
= this->linebuf_
+ this->linesize_
;
1733 const char* pcheck
= p
;
1734 bool saw_error
= false;
1735 while (pcheck
< pend
)
1737 this->lineoff_
= pcheck
- this->linebuf_
;
1740 pcheck
= this->advance_one_utf8_char(pcheck
, &c
, &issued_error
);
1748 // Recognize various magic comments at the start of a line.
1752 // Not at the start of the line. (lineoff == 2 because of the
1753 // two characters in "//").
1758 && (pend
[-1] == ' ' || pend
[-1] == '\t'
1759 || pend
[-1] == '\r' || pend
[-1] == '\n'))
1762 // A C++ comment at the start of the line of the form
1763 // //line FILE:LINENO
1764 // is interpreted as setting the file name and line number of the
1765 // next source line.
1766 if (pend
- p
> 5 && memcmp(p
, "line ", 5) == 0)
1769 while (p
< pend
&& *p
== ' ')
1771 const char* pcolon
= static_cast<const char*>(memchr(p
, ':', pend
- p
));
1774 && pcolon
[1] <= '9')
1777 long lineno
= strtol(pcolon
+ 1, &plend
, 10);
1778 if (plend
> pcolon
+ 1
1783 && lineno
< 0x7fffffff)
1785 unsigned int filelen
= pcolon
- p
;
1786 char* file
= new char[filelen
+ 1];
1787 memcpy(file
, p
, filelen
);
1788 file
[filelen
] = '\0';
1790 this->linemap_
->start_file(file
, lineno
);
1791 this->lineno_
= lineno
- 1;
1799 // As a special gccgo extension, a C++ comment at the start of the
1802 // which immediately precedes a function declaration means that the
1803 // external name of the function declaration is NAME. This is
1804 // normally used to permit Go code to call a C function.
1805 if (pend
- p
> 7 && memcmp(p
, "extern ", 7) == 0)
1808 while (p
< pend
&& (*p
== ' ' || *p
== '\t'))
1811 this->extern_
= std::string(p
, pend
- p
);
1815 // All other special comments start with "go:".
1817 if (pend
- p
< 4 || memcmp(p
, "go:", 3) != 0)
1820 const char *ps
= p
+ 3;
1821 while (ps
< pend
&& *ps
!= ' ' && *ps
!= '\t')
1823 std::string verb
= std::string(p
, ps
- p
);
1825 if (verb
== "go:linkname")
1827 // As in the gc compiler, set the external link name for a Go symbol.
1828 std::string go_name
;
1829 std::string ext_name
;
1830 bool is_exported
= false;
1833 while (ps
< pend
&& (*ps
== ' ' || *ps
== '\t'))
1837 const char* pg
= ps
;
1841 ps
= this->advance_one_utf8_char(ps
, &c
, &issued_error
);
1842 is_exported
= Lex::is_unicode_uppercase(c
);
1844 while (ps
< pend
&& *ps
!= ' ' && *ps
!= '\t')
1847 go_name
= std::string(pg
, ps
- pg
);
1848 while (ps
< pend
&& (*ps
== ' ' || *ps
== '\t'))
1853 const char* pc
= ps
;
1854 while (ps
< pend
&& *ps
!= ' ' && *ps
!= '\t')
1857 ext_name
= std::string(pc
, ps
- pc
);
1865 if (go_name
.empty() || ext_name
.empty())
1866 go_error_at(loc
, "usage: //go:linkname localname linkname");
1869 if (this->linknames_
== NULL
)
1870 this->linknames_
= new Linknames();
1871 (*this->linknames_
)[go_name
] = Linkname(ext_name
, is_exported
, loc
);
1874 else if (verb
== "go:nointerface")
1876 // For field tracking analysis: a //go:nointerface comment means
1877 // that the next interface method should not be stored in the
1878 // type descriptor. This permits it to be discarded if it is
1880 this->pragmas_
|= GOPRAGMA_NOINTERFACE
;
1882 else if (verb
== "go:noescape")
1884 // Applies to the next function declaration. Any arguments do
1886 // FIXME: Not implemented.
1887 this->pragmas_
|= GOPRAGMA_NOESCAPE
;
1889 else if (verb
== "go:nosplit")
1891 // Applies to the next function. Do not split the stack when
1892 // entering the function.
1893 this->pragmas_
|= GOPRAGMA_NOSPLIT
;
1895 else if (verb
== "go:noinline")
1897 // Applies to the next function. Do not inline the function.
1898 this->pragmas_
|= GOPRAGMA_NOINLINE
;
1900 else if (verb
== "go:notinheap")
1902 // Applies to the next type. The type does not live in the heap.
1903 this->pragmas_
|= GOPRAGMA_NOTINHEAP
;
1905 else if (verb
== "go:systemstack")
1907 // Applies to the next function. It must run on the system stack.
1908 // FIXME: Should only work when compiling the runtime package.
1909 // FIXME: Not implemented.
1910 this->pragmas_
|= GOPRAGMA_SYSTEMSTACK
;
1912 else if (verb
== "go:nowritebarrier")
1914 // Applies to the next function. If the function needs to use
1915 // any write barriers, it should emit an error instead.
1916 // FIXME: Should only work when compiling the runtime package.
1917 this->pragmas_
|= GOPRAGMA_NOWRITEBARRIER
;
1919 else if (verb
== "go:nowritebarrierrec")
1921 // Applies to the next function. If the function, or any
1922 // function that it calls, needs to use any write barriers, it
1923 // should emit an error instead.
1924 // FIXME: Should only work when compiling the runtime package.
1925 // FIXME: currently treated the same as go:nowritebarrier
1926 this->pragmas_
|= GOPRAGMA_NOWRITEBARRIERREC
;
1928 else if (verb
== "go:cgo_unsafe_args")
1930 // Applies to the next function. Taking the address of any
1931 // argument implies taking the address of all arguments.
1932 // FIXME: Not implemented.
1933 this->pragmas_
|= GOPRAGMA_CGOUNSAFEARGS
;
1935 else if (verb
== "go:uintptrescapes")
1937 // Applies to the next function. If an argument is a pointer
1938 // converted to uintptr, then the pointer escapes.
1939 // FIXME: Not implemented.
1940 this->pragmas_
|= GOPRAGMA_UINTPTRESCAPES
;
1944 // The Unicode tables use this struct.
1946 struct Unicode_range
1948 // The low end of the range.
1950 // The high end of the range.
1952 // The stride. This entries represents low, low + stride, low + 2 *
1953 // stride, etc., up to high.
1954 unsigned int stride
;
1957 // A table of whitespace characters--Unicode code points classified as
1958 // "Space", "C" locale whitespace characters, the "next line" control
1959 // character (0085), the line separator (2028), the paragraph
1960 // separator (2029), and the "zero-width non-break space" (feff).
1962 static const Unicode_range unicode_space
[] =
1964 { 0x0009, 0x000d, 1 },
1965 { 0x0020, 0x0020, 1 },
1966 { 0x0085, 0x0085, 1 },
1967 { 0x00a0, 0x00a0, 1 },
1968 { 0x1680, 0x1680, 1 },
1969 { 0x180e, 0x180e, 1 },
1970 { 0x2000, 0x200a, 1 },
1971 { 0x2028, 0x2029, 1 },
1972 { 0x202f, 0x202f, 1 },
1973 { 0x205f, 0x205f, 1 },
1974 { 0x3000, 0x3000, 1 },
1975 { 0xfeff, 0xfeff, 1 },
1978 // A table of Unicode digits--Unicode code points classified as
1981 static const Unicode_range unicode_digits
[] =
1983 { 0x0030, 0x0039, 1},
1984 { 0x0660, 0x0669, 1},
1985 { 0x06f0, 0x06f9, 1},
1986 { 0x07c0, 0x07c9, 1},
1987 { 0x0966, 0x096f, 1},
1988 { 0x09e6, 0x09ef, 1},
1989 { 0x0a66, 0x0a6f, 1},
1990 { 0x0ae6, 0x0aef, 1},
1991 { 0x0b66, 0x0b6f, 1},
1992 { 0x0be6, 0x0bef, 1},
1993 { 0x0c66, 0x0c6f, 1},
1994 { 0x0ce6, 0x0cef, 1},
1995 { 0x0d66, 0x0d6f, 1},
1996 { 0x0e50, 0x0e59, 1},
1997 { 0x0ed0, 0x0ed9, 1},
1998 { 0x0f20, 0x0f29, 1},
1999 { 0x1040, 0x1049, 1},
2000 { 0x17e0, 0x17e9, 1},
2001 { 0x1810, 0x1819, 1},
2002 { 0x1946, 0x194f, 1},
2003 { 0x19d0, 0x19d9, 1},
2004 { 0x1b50, 0x1b59, 1},
2005 { 0xff10, 0xff19, 1},
2006 { 0x104a0, 0x104a9, 1},
2007 { 0x1d7ce, 0x1d7ff, 1},
2010 // A table of Unicode letters--Unicode code points classified as
2013 static const Unicode_range unicode_letters
[] =
2015 { 0x0041, 0x005a, 1},
2016 { 0x0061, 0x007a, 1},
2017 { 0x00aa, 0x00b5, 11},
2018 { 0x00ba, 0x00c0, 6},
2019 { 0x00c1, 0x00d6, 1},
2020 { 0x00d8, 0x00f6, 1},
2021 { 0x00f8, 0x02c1, 1},
2022 { 0x02c6, 0x02d1, 1},
2023 { 0x02e0, 0x02e4, 1},
2024 { 0x02ec, 0x02ee, 2},
2025 { 0x0370, 0x0374, 1},
2026 { 0x0376, 0x0377, 1},
2027 { 0x037a, 0x037d, 1},
2028 { 0x037f, 0x0386, 7},
2029 { 0x0388, 0x038a, 1},
2030 { 0x038c, 0x038e, 2},
2031 { 0x038f, 0x03a1, 1},
2032 { 0x03a3, 0x03f5, 1},
2033 { 0x03f7, 0x0481, 1},
2034 { 0x048a, 0x052f, 1},
2035 { 0x0531, 0x0556, 1},
2036 { 0x0559, 0x0561, 8},
2037 { 0x0562, 0x0587, 1},
2038 { 0x05d0, 0x05ea, 1},
2039 { 0x05f0, 0x05f2, 1},
2040 { 0x0620, 0x064a, 1},
2041 { 0x066e, 0x066f, 1},
2042 { 0x0671, 0x06d3, 1},
2043 { 0x06d5, 0x06e5, 16},
2044 { 0x06e6, 0x06ee, 8},
2045 { 0x06ef, 0x06fa, 11},
2046 { 0x06fb, 0x06fc, 1},
2047 { 0x06ff, 0x0710, 17},
2048 { 0x0712, 0x072f, 1},
2049 { 0x074d, 0x07a5, 1},
2050 { 0x07b1, 0x07ca, 25},
2051 { 0x07cb, 0x07ea, 1},
2052 { 0x07f4, 0x07f5, 1},
2053 { 0x07fa, 0x0800, 6},
2054 { 0x0801, 0x0815, 1},
2055 { 0x081a, 0x0824, 10},
2056 { 0x0828, 0x0840, 24},
2057 { 0x0841, 0x0858, 1},
2058 { 0x08a0, 0x08b4, 1},
2059 { 0x0904, 0x0939, 1},
2060 { 0x093d, 0x0950, 19},
2061 { 0x0958, 0x0961, 1},
2062 { 0x0971, 0x0980, 1},
2063 { 0x0985, 0x098c, 1},
2064 { 0x098f, 0x0990, 1},
2065 { 0x0993, 0x09a8, 1},
2066 { 0x09aa, 0x09b0, 1},
2067 { 0x09b2, 0x09b6, 4},
2068 { 0x09b7, 0x09b9, 1},
2069 { 0x09bd, 0x09ce, 17},
2070 { 0x09dc, 0x09dd, 1},
2071 { 0x09df, 0x09e1, 1},
2072 { 0x09f0, 0x09f1, 1},
2073 { 0x0a05, 0x0a0a, 1},
2074 { 0x0a0f, 0x0a10, 1},
2075 { 0x0a13, 0x0a28, 1},
2076 { 0x0a2a, 0x0a30, 1},
2077 { 0x0a32, 0x0a33, 1},
2078 { 0x0a35, 0x0a36, 1},
2079 { 0x0a38, 0x0a39, 1},
2080 { 0x0a59, 0x0a5c, 1},
2081 { 0x0a5e, 0x0a72, 20},
2082 { 0x0a73, 0x0a74, 1},
2083 { 0x0a85, 0x0a8d, 1},
2084 { 0x0a8f, 0x0a91, 1},
2085 { 0x0a93, 0x0aa8, 1},
2086 { 0x0aaa, 0x0ab0, 1},
2087 { 0x0ab2, 0x0ab3, 1},
2088 { 0x0ab5, 0x0ab9, 1},
2089 { 0x0abd, 0x0ad0, 19},
2090 { 0x0ae0, 0x0ae1, 1},
2091 { 0x0af9, 0x0b05, 12},
2092 { 0x0b06, 0x0b0c, 1},
2093 { 0x0b0f, 0x0b10, 1},
2094 { 0x0b13, 0x0b28, 1},
2095 { 0x0b2a, 0x0b30, 1},
2096 { 0x0b32, 0x0b33, 1},
2097 { 0x0b35, 0x0b39, 1},
2098 { 0x0b3d, 0x0b5c, 31},
2099 { 0x0b5d, 0x0b5f, 2},
2100 { 0x0b60, 0x0b61, 1},
2101 { 0x0b71, 0x0b83, 18},
2102 { 0x0b85, 0x0b8a, 1},
2103 { 0x0b8e, 0x0b90, 1},
2104 { 0x0b92, 0x0b95, 1},
2105 { 0x0b99, 0x0b9a, 1},
2106 { 0x0b9c, 0x0b9e, 2},
2107 { 0x0b9f, 0x0ba3, 4},
2108 { 0x0ba4, 0x0ba8, 4},
2109 { 0x0ba9, 0x0baa, 1},
2110 { 0x0bae, 0x0bb9, 1},
2111 { 0x0bd0, 0x0c05, 53},
2112 { 0x0c06, 0x0c0c, 1},
2113 { 0x0c0e, 0x0c10, 1},
2114 { 0x0c12, 0x0c28, 1},
2115 { 0x0c2a, 0x0c39, 1},
2116 { 0x0c3d, 0x0c58, 27},
2117 { 0x0c59, 0x0c5a, 1},
2118 { 0x0c60, 0x0c61, 1},
2119 { 0x0c85, 0x0c8c, 1},
2120 { 0x0c8e, 0x0c90, 1},
2121 { 0x0c92, 0x0ca8, 1},
2122 { 0x0caa, 0x0cb3, 1},
2123 { 0x0cb5, 0x0cb9, 1},
2124 { 0x0cbd, 0x0cde, 33},
2125 { 0x0ce0, 0x0ce1, 1},
2126 { 0x0cf1, 0x0cf2, 1},
2127 { 0x0d05, 0x0d0c, 1},
2128 { 0x0d0e, 0x0d10, 1},
2129 { 0x0d12, 0x0d3a, 1},
2130 { 0x0d3d, 0x0d5f, 17},
2131 { 0x0d60, 0x0d61, 1},
2132 { 0x0d7a, 0x0d7f, 1},
2133 { 0x0d85, 0x0d96, 1},
2134 { 0x0d9a, 0x0db1, 1},
2135 { 0x0db3, 0x0dbb, 1},
2136 { 0x0dbd, 0x0dc0, 3},
2137 { 0x0dc1, 0x0dc6, 1},
2138 { 0x0e01, 0x0e30, 1},
2139 { 0x0e32, 0x0e33, 1},
2140 { 0x0e40, 0x0e46, 1},
2141 { 0x0e81, 0x0e82, 1},
2142 { 0x0e84, 0x0e87, 3},
2143 { 0x0e88, 0x0e8a, 2},
2144 { 0x0e8d, 0x0e94, 7},
2145 { 0x0e95, 0x0e97, 1},
2146 { 0x0e99, 0x0e9f, 1},
2147 { 0x0ea1, 0x0ea3, 1},
2148 { 0x0ea5, 0x0ea7, 2},
2149 { 0x0eaa, 0x0eab, 1},
2150 { 0x0ead, 0x0eb0, 1},
2151 { 0x0eb2, 0x0eb3, 1},
2152 { 0x0ebd, 0x0ec0, 3},
2153 { 0x0ec1, 0x0ec4, 1},
2154 { 0x0ec6, 0x0edc, 22},
2155 { 0x0edd, 0x0edf, 1},
2156 { 0x0f00, 0x0f40, 64},
2157 { 0x0f41, 0x0f47, 1},
2158 { 0x0f49, 0x0f6c, 1},
2159 { 0x0f88, 0x0f8c, 1},
2160 { 0x1000, 0x102a, 1},
2161 { 0x103f, 0x1050, 17},
2162 { 0x1051, 0x1055, 1},
2163 { 0x105a, 0x105d, 1},
2164 { 0x1061, 0x1065, 4},
2165 { 0x1066, 0x106e, 8},
2166 { 0x106f, 0x1070, 1},
2167 { 0x1075, 0x1081, 1},
2168 { 0x108e, 0x10a0, 18},
2169 { 0x10a1, 0x10c5, 1},
2170 { 0x10c7, 0x10cd, 6},
2171 { 0x10d0, 0x10fa, 1},
2172 { 0x10fc, 0x1248, 1},
2173 { 0x124a, 0x124d, 1},
2174 { 0x1250, 0x1256, 1},
2175 { 0x1258, 0x125a, 2},
2176 { 0x125b, 0x125d, 1},
2177 { 0x1260, 0x1288, 1},
2178 { 0x128a, 0x128d, 1},
2179 { 0x1290, 0x12b0, 1},
2180 { 0x12b2, 0x12b5, 1},
2181 { 0x12b8, 0x12be, 1},
2182 { 0x12c0, 0x12c2, 2},
2183 { 0x12c3, 0x12c5, 1},
2184 { 0x12c8, 0x12d6, 1},
2185 { 0x12d8, 0x1310, 1},
2186 { 0x1312, 0x1315, 1},
2187 { 0x1318, 0x135a, 1},
2188 { 0x1380, 0x138f, 1},
2189 { 0x13a0, 0x13f5, 1},
2190 { 0x13f8, 0x13fd, 1},
2191 { 0x1401, 0x166c, 1},
2192 { 0x166f, 0x167f, 1},
2193 { 0x1681, 0x169a, 1},
2194 { 0x16a0, 0x16ea, 1},
2195 { 0x16f1, 0x16f8, 1},
2196 { 0x1700, 0x170c, 1},
2197 { 0x170e, 0x1711, 1},
2198 { 0x1720, 0x1731, 1},
2199 { 0x1740, 0x1751, 1},
2200 { 0x1760, 0x176c, 1},
2201 { 0x176e, 0x1770, 1},
2202 { 0x1780, 0x17b3, 1},
2203 { 0x17d7, 0x17dc, 5},
2204 { 0x1820, 0x1877, 1},
2205 { 0x1880, 0x18a8, 1},
2206 { 0x18aa, 0x18b0, 6},
2207 { 0x18b1, 0x18f5, 1},
2208 { 0x1900, 0x191e, 1},
2209 { 0x1950, 0x196d, 1},
2210 { 0x1970, 0x1974, 1},
2211 { 0x1980, 0x19ab, 1},
2212 { 0x19b0, 0x19c9, 1},
2213 { 0x1a00, 0x1a16, 1},
2214 { 0x1a20, 0x1a54, 1},
2215 { 0x1aa7, 0x1b05, 94},
2216 { 0x1b06, 0x1b33, 1},
2217 { 0x1b45, 0x1b4b, 1},
2218 { 0x1b83, 0x1ba0, 1},
2219 { 0x1bae, 0x1baf, 1},
2220 { 0x1bba, 0x1be5, 1},
2221 { 0x1c00, 0x1c23, 1},
2222 { 0x1c4d, 0x1c4f, 1},
2223 { 0x1c5a, 0x1c7d, 1},
2224 { 0x1ce9, 0x1cec, 1},
2225 { 0x1cee, 0x1cf1, 1},
2226 { 0x1cf5, 0x1cf6, 1},
2227 { 0x1d00, 0x1dbf, 1},
2228 { 0x1e00, 0x1f15, 1},
2229 { 0x1f18, 0x1f1d, 1},
2230 { 0x1f20, 0x1f45, 1},
2231 { 0x1f48, 0x1f4d, 1},
2232 { 0x1f50, 0x1f57, 1},
2233 { 0x1f59, 0x1f5f, 2},
2234 { 0x1f60, 0x1f7d, 1},
2235 { 0x1f80, 0x1fb4, 1},
2236 { 0x1fb6, 0x1fbc, 1},
2237 { 0x1fbe, 0x1fc2, 4},
2238 { 0x1fc3, 0x1fc4, 1},
2239 { 0x1fc6, 0x1fcc, 1},
2240 { 0x1fd0, 0x1fd3, 1},
2241 { 0x1fd6, 0x1fdb, 1},
2242 { 0x1fe0, 0x1fec, 1},
2243 { 0x1ff2, 0x1ff4, 1},
2244 { 0x1ff6, 0x1ffc, 1},
2245 { 0x2071, 0x207f, 14},
2246 { 0x2090, 0x209c, 1},
2247 { 0x2102, 0x2107, 5},
2248 { 0x210a, 0x2113, 1},
2249 { 0x2115, 0x2119, 4},
2250 { 0x211a, 0x211d, 1},
2251 { 0x2124, 0x212a, 2},
2252 { 0x212b, 0x212d, 1},
2253 { 0x212f, 0x2139, 1},
2254 { 0x213c, 0x213f, 1},
2255 { 0x2145, 0x2149, 1},
2256 { 0x214e, 0x2183, 53},
2257 { 0x2184, 0x2c00, 2684},
2258 { 0x2c01, 0x2c2e, 1},
2259 { 0x2c30, 0x2c5e, 1},
2260 { 0x2c60, 0x2ce4, 1},
2261 { 0x2ceb, 0x2cee, 1},
2262 { 0x2cf2, 0x2cf3, 1},
2263 { 0x2d00, 0x2d25, 1},
2264 { 0x2d27, 0x2d2d, 6},
2265 { 0x2d30, 0x2d67, 1},
2266 { 0x2d6f, 0x2d80, 17},
2267 { 0x2d81, 0x2d96, 1},
2268 { 0x2da0, 0x2da6, 1},
2269 { 0x2da8, 0x2dae, 1},
2270 { 0x2db0, 0x2db6, 1},
2271 { 0x2db8, 0x2dbe, 1},
2272 { 0x2dc0, 0x2dc6, 1},
2273 { 0x2dc8, 0x2dce, 1},
2274 { 0x2dd0, 0x2dd6, 1},
2275 { 0x2dd8, 0x2dde, 1},
2276 { 0x2e2f, 0x3005, 470},
2277 { 0x3006, 0x3031, 43},
2278 { 0x3032, 0x3035, 1},
2279 { 0x303b, 0x303c, 1},
2280 { 0x3041, 0x3096, 1},
2281 { 0x309d, 0x309f, 1},
2282 { 0x30a1, 0x30fa, 1},
2283 { 0x30fc, 0x30ff, 1},
2284 { 0x3105, 0x312d, 1},
2285 { 0x3131, 0x318e, 1},
2286 { 0x31a0, 0x31ba, 1},
2287 { 0x31f0, 0x31ff, 1},
2288 { 0x3400, 0x4db5, 1},
2289 { 0x4e00, 0x9fd5, 1},
2290 { 0xa000, 0xa48c, 1},
2291 { 0xa4d0, 0xa4fd, 1},
2292 { 0xa500, 0xa60c, 1},
2293 { 0xa610, 0xa61f, 1},
2294 { 0xa62a, 0xa62b, 1},
2295 { 0xa640, 0xa66e, 1},
2296 { 0xa67f, 0xa69d, 1},
2297 { 0xa6a0, 0xa6e5, 1},
2298 { 0xa717, 0xa71f, 1},
2299 { 0xa722, 0xa788, 1},
2300 { 0xa78b, 0xa7ad, 1},
2301 { 0xa7b0, 0xa7b7, 1},
2302 { 0xa7f7, 0xa801, 1},
2303 { 0xa803, 0xa805, 1},
2304 { 0xa807, 0xa80a, 1},
2305 { 0xa80c, 0xa822, 1},
2306 { 0xa840, 0xa873, 1},
2307 { 0xa882, 0xa8b3, 1},
2308 { 0xa8f2, 0xa8f7, 1},
2309 { 0xa8fb, 0xa8fd, 2},
2310 { 0xa90a, 0xa925, 1},
2311 { 0xa930, 0xa946, 1},
2312 { 0xa960, 0xa97c, 1},
2313 { 0xa984, 0xa9b2, 1},
2314 { 0xa9cf, 0xa9e0, 17},
2315 { 0xa9e1, 0xa9e4, 1},
2316 { 0xa9e6, 0xa9ef, 1},
2317 { 0xa9fa, 0xa9fe, 1},
2318 { 0xaa00, 0xaa28, 1},
2319 { 0xaa40, 0xaa42, 1},
2320 { 0xaa44, 0xaa4b, 1},
2321 { 0xaa60, 0xaa76, 1},
2322 { 0xaa7a, 0xaa7e, 4},
2323 { 0xaa7f, 0xaaaf, 1},
2324 { 0xaab1, 0xaab5, 4},
2325 { 0xaab6, 0xaab9, 3},
2326 { 0xaaba, 0xaabd, 1},
2327 { 0xaac0, 0xaac2, 2},
2328 { 0xaadb, 0xaadd, 1},
2329 { 0xaae0, 0xaaea, 1},
2330 { 0xaaf2, 0xaaf4, 1},
2331 { 0xab01, 0xab06, 1},
2332 { 0xab09, 0xab0e, 1},
2333 { 0xab11, 0xab16, 1},
2334 { 0xab20, 0xab26, 1},
2335 { 0xab28, 0xab2e, 1},
2336 { 0xab30, 0xab5a, 1},
2337 { 0xab5c, 0xab65, 1},
2338 { 0xab70, 0xabe2, 1},
2339 { 0xac00, 0xd7a3, 1},
2340 { 0xd7b0, 0xd7c6, 1},
2341 { 0xd7cb, 0xd7fb, 1},
2342 { 0xf900, 0xfa6d, 1},
2343 { 0xfa70, 0xfad9, 1},
2344 { 0xfb00, 0xfb06, 1},
2345 { 0xfb13, 0xfb17, 1},
2346 { 0xfb1d, 0xfb1f, 2},
2347 { 0xfb20, 0xfb28, 1},
2348 { 0xfb2a, 0xfb36, 1},
2349 { 0xfb38, 0xfb3c, 1},
2350 { 0xfb3e, 0xfb40, 2},
2351 { 0xfb41, 0xfb43, 2},
2352 { 0xfb44, 0xfb46, 2},
2353 { 0xfb47, 0xfbb1, 1},
2354 { 0xfbd3, 0xfd3d, 1},
2355 { 0xfd50, 0xfd8f, 1},
2356 { 0xfd92, 0xfdc7, 1},
2357 { 0xfdf0, 0xfdfb, 1},
2358 { 0xfe70, 0xfe74, 1},
2359 { 0xfe76, 0xfefc, 1},
2360 { 0xff21, 0xff3a, 1},
2361 { 0xff41, 0xff5a, 1},
2362 { 0xff66, 0xffbe, 1},
2363 { 0xffc2, 0xffc7, 1},
2364 { 0xffca, 0xffcf, 1},
2365 { 0xffd2, 0xffd7, 1},
2366 { 0xffda, 0xffdc, 1},
2367 { 0x10000, 0x1000b, 1},
2368 { 0x1000d, 0x10026, 1},
2369 { 0x10028, 0x1003a, 1},
2370 { 0x1003c, 0x1003d, 1},
2371 { 0x1003f, 0x1004d, 1},
2372 { 0x10050, 0x1005d, 1},
2373 { 0x10080, 0x100fa, 1},
2374 { 0x10280, 0x1029c, 1},
2375 { 0x102a0, 0x102d0, 1},
2376 { 0x10300, 0x1031f, 1},
2377 { 0x10330, 0x10340, 1},
2378 { 0x10342, 0x10349, 1},
2379 { 0x10350, 0x10375, 1},
2380 { 0x10380, 0x1039d, 1},
2381 { 0x103a0, 0x103c3, 1},
2382 { 0x103c8, 0x103cf, 1},
2383 { 0x10400, 0x1049d, 1},
2384 { 0x10500, 0x10527, 1},
2385 { 0x10530, 0x10563, 1},
2386 { 0x10600, 0x10736, 1},
2387 { 0x10740, 0x10755, 1},
2388 { 0x10760, 0x10767, 1},
2389 { 0x10800, 0x10805, 1},
2390 { 0x10808, 0x1080a, 2},
2391 { 0x1080b, 0x10835, 1},
2392 { 0x10837, 0x10838, 1},
2393 { 0x1083c, 0x1083f, 3},
2394 { 0x10840, 0x10855, 1},
2395 { 0x10860, 0x10876, 1},
2396 { 0x10880, 0x1089e, 1},
2397 { 0x108e0, 0x108f2, 1},
2398 { 0x108f4, 0x108f5, 1},
2399 { 0x10900, 0x10915, 1},
2400 { 0x10920, 0x10939, 1},
2401 { 0x10980, 0x109b7, 1},
2402 { 0x109be, 0x109bf, 1},
2403 { 0x10a00, 0x10a10, 16},
2404 { 0x10a11, 0x10a13, 1},
2405 { 0x10a15, 0x10a17, 1},
2406 { 0x10a19, 0x10a33, 1},
2407 { 0x10a60, 0x10a7c, 1},
2408 { 0x10a80, 0x10a9c, 1},
2409 { 0x10ac0, 0x10ac7, 1},
2410 { 0x10ac9, 0x10ae4, 1},
2411 { 0x10b00, 0x10b35, 1},
2412 { 0x10b40, 0x10b55, 1},
2413 { 0x10b60, 0x10b72, 1},
2414 { 0x10b80, 0x10b91, 1},
2415 { 0x10c00, 0x10c48, 1},
2416 { 0x10c80, 0x10cb2, 1},
2417 { 0x10cc0, 0x10cf2, 1},
2418 { 0x11003, 0x11037, 1},
2419 { 0x11083, 0x110af, 1},
2420 { 0x110d0, 0x110e8, 1},
2421 { 0x11103, 0x11126, 1},
2422 { 0x11150, 0x11172, 1},
2423 { 0x11176, 0x11183, 13},
2424 { 0x11184, 0x111b2, 1},
2425 { 0x111c1, 0x111c4, 1},
2426 { 0x111da, 0x111dc, 2},
2427 { 0x11200, 0x11211, 1},
2428 { 0x11213, 0x1122b, 1},
2429 { 0x11280, 0x11286, 1},
2430 { 0x11288, 0x1128a, 2},
2431 { 0x1128b, 0x1128d, 1},
2432 { 0x1128f, 0x1129d, 1},
2433 { 0x1129f, 0x112a8, 1},
2434 { 0x112b0, 0x112de, 1},
2435 { 0x11305, 0x1130c, 1},
2436 { 0x1130f, 0x11310, 1},
2437 { 0x11313, 0x11328, 1},
2438 { 0x1132a, 0x11330, 1},
2439 { 0x11332, 0x11333, 1},
2440 { 0x11335, 0x11339, 1},
2441 { 0x1133d, 0x11350, 19},
2442 { 0x1135d, 0x11361, 1},
2443 { 0x11480, 0x114af, 1},
2444 { 0x114c4, 0x114c5, 1},
2445 { 0x114c7, 0x11580, 185},
2446 { 0x11581, 0x115ae, 1},
2447 { 0x115d8, 0x115db, 1},
2448 { 0x11600, 0x1162f, 1},
2449 { 0x11644, 0x11680, 60},
2450 { 0x11681, 0x116aa, 1},
2451 { 0x11700, 0x11719, 1},
2452 { 0x118a0, 0x118df, 1},
2453 { 0x118ff, 0x11ac0, 449},
2454 { 0x11ac1, 0x11af8, 1},
2455 { 0x12000, 0x12399, 1},
2456 { 0x12480, 0x12543, 1},
2457 { 0x13000, 0x1342e, 1},
2458 { 0x14400, 0x14646, 1},
2459 { 0x16800, 0x16a38, 1},
2460 { 0x16a40, 0x16a5e, 1},
2461 { 0x16ad0, 0x16aed, 1},
2462 { 0x16b00, 0x16b2f, 1},
2463 { 0x16b40, 0x16b43, 1},
2464 { 0x16b63, 0x16b77, 1},
2465 { 0x16b7d, 0x16b8f, 1},
2466 { 0x16f00, 0x16f44, 1},
2467 { 0x16f50, 0x16f93, 67},
2468 { 0x16f94, 0x16f9f, 1},
2469 { 0x1b000, 0x1b001, 1},
2470 { 0x1bc00, 0x1bc6a, 1},
2471 { 0x1bc70, 0x1bc7c, 1},
2472 { 0x1bc80, 0x1bc88, 1},
2473 { 0x1bc90, 0x1bc99, 1},
2474 { 0x1d400, 0x1d454, 1},
2475 { 0x1d456, 0x1d49c, 1},
2476 { 0x1d49e, 0x1d49f, 1},
2477 { 0x1d4a2, 0x1d4a5, 3},
2478 { 0x1d4a6, 0x1d4a9, 3},
2479 { 0x1d4aa, 0x1d4ac, 1},
2480 { 0x1d4ae, 0x1d4b9, 1},
2481 { 0x1d4bb, 0x1d4bd, 2},
2482 { 0x1d4be, 0x1d4c3, 1},
2483 { 0x1d4c5, 0x1d505, 1},
2484 { 0x1d507, 0x1d50a, 1},
2485 { 0x1d50d, 0x1d514, 1},
2486 { 0x1d516, 0x1d51c, 1},
2487 { 0x1d51e, 0x1d539, 1},
2488 { 0x1d53b, 0x1d53e, 1},
2489 { 0x1d540, 0x1d544, 1},
2490 { 0x1d546, 0x1d54a, 4},
2491 { 0x1d54b, 0x1d550, 1},
2492 { 0x1d552, 0x1d6a5, 1},
2493 { 0x1d6a8, 0x1d6c0, 1},
2494 { 0x1d6c2, 0x1d6da, 1},
2495 { 0x1d6dc, 0x1d6fa, 1},
2496 { 0x1d6fc, 0x1d714, 1},
2497 { 0x1d716, 0x1d734, 1},
2498 { 0x1d736, 0x1d74e, 1},
2499 { 0x1d750, 0x1d76e, 1},
2500 { 0x1d770, 0x1d788, 1},
2501 { 0x1d78a, 0x1d7a8, 1},
2502 { 0x1d7aa, 0x1d7c2, 1},
2503 { 0x1d7c4, 0x1d7cb, 1},
2504 { 0x1e800, 0x1e8c4, 1},
2505 { 0x1ee00, 0x1ee03, 1},
2506 { 0x1ee05, 0x1ee1f, 1},
2507 { 0x1ee21, 0x1ee22, 1},
2508 { 0x1ee24, 0x1ee27, 3},
2509 { 0x1ee29, 0x1ee32, 1},
2510 { 0x1ee34, 0x1ee37, 1},
2511 { 0x1ee39, 0x1ee3b, 2},
2512 { 0x1ee42, 0x1ee47, 5},
2513 { 0x1ee49, 0x1ee4d, 2},
2514 { 0x1ee4e, 0x1ee4f, 1},
2515 { 0x1ee51, 0x1ee52, 1},
2516 { 0x1ee54, 0x1ee57, 3},
2517 { 0x1ee59, 0x1ee61, 2},
2518 { 0x1ee62, 0x1ee64, 2},
2519 { 0x1ee67, 0x1ee6a, 1},
2520 { 0x1ee6c, 0x1ee72, 1},
2521 { 0x1ee74, 0x1ee77, 1},
2522 { 0x1ee79, 0x1ee7c, 1},
2523 { 0x1ee7e, 0x1ee80, 2},
2524 { 0x1ee81, 0x1ee89, 1},
2525 { 0x1ee8b, 0x1ee9b, 1},
2526 { 0x1eea1, 0x1eea3, 1},
2527 { 0x1eea5, 0x1eea9, 1},
2528 { 0x1eeab, 0x1eebb, 1},
2529 { 0x20000, 0x2a6d6, 1},
2530 { 0x2a700, 0x2b734, 1},
2531 { 0x2b740, 0x2b81d, 1},
2532 { 0x2b820, 0x2cea1, 1},
2533 { 0x2f800, 0x2fa1d, 1},
2536 // A table of Unicode uppercase letters--Unicode code points
2537 // classified as "Letter, uppercase".
2539 static const Unicode_range unicode_uppercase_letters
[] =
2541 { 0x0041, 0x005a, 1},
2542 { 0x00c0, 0x00d6, 1},
2543 { 0x00d8, 0x00de, 1},
2544 { 0x0100, 0x0136, 2},
2545 { 0x0139, 0x0147, 2},
2546 { 0x014a, 0x0178, 2},
2547 { 0x0179, 0x017d, 2},
2548 { 0x0181, 0x0182, 1},
2549 { 0x0184, 0x0186, 2},
2550 { 0x0187, 0x0189, 2},
2551 { 0x018a, 0x018b, 1},
2552 { 0x018e, 0x0191, 1},
2553 { 0x0193, 0x0194, 1},
2554 { 0x0196, 0x0198, 1},
2555 { 0x019c, 0x019d, 1},
2556 { 0x019f, 0x01a0, 1},
2557 { 0x01a2, 0x01a6, 2},
2558 { 0x01a7, 0x01a9, 2},
2559 { 0x01ac, 0x01ae, 2},
2560 { 0x01af, 0x01b1, 2},
2561 { 0x01b2, 0x01b3, 1},
2562 { 0x01b5, 0x01b7, 2},
2563 { 0x01b8, 0x01bc, 4},
2564 { 0x01c4, 0x01cd, 3},
2565 { 0x01cf, 0x01db, 2},
2566 { 0x01de, 0x01ee, 2},
2567 { 0x01f1, 0x01f4, 3},
2568 { 0x01f6, 0x01f8, 1},
2569 { 0x01fa, 0x0232, 2},
2570 { 0x023a, 0x023b, 1},
2571 { 0x023d, 0x023e, 1},
2572 { 0x0241, 0x0243, 2},
2573 { 0x0244, 0x0246, 1},
2574 { 0x0248, 0x024e, 2},
2575 { 0x0370, 0x0372, 2},
2576 { 0x0376, 0x037f, 9},
2577 { 0x0386, 0x0388, 2},
2578 { 0x0389, 0x038a, 1},
2579 { 0x038c, 0x038e, 2},
2580 { 0x038f, 0x0391, 2},
2581 { 0x0392, 0x03a1, 1},
2582 { 0x03a3, 0x03ab, 1},
2583 { 0x03cf, 0x03d2, 3},
2584 { 0x03d3, 0x03d4, 1},
2585 { 0x03d8, 0x03ee, 2},
2586 { 0x03f4, 0x03f7, 3},
2587 { 0x03f9, 0x03fa, 1},
2588 { 0x03fd, 0x042f, 1},
2589 { 0x0460, 0x0480, 2},
2590 { 0x048a, 0x04c0, 2},
2591 { 0x04c1, 0x04cd, 2},
2592 { 0x04d0, 0x052e, 2},
2593 { 0x0531, 0x0556, 1},
2594 { 0x10a0, 0x10c5, 1},
2595 { 0x10c7, 0x10cd, 6},
2596 { 0x1e00, 0x1e94, 2},
2597 { 0x1e9e, 0x1efe, 2},
2598 { 0x1f08, 0x1f0f, 1},
2599 { 0x1f18, 0x1f1d, 1},
2600 { 0x1f28, 0x1f2f, 1},
2601 { 0x1f38, 0x1f3f, 1},
2602 { 0x1f48, 0x1f4d, 1},
2603 { 0x1f59, 0x1f5f, 2},
2604 { 0x1f68, 0x1f6f, 1},
2605 { 0x1fb8, 0x1fbb, 1},
2606 { 0x1fc8, 0x1fcb, 1},
2607 { 0x1fd8, 0x1fdb, 1},
2608 { 0x1fe8, 0x1fec, 1},
2609 { 0x1ff8, 0x1ffb, 1},
2610 { 0x2102, 0x2107, 5},
2611 { 0x210b, 0x210d, 1},
2612 { 0x2110, 0x2112, 1},
2613 { 0x2115, 0x2119, 4},
2614 { 0x211a, 0x211d, 1},
2615 { 0x2124, 0x212a, 2},
2616 { 0x212b, 0x212d, 1},
2617 { 0x2130, 0x2133, 1},
2618 { 0x213e, 0x213f, 1},
2619 { 0x2145, 0x2183, 62},
2620 { 0x2c00, 0x2c2e, 1},
2621 { 0x2c60, 0x2c62, 2},
2622 { 0x2c63, 0x2c64, 1},
2623 { 0x2c67, 0x2c6d, 2},
2624 { 0x2c6e, 0x2c70, 1},
2625 { 0x2c72, 0x2c75, 3},
2626 { 0x2c7e, 0x2c80, 1},
2627 { 0x2c82, 0x2ce2, 2},
2628 { 0x2ceb, 0x2ced, 2},
2629 { 0x2cf2, 0xa640, 31054},
2630 { 0xa642, 0xa66c, 2},
2631 { 0xa680, 0xa69a, 2},
2632 { 0xa722, 0xa72e, 2},
2633 { 0xa732, 0xa76e, 2},
2634 { 0xa779, 0xa77d, 2},
2635 { 0xa77e, 0xa786, 2},
2636 { 0xa78b, 0xa78d, 2},
2637 { 0xa790, 0xa792, 2},
2638 { 0xa796, 0xa7aa, 2},
2639 { 0xa7ab, 0xa7ad, 1},
2640 { 0xa7b0, 0xa7b1, 1},
2641 { 0xff21, 0xff3a, 1},
2642 { 0x10400, 0x10427, 1},
2643 { 0x118a0, 0x118bf, 1},
2644 { 0x1d400, 0x1d419, 1},
2645 { 0x1d434, 0x1d44d, 1},
2646 { 0x1d468, 0x1d481, 1},
2647 { 0x1d49c, 0x1d49e, 2},
2648 { 0x1d49f, 0x1d4a5, 3},
2649 { 0x1d4a6, 0x1d4a9, 3},
2650 { 0x1d4aa, 0x1d4ac, 1},
2651 { 0x1d4ae, 0x1d4b5, 1},
2652 { 0x1d4d0, 0x1d4e9, 1},
2653 { 0x1d504, 0x1d505, 1},
2654 { 0x1d507, 0x1d50a, 1},
2655 { 0x1d50d, 0x1d514, 1},
2656 { 0x1d516, 0x1d51c, 1},
2657 { 0x1d538, 0x1d539, 1},
2658 { 0x1d53b, 0x1d53e, 1},
2659 { 0x1d540, 0x1d544, 1},
2660 { 0x1d546, 0x1d54a, 4},
2661 { 0x1d54b, 0x1d550, 1},
2662 { 0x1d56c, 0x1d585, 1},
2663 { 0x1d5a0, 0x1d5b9, 1},
2664 { 0x1d5d4, 0x1d5ed, 1},
2665 { 0x1d608, 0x1d621, 1},
2666 { 0x1d63c, 0x1d655, 1},
2667 { 0x1d670, 0x1d689, 1},
2668 { 0x1d6a8, 0x1d6c0, 1},
2669 { 0x1d6e2, 0x1d6fa, 1},
2670 { 0x1d71c, 0x1d734, 1},
2671 { 0x1d756, 0x1d76e, 1},
2672 { 0x1d790, 0x1d7a8, 1},
2673 { 0x1d7ca, 0x1d7ca, 1},
2676 // Return true if C is in RANGES.
2679 Lex::is_in_unicode_range(unsigned int c
, const Unicode_range
* ranges
,
2684 // The common case is a small value, and we know that it will be
2685 // in the first few entries of the table. Do a linear scan
2686 // rather than a binary search.
2687 for (size_t i
= 0; i
< range_size
; ++i
)
2689 const Unicode_range
* p
= &ranges
[i
];
2694 return (c
- p
->low
) % p
->stride
== 0;
2702 size_t hi
= range_size
;
2705 size_t mid
= lo
+ (hi
- lo
) / 2;
2706 const Unicode_range
* p
= &ranges
[mid
];
2709 else if (c
> p
->high
)
2712 return (c
- p
->low
) % p
->stride
== 0;
2718 // Return whether C is a space character.
2721 Lex::is_unicode_space(unsigned int c
)
2723 return Lex::is_in_unicode_range(c
, unicode_space
,
2724 ARRAY_SIZE(unicode_space
));
2727 // Return whether C is a Unicode digit--a Unicode code point
2728 // classified as "Digit".
2731 Lex::is_unicode_digit(unsigned int c
)
2733 return Lex::is_in_unicode_range(c
, unicode_digits
,
2734 ARRAY_SIZE(unicode_digits
));
2737 // Return whether C is a Unicode letter--a Unicode code point
2738 // classified as "Letter".
2741 Lex::is_unicode_letter(unsigned int c
)
2743 return Lex::is_in_unicode_range(c
, unicode_letters
,
2744 ARRAY_SIZE(unicode_letters
));
2747 // Return whether C is a Unicode uppercase letter. a Unicode code
2748 // point classified as "Letter, uppercase".
2751 Lex::is_unicode_uppercase(unsigned int c
)
2753 return Lex::is_in_unicode_range(c
, unicode_uppercase_letters
,
2754 ARRAY_SIZE(unicode_uppercase_letters
));
2757 // Return whether the identifier NAME should be exported. NAME is a
2758 // mangled name which includes only ASCII characters.
2761 Lex::is_exported_name(const std::string
& name
)
2763 unsigned char c
= name
[0];
2765 return c
>= 'A' && c
<= 'Z';
2768 const char* p
= name
.data();
2769 size_t len
= name
.length();
2770 if (len
< 2 || p
[1] != 'U')
2772 unsigned int ci
= 0;
2773 for (size_t i
= 2; i
< len
&& p
[i
] != '$'; ++i
)
2776 if (!Lex::is_hex_digit(c
))
2779 ci
|= Lex::hex_val(c
);
2781 return Lex::is_unicode_uppercase(ci
);
2785 // Return whether the identifier NAME contains an invalid character.
2786 // This is based on how we handle invalid characters in
2787 // gather_identifier.
2790 Lex::is_invalid_identifier(const std::string
& name
)
2792 return name
.find("$INVALID$") != std::string::npos
;