gcc/go/gofrontend/lex.h

   1 // lex.h -- Go frontend lexer.     -*- C++ -*-
   2
   3 // Copyright 2009 The Go Authors. All rights reserved.
   4 // Use of this source code is governed by a BSD-style
   5 // license that can be found in the LICENSE file.
   6
   7 #ifndef GO_LEX_H
   8 #define GO_LEX_H
   9
  10 #include <mpfr.h>
  11
  12 #include "operator.h"
  13 #include "go-linemap.h"
  14
  15 struct Unicode_range;
  16
  17 // The keywords.  These must be in sorted order, other than
  18 // KEYWORD_INVALID.  They must match the Keywords::mapping_ array in
  19 // lex.cc.
  20
  21 enum Keyword
  22 {
  23   KEYWORD_INVALID,      // Not a keyword.
  24   KEYWORD_ASM,
  25   KEYWORD_BREAK,
  26   KEYWORD_CASE,
  27   KEYWORD_CHAN,
  28   KEYWORD_CONST,
  29   KEYWORD_CONTINUE,
  30   KEYWORD_DEFAULT,
  31   KEYWORD_DEFER,
  32   KEYWORD_ELSE,
  33   KEYWORD_FALLTHROUGH,
  34   KEYWORD_FOR,
  35   KEYWORD_FUNC,
  36   KEYWORD_GO,
  37   KEYWORD_GOTO,
  38   KEYWORD_IF,
  39   KEYWORD_IMPORT,
  40   KEYWORD_INTERFACE,
  41   KEYWORD_MAP,
  42   KEYWORD_PACKAGE,
  43   KEYWORD_RANGE,
  44   KEYWORD_RETURN,
  45   KEYWORD_SELECT,
  46   KEYWORD_STRUCT,
  47   KEYWORD_SWITCH,
  48   KEYWORD_TYPE,
  49   KEYWORD_VAR
  50 };
  51
  52 // Pragmas built from magic comments and recorded for functions.
  53 // These are used as bits in a bitmask.
  54 // The set of values is intended to be the same as the gc compiler.
  55
  56 enum GoPragma
  57 {
  58   GOPRAGMA_NOINTERFACE = 1 << 0,        // Method not in type descriptor.
  59   GOPRAGMA_NOESCAPE = 1 << 1,           // Args do not escape.
  60   GOPRAGMA_NORACE = 1 << 2,             // No race detector.
  61   GOPRAGMA_NOSPLIT = 1 << 3,            // Do not split stack.
  62   GOPRAGMA_NOINLINE = 1 << 4,           // Do not inline.
  63   GOPRAGMA_SYSTEMSTACK = 1 << 5,        // Must run on system stack.
  64   GOPRAGMA_NOWRITEBARRIER = 1 << 6,     // No write barriers.
  65   GOPRAGMA_NOWRITEBARRIERREC = 1 << 7,  // No write barriers here or callees.
  66   GOPRAGMA_CGOUNSAFEARGS = 1 << 8,      // Pointer to arg is pointer to all.
  67   GOPRAGMA_UINTPTRESCAPES = 1 << 9,     // uintptr(p) escapes.
  68   GOPRAGMA_NOTINHEAP = 1 << 10          // type is not in heap.
  69 };
  70
  71 // A token returned from the lexer.
  72
  73 class Token
  74 {
  75  public:
  76   // Token classification.
  77   enum Classification
  78   {
  79     // Token is invalid.
  80     TOKEN_INVALID,
  81     // Token indicates end of input.
  82     TOKEN_EOF,
  83     // Token is a keyword.
  84     TOKEN_KEYWORD,
  85     // Token is an identifier.
  86     TOKEN_IDENTIFIER,
  87     // Token is a string of characters.
  88     TOKEN_STRING,
  89     // Token is an operator.
  90     TOKEN_OPERATOR,
  91     // Token is a character constant.
  92     TOKEN_CHARACTER,
  93     // Token is an integer.
  94     TOKEN_INTEGER,
  95     // Token is a floating point number.
  96     TOKEN_FLOAT,
  97     // Token is an imaginary number.
  98     TOKEN_IMAGINARY
  99   };
 100
 101   ~Token();
 102   Token(const Token&);
 103   Token& operator=(const Token&);
 104
 105   // Get token classification.
 106   Classification
 107   classification() const
 108   { return this->classification_; }
 109
 110   // Make a token for an invalid value.
 111   static Token
 112   make_invalid_token(Location location)
 113   { return Token(TOKEN_INVALID, location); }
 114
 115   // Make a token representing end of file.
 116   static Token
 117   make_eof_token(Location location)
 118   { return Token(TOKEN_EOF, location); }
 119
 120   // Make a keyword token.
 121   static Token
 122   make_keyword_token(Keyword keyword, Location location)
 123   {
 124     Token tok(TOKEN_KEYWORD, location);
 125     tok.u_.keyword = keyword;
 126     return tok;
 127   }
 128
 129   // Make an identifier token.
 130   static Token
 131   make_identifier_token(const std::string& value, bool is_exported,
 132                         Location location)
 133   {
 134     Token tok(TOKEN_IDENTIFIER, location);
 135     tok.u_.identifier_value.name = new std::string(value);
 136     tok.u_.identifier_value.is_exported = is_exported;
 137     return tok;
 138   }
 139
 140   // Make a quoted string token.
 141   static Token
 142   make_string_token(const std::string& value, Location location)
 143   {
 144     Token tok(TOKEN_STRING, location);
 145     tok.u_.string_value = new std::string(value);
 146     return tok;
 147   }
 148
 149   // Make an operator token.
 150   static Token
 151   make_operator_token(Operator op, Location location)
 152   {
 153     Token tok(TOKEN_OPERATOR, location);
 154     tok.u_.op = op;
 155     return tok;
 156   }
 157
 158   // Make a character constant token.
 159   static Token
 160   make_character_token(mpz_t val, Location location)
 161   {
 162     Token tok(TOKEN_CHARACTER, location);
 163     mpz_init(tok.u_.integer_value);
 164     mpz_swap(tok.u_.integer_value, val);
 165     return tok;
 166   }
 167
 168   // Make an integer token.
 169   static Token
 170   make_integer_token(mpz_t val, Location location)
 171   {
 172     Token tok(TOKEN_INTEGER, location);
 173     mpz_init(tok.u_.integer_value);
 174     mpz_swap(tok.u_.integer_value, val);
 175     return tok;
 176   }
 177
 178   // Make a float token.
 179   static Token
 180   make_float_token(mpfr_t val, Location location)
 181   {
 182     Token tok(TOKEN_FLOAT, location);
 183     mpfr_init(tok.u_.float_value);
 184     mpfr_swap(tok.u_.float_value, val);
 185     return tok;
 186   }
 187
 188   // Make a token for an imaginary number.
 189   static Token
 190   make_imaginary_token(mpfr_t val, Location location)
 191   {
 192     Token tok(TOKEN_IMAGINARY, location);
 193     mpfr_init(tok.u_.float_value);
 194     mpfr_swap(tok.u_.float_value, val);
 195     return tok;
 196   }
 197
 198   // Get the location of the token.
 199   Location
 200   location() const
 201   { return this->location_; }
 202
 203   // Return whether this is an invalid token.
 204   bool
 205   is_invalid() const
 206   { return this->classification_ == TOKEN_INVALID; }
 207
 208   // Return whether this is the EOF token.
 209   bool
 210   is_eof() const
 211   { return this->classification_ == TOKEN_EOF; }
 212
 213   // Return the keyword value for a keyword token.
 214   Keyword
 215   keyword() const
 216   {
 217     go_assert(this->classification_ == TOKEN_KEYWORD);
 218     return this->u_.keyword;
 219   }
 220
 221   // Return whether this is an identifier.
 222   bool
 223   is_identifier() const
 224   { return this->classification_ == TOKEN_IDENTIFIER; }
 225
 226   // Return the identifier.
 227   const std::string&
 228   identifier() const
 229   {
 230     go_assert(this->classification_ == TOKEN_IDENTIFIER);
 231     return *this->u_.identifier_value.name;
 232   }
 233
 234   // Return whether the identifier is exported.
 235   bool
 236   is_identifier_exported() const
 237   {
 238     go_assert(this->classification_ == TOKEN_IDENTIFIER);
 239     return this->u_.identifier_value.is_exported;
 240   }
 241
 242   // Return whether this is a string.
 243   bool
 244   is_string() const
 245   {
 246     return this->classification_ == TOKEN_STRING;
 247   }
 248
 249   // Return the value of a string.  The returned value is a string of
 250   // UTF-8 characters.
 251   std::string
 252   string_value() const
 253   {
 254     go_assert(this->classification_ == TOKEN_STRING);
 255     return *this->u_.string_value;
 256   }
 257
 258   // Return the value of a character constant.
 259   const mpz_t*
 260   character_value() const
 261   {
 262     go_assert(this->classification_ == TOKEN_CHARACTER);
 263     return &this->u_.integer_value;
 264   }
 265
 266   // Return the value of an integer.
 267   const mpz_t*
 268   integer_value() const
 269   {
 270     go_assert(this->classification_ == TOKEN_INTEGER);
 271     return &this->u_.integer_value;
 272   }
 273
 274   // Return the value of a float.
 275   const mpfr_t*
 276   float_value() const
 277   {
 278     go_assert(this->classification_ == TOKEN_FLOAT);
 279     return &this->u_.float_value;
 280   }
 281
 282   // Return the value of an imaginary number.
 283   const mpfr_t*
 284   imaginary_value() const
 285   {
 286     go_assert(this->classification_ == TOKEN_IMAGINARY);
 287     return &this->u_.float_value;
 288   }
 289
 290   // Return the operator value for an operator token.
 291   Operator
 292   op() const
 293   {
 294     go_assert(this->classification_ == TOKEN_OPERATOR);
 295     return this->u_.op;
 296   }
 297
 298   // Return whether this token is KEYWORD.
 299   bool
 300   is_keyword(Keyword keyword) const
 301   {
 302     return (this->classification_ == TOKEN_KEYWORD
 303             && this->u_.keyword == keyword);
 304   }
 305
 306   // Return whether this token is OP.
 307   bool
 308   is_op(Operator op) const
 309   { return this->classification_ == TOKEN_OPERATOR && this->u_.op == op; }
 310
 311   // Print the token for debugging.
 312   void
 313   print(FILE*) const;
 314
 315  private:
 316   // Private constructor used by make_..._token functions above.
 317   Token(Classification, Location);
 318
 319   // Clear the token.
 320   void
 321   clear();
 322
 323   // The token classification.
 324   Classification classification_;
 325   union
 326   {
 327     // The keyword value for TOKEN_KEYWORD.
 328     Keyword keyword;
 329     // The token value for TOKEN_IDENTIFIER.
 330     struct
 331     {
 332       // The name of the identifier.  This has been mangled to only
 333       // include ASCII characters.
 334       std::string* name;
 335       // Whether this name should be exported.  This is true if the
 336       // first letter in the name is upper case.
 337       bool is_exported;
 338     } identifier_value;
 339     // The string value for TOKEN_STRING.
 340     std::string* string_value;
 341     // The token value for TOKEN_CHARACTER or TOKEN_INTEGER.
 342     mpz_t integer_value;
 343     // The token value for TOKEN_FLOAT or TOKEN_IMAGINARY.
 344     mpfr_t float_value;
 345     // The token value for TOKEN_OPERATOR or the keyword value
 346     Operator op;
 347   } u_;
 348   // The source location.
 349   Location location_;
 350 };
 351
 352 // The lexer itself.
 353
 354 class Lex
 355 {
 356  public:
 357   Lex(const char* input_file_name, FILE* input_file, Linemap *linemap);
 358
 359   ~Lex();
 360
 361   // Return the next token.
 362   Token
 363   next_token();
 364
 365   // Return the contents of any current //extern comment.
 366   const std::string&
 367   extern_name() const
 368   { return this->extern_; }
 369
 370   // Return the current set of pragmas, and clear them.
 371   unsigned int
 372   get_and_clear_pragmas()
 373   {
 374     unsigned int ret = this->pragmas_;
 375     this->pragmas_ = 0;
 376     return ret;
 377   }
 378
 379   struct Linkname
 380   {
 381     std::string ext_name;       // External name.
 382     bool is_exported;           // Whether the internal name is exported.
 383     Location loc;               // Location of go:linkname directive.
 384
 385     Linkname()
 386       : ext_name(), is_exported(false), loc()
 387     { }
 388
 389     Linkname(const std::string& ext_name_a, bool is_exported_a, Location loc_a)
 390       : ext_name(ext_name_a), is_exported(is_exported_a), loc(loc_a)
 391     { }
 392   };
 393
 394   typedef std::map<std::string, Linkname> Linknames;
 395
 396   // Return the linknames seen so far, or NULL if none, and clear the
 397   // set.  These are from go:linkname compiler directives.
 398   Linknames*
 399   get_and_clear_linknames()
 400   {
 401     Linknames* ret = this->linknames_;
 402     this->linknames_ = NULL;
 403     return ret;
 404   }
 405
 406   // Return whether the identifier NAME should be exported.  NAME is a
 407   // mangled name which includes only ASCII characters.
 408   static bool
 409   is_exported_name(const std::string& name);
 410
 411   // Return whether the identifier NAME is invalid.  When we see an
 412   // invalid character we still build an identifier, but we use a
 413   // magic string to indicate that the identifier is invalid.  We then
 414   // use this to avoid knockon errors.
 415   static bool
 416   is_invalid_identifier(const std::string& name);
 417
 418   // A helper function.  Append V to STR.  IS_CHARACTER is true if V
 419   // is a Unicode character which should be converted into UTF-8,
 420   // false if it is a byte value to be appended directly.  The
 421   // location is used to warn about an out of range character.
 422   static void
 423   append_char(unsigned int v, bool is_charater, std::string* str,
 424               Location);
 425
 426   // A helper function.  Fetch a UTF-8 character from STR and store it
 427   // in *VALUE.  Return the number of bytes read from STR.  Return 0
 428   // if STR does not point to a valid UTF-8 character.
 429   static int
 430   fetch_char(const char* str, unsigned int *value);
 431
 432   // Return whether C is a Unicode or "C" locale space character.
 433   static bool
 434   is_unicode_space(unsigned int c);
 435
 436  private:
 437   ssize_t
 438   get_line();
 439
 440   bool
 441   require_line();
 442
 443   // The current location.
 444   Location
 445   location() const;
 446
 447   // A position CHARS column positions before the current location.
 448   Location
 449   earlier_location(int chars) const;
 450
 451   static bool
 452   is_hex_digit(char);
 453
 454   static unsigned char
 455   octal_value(char c)
 456   { return c - '0'; }
 457
 458   static unsigned
 459   hex_val(char c);
 460
 461   Token
 462   make_invalid_token()
 463   { return Token::make_invalid_token(this->location()); }
 464
 465   Token
 466   make_eof_token()
 467   { return Token::make_eof_token(this->location()); }
 468
 469   Token
 470   make_operator(Operator op, int chars)
 471   { return Token::make_operator_token(op, this->earlier_location(chars)); }
 472
 473   Token
 474   gather_identifier();
 475
 476   static bool
 477   could_be_exponent(const char*, const char*);
 478
 479   Token
 480   gather_number();
 481
 482   Token
 483   gather_character();
 484
 485   Token
 486   gather_string();
 487
 488   Token
 489   gather_raw_string();
 490
 491   const char*
 492   advance_one_utf8_char(const char*, unsigned int*, bool*);
 493
 494   const char*
 495   advance_one_char(const char*, bool, unsigned int*, bool*);
 496
 497   static bool
 498   is_unicode_digit(unsigned int c);
 499
 500   static bool
 501   is_unicode_letter(unsigned int c);
 502
 503   static bool
 504   is_unicode_uppercase(unsigned int c);
 505
 506   static bool
 507   is_in_unicode_range(unsigned int C, const Unicode_range* ranges,
 508                       size_t range_size);
 509
 510   Operator
 511   three_character_operator(char, char, char);
 512
 513   Operator
 514   two_character_operator(char, char);
 515
 516   Operator
 517   one_character_operator(char);
 518
 519   bool
 520   skip_c_comment(bool* found_newline);
 521
 522   void
 523   skip_cpp_comment();
 524
 525   // The input file name.
 526   const char* input_file_name_;
 527   // The input file.
 528   FILE* input_file_;
 529   // The object used to keep track of file names and line numbers.
 530   Linemap* linemap_;
 531   // The line buffer.  This holds the current line.
 532   char* linebuf_;
 533   // The size of the line buffer.
 534   size_t linebufsize_;
 535   // The nmber of characters in the current line.
 536   size_t linesize_;
 537   // The current offset in linebuf_.
 538   size_t lineoff_;
 539   // The current line number.
 540   size_t lineno_;
 541   // Whether to add a semicolon if we see a newline now.
 542   bool add_semi_at_eol_;
 543   // Pragmas for the next function, from magic comments.
 544   unsigned int pragmas_;
 545   // The external name to use for a function declaration, from a magic
 546   // //extern comment.
 547   std::string extern_;
 548   // The list of //go:linkname comments, if any.
 549   Linknames* linknames_;
 550 };
 551
 552 #endif // !defined(GO_LEX_H)