dmd2/lexer.h

   1
   2 // Compiler implementation of the D programming language
   3 // Copyright (c) 1999-2008 by Digital Mars
   4 // All Rights Reserved
   5 // written by Walter Bright
   6 // http://www.digitalmars.com
   7 // License for redistribution is by either the Artistic License
   8 // in artistic.txt, or the GNU General Public License in gnu.txt.
   9 // See the included readme.txt for details.
  10
  11 #ifndef DMD_LEXER_H
  12 #define DMD_LEXER_H
  13
  14 #ifdef __DMC__
  15 #pragma once
  16 #endif /* __DMC__ */
  17
  18 #include "root.h"
  19 #include "mars.h"
  20
  21 struct StringTable;
  22 struct Identifier;
  23 struct Module;
  24
  25 /* Tokens:
  26         (       )
  27         [       ]
  28         {       }
  29         <       >       <=      >=      ==      !=      ===     !==
  30         <<      >>      <<=     >>=     >>>     >>>=
  31         +       -       +=      -=
  32         *       /       %       *=      /=      %=
  33         &       |       ^       &=      |=      ^=
  34         =       !       ~
  35         ++      --
  36         .       ->      :       ,
  37         ?       &&      ||
  38         @
  39  */
  40
  41 enum TOK
  42 {
  43         TOKreserved,
  44
  45         // Other
  46         TOKlparen,      TOKrparen,
  47         TOKlbracket,    TOKrbracket,
  48         TOKlcurly,      TOKrcurly,
  49         TOKcolon,       TOKneg,
  50         TOKsemicolon,   TOKdotdotdot,
  51         TOKeof,         TOKcast,
  52         TOKnull,        TOKassert,
  53         TOKtrue,        TOKfalse,
  54         TOKarray,       TOKcall,
  55         TOKaddress,     TOKtypedot,
  56         TOKtype,        TOKthrow,
  57         TOKnew,         TOKdelete,
  58         TOKstar,        TOKsymoff,
  59         TOKvar,         TOKdotvar,
  60         TOKdotti,       TOKdotexp,
  61         TOKdottype,     TOKslice,
  62         TOKarraylength, TOKversion,
  63         TOKmodule,      TOKdollar,
  64         TOKtemplate,    TOKdottd,
  65         TOKdeclaration, TOKtypeof,
  66         TOKpragma,      TOKdsymbol,
  67         TOKtypeid,      TOKuadd,
  68         TOKremove,
  69         TOKnewanonclass, TOKcomment,
  70         TOKarrayliteral, TOKassocarrayliteral,
  71         TOKstructliteral,
  72
  73         // Operators
  74         TOKlt,          TOKgt,
  75         TOKle,          TOKge,
  76         TOKequal,       TOKnotequal,
  77         TOKidentity,    TOKnotidentity,
  78         TOKindex,       TOKis,
  79         TOKtobool,
  80
  81 // 60
  82         // NCEG floating point compares
  83         // !<>=     <>    <>=    !>     !>=   !<     !<=   !<>
  84         TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
  85
  86         TOKshl,         TOKshr,
  87         TOKshlass,      TOKshrass,
  88         TOKushr,        TOKushrass,
  89         TOKcat,         TOKcatass,      // ~ ~=
  90         TOKadd,         TOKmin,         TOKaddass,      TOKminass,
  91         TOKmul,         TOKdiv,         TOKmod,
  92         TOKmulass,      TOKdivass,      TOKmodass,
  93         TOKand,         TOKor,          TOKxor,
  94         TOKandass,      TOKorass,       TOKxorass,
  95         TOKassign,      TOKnot,         TOKtilde,
  96         TOKplusplus,    TOKminusminus,  TOKconstruct,   TOKblit,
  97         TOKdot,         TOKarrow,       TOKcomma,
  98         TOKquestion,    TOKandand,      TOKoror,
  99
 100 // 104
 101         // Numeric literals
 102         TOKint32v, TOKuns32v,
 103         TOKint64v, TOKuns64v,
 104         TOKfloat32v, TOKfloat64v, TOKfloat80v,
 105         TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
 106
 107         // Char constants
 108         TOKcharv, TOKwcharv, TOKdcharv,
 109
 110         // Leaf operators
 111         TOKidentifier,  TOKstring,
 112         TOKthis,        TOKsuper,
 113         TOKhalt,        TOKtuple,
 114
 115         // Basic types
 116         TOKvoid,
 117         TOKint8, TOKuns8,
 118         TOKint16, TOKuns16,
 119         TOKint32, TOKuns32,
 120         TOKint64, TOKuns64,
 121         TOKfloat32, TOKfloat64, TOKfloat80,
 122         TOKimaginary32, TOKimaginary64, TOKimaginary80,
 123         TOKcomplex32, TOKcomplex64, TOKcomplex80,
 124         TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
 125         TOKcent, TOKucent,
 126
 127         // Aggregates
 128         TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
 129         TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
 130         TOKmixin,
 131
 132         TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
 133         TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
 134         TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
 135         TOKauto, TOKpackage, TOKmanifest,
 136
 137         // Statements
 138         TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
 139         TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
 140         TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
 141         TOKasm, TOKforeach, TOKforeach_reverse, TOKreversed,
 142         TOKscope,
 143         TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
 144         TOKlog_error, TOKlog_warning, TOKlog_info, TOKlog_trace,
 145
 146         // Contracts
 147         TOKbody, TOKinvariant,
 148
 149         // Testing
 150         TOKunittest,
 151
 152         // Added after 1.0
 153         TOKref,
 154         TOKmacro,
 155 #if V2
 156         TOKtraits,
 157         TOKoverloadset,
 158         TOKpure,
 159         TOKnothrow,
 160         TOKtls,
 161         TOKline,
 162         TOKfile,
 163 #endif
 164
 165         // TAL
 166         TOKextends,
 167         TOKimplements,
 168         TOKendline,
 169         TOKat,
 170         TOKlogger,              // Not a real symbol
 171
 172         TOKMAX
 173 };
 174
 175 #define CASE_BASIC_TYPES                        \
 176         case TOKwchar: case TOKdchar:           \
 177         case TOKbit: case TOKbool: case TOKchar:        \
 178         case TOKint8: case TOKuns8:             \
 179         case TOKint16: case TOKuns16:           \
 180         case TOKint32: case TOKuns32:           \
 181         case TOKint64: case TOKuns64:           \
 182         case TOKfloat32: case TOKfloat64: case TOKfloat80:              \
 183         case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:  \
 184         case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:        \
 185         case TOKvoid
 186
 187 #define CASE_BASIC_TYPES_X(t)                                   \
 188         case TOKvoid:    t = Type::tvoid;  goto LabelX;         \
 189         case TOKint8:    t = Type::tint8;  goto LabelX;         \
 190         case TOKuns8:    t = Type::tuns8;  goto LabelX;         \
 191         case TOKint16:   t = Type::tint16; goto LabelX;         \
 192         case TOKuns16:   t = Type::tuns16; goto LabelX;         \
 193         case TOKint32:   t = Type::tint32; goto LabelX;         \
 194         case TOKuns32:   t = Type::tuns32; goto LabelX;         \
 195         case TOKint64:   t = Type::tint64; goto LabelX;         \
 196         case TOKuns64:   t = Type::tuns64; goto LabelX;         \
 197         case TOKfloat32: t = Type::tfloat32; goto LabelX;       \
 198         case TOKfloat64: t = Type::tfloat64; goto LabelX;       \
 199         case TOKfloat80: t = Type::tfloat80; goto LabelX;       \
 200         case TOKimaginary32: t = Type::timaginary32; goto LabelX;       \
 201         case TOKimaginary64: t = Type::timaginary64; goto LabelX;       \
 202         case TOKimaginary80: t = Type::timaginary80; goto LabelX;       \
 203         case TOKcomplex32: t = Type::tcomplex32; goto LabelX;   \
 204         case TOKcomplex64: t = Type::tcomplex64; goto LabelX;   \
 205         case TOKcomplex80: t = Type::tcomplex80; goto LabelX;   \
 206         case TOKbit:     t = Type::tbit;     goto LabelX;       \
 207         case TOKbool:    t = Type::tbool;    goto LabelX;       \
 208         case TOKchar:    t = Type::tchar;    goto LabelX;       \
 209         case TOKwchar:   t = Type::twchar; goto LabelX; \
 210         case TOKdchar:   t = Type::tdchar; goto LabelX; \
 211         LabelX
 212
 213 struct Token
 214 {
 215     Token *next;
 216     unsigned char *ptr;         // pointer to first character of this token within buffer
 217     enum TOK value;
 218     unsigned char *blockComment; // doc comment string prior to this token
 219     unsigned char *lineComment;  // doc comment for previous token
 220     union
 221     {
 222         // Integers
 223         d_int32 int32value;
 224         d_uns32 uns32value;
 225         d_int64 int64value;
 226         d_uns64 uns64value;
 227
 228         // Floats
 229 #ifdef IN_GCC
 230         // real_t float80value; // can't use this in a union!
 231 #else
 232         d_float80 float80value;
 233 #endif
 234
 235         struct
 236         {   unsigned char *ustring;     // UTF8 string
 237             unsigned len;
 238             unsigned char postfix;      // 'c', 'w', 'd'
 239         };
 240
 241         Identifier *ident;
 242     };
 243 #ifdef IN_GCC
 244     real_t float80value; // can't use this in a union!
 245 #endif
 246
 247     static char *tochars[TOKMAX];
 248     static void *operator new(size_t sz);
 249
 250     int isKeyword();
 251     void print();
 252     char *toChars();
 253     static char *toChars(enum TOK);
 254 };
 255
 256 struct Lexer
 257 {
 258     static StringTable stringtable;
 259     static OutBuffer stringbuffer;
 260     static Token *freelist;
 261
 262     Loc loc;                    // for error messages
 263
 264     unsigned char *base;        // pointer to start of buffer
 265     unsigned char *end;         // past end of buffer
 266     unsigned char *p;           // current character
 267     Token token;
 268     Module *mod;
 269     int doDocComment;           // collect doc comment information
 270     int anyToken;               // !=0 means seen at least one token
 271     int commentToken;           // !=0 means comments are TOKcomment's
 272
 273     bool dltSyntax;
 274     unsigned int indent;        // Current indent level
 275     int atStartOfLine;
 276     int nesting;                // Counts { [ (; we ignore indents inside these
 277     int incLineno;              // Increate line number at next opportunity
 278
 279     Lexer(Module *mod,
 280         unsigned char *base, unsigned begoffset, unsigned endoffset,
 281         int doDocComment, int commentToken, bool dltSyntax);
 282
 283     static void initKeywords();
 284     static Identifier *idPool(const char *s);
 285     static Identifier *uniqueId(const char *s);
 286     static Identifier *uniqueId(const char *s, int num);
 287
 288     TOK nextToken();
 289     void scan(Token *t);
 290     Token *peek(Token *t);
 291     Token *peekPastParen(Token *t);
 292     unsigned escapeSequence();
 293     TOK wysiwygStringConstant(Token *t, int tc);
 294     TOK hexStringConstant(Token *t);
 295 #if V2
 296     TOK delimitedStringConstant(Token *t);
 297     TOK tokenStringConstant(Token *t);
 298 #endif
 299     TOK escapeStringConstant(Token *t, int wide);
 300     TOK charConstant(Token *t, int wide);
 301     void stringPostfix(Token *t);
 302     unsigned wchar(unsigned u);
 303     TOK number(Token *t);
 304     TOK inreal(Token *t);
 305     void error(const char *format, ...);
 306     void error(Loc loc, const char *format, ...);
 307     void pragma();
 308     unsigned decodeUTF();
 309     void getDocComment(Token *t, unsigned lineComment);
 310
 311     static int isValidIdentifier(char *p);
 312     static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
 313 };
 314
 315 #endif /* DMD_LEXER_H */