dmd2/lexer.h

   1
   2 // Compiler implementation of the D programming language
   3 // Copyright (c) 1999-2008 by Digital Mars
   4 // All Rights Reserved
   5 // written by Walter Bright
   6 // http://www.digitalmars.com
   7 // License for redistribution is by either the Artistic License
   8 // in artistic.txt, or the GNU General Public License in gnu.txt.
   9 // See the included readme.txt for details.
  10
  11 #ifndef DMD_LEXER_H
  12 #define DMD_LEXER_H
  13
  14 #ifdef __DMC__
  15 #pragma once
  16 #endif /* __DMC__ */
  17
  18 #include "root.h"
  19 #include "mars.h"
  20
  21 struct StringTable;
  22 struct Identifier;
  23 struct Module;
  24
  25 /* Tokens:
  26         (       )
  27         [       ]
  28         {       }
  29         <       >       <=      >=      ==      !=      ===     !==
  30         <<      >>      <<=     >>=     >>>     >>>=
  31         +       -       +=      -=
  32         *       /       %       *=      /=      %=
  33         &       |       ^       &=      |=      ^=
  34         =       !       ~
  35         ++      --
  36         .       ->      :       ,
  37         ?       &&      ||
  38  */
  39
  40 enum TOK
  41 {
  42         TOKreserved,
  43
  44         // Other
  45         TOKlparen,      TOKrparen,
  46         TOKlbracket,    TOKrbracket,
  47         TOKlcurly,      TOKrcurly,
  48         TOKcolon,       TOKneg,
  49         TOKsemicolon,   TOKdotdotdot,
  50         TOKeof,         TOKcast,
  51         TOKnull,        TOKassert,
  52         TOKtrue,        TOKfalse,
  53         TOKarray,       TOKcall,
  54         TOKaddress,     TOKtypedot,
  55         TOKtype,        TOKthrow,
  56         TOKnew,         TOKdelete,
  57         TOKstar,        TOKsymoff,
  58         TOKvar,         TOKdotvar,
  59         TOKdotti,       TOKdotexp,
  60         TOKdottype,     TOKslice,
  61         TOKarraylength, TOKversion,
  62         TOKmodule,      TOKdollar,
  63         TOKtemplate,    TOKdottd,
  64         TOKdeclaration, TOKtypeof,
  65         TOKpragma,      TOKdsymbol,
  66         TOKtypeid,      TOKuadd,
  67         TOKremove,
  68         TOKnewanonclass, TOKcomment,
  69         TOKarrayliteral, TOKassocarrayliteral,
  70         TOKstructliteral,
  71
  72         // Operators
  73         TOKlt,          TOKgt,
  74         TOKle,          TOKge,
  75         TOKequal,       TOKnotequal,
  76         TOKidentity,    TOKnotidentity,
  77         TOKindex,       TOKis,
  78         TOKtobool,
  79
  80 // 60
  81         // NCEG floating point compares
  82         // !<>=     <>    <>=    !>     !>=   !<     !<=   !<>
  83         TOKunord,TOKlg,TOKleg,TOKule,TOKul,TOKuge,TOKug,TOKue,
  84
  85         TOKshl,         TOKshr,
  86         TOKshlass,      TOKshrass,
  87         TOKushr,        TOKushrass,
  88         TOKcat,         TOKcatass,      // ~ ~=
  89         TOKadd,         TOKmin,         TOKaddass,      TOKminass,
  90         TOKmul,         TOKdiv,         TOKmod,
  91         TOKmulass,      TOKdivass,      TOKmodass,
  92         TOKand,         TOKor,          TOKxor,
  93         TOKandass,      TOKorass,       TOKxorass,
  94         TOKassign,      TOKnot,         TOKtilde,
  95         TOKplusplus,    TOKminusminus,  TOKconstruct,   TOKblit,
  96         TOKdot,         TOKarrow,       TOKcomma,
  97         TOKquestion,    TOKandand,      TOKoror,
  98
  99 // 104
 100         // Numeric literals
 101         TOKint32v, TOKuns32v,
 102         TOKint64v, TOKuns64v,
 103         TOKfloat32v, TOKfloat64v, TOKfloat80v,
 104         TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,
 105
 106         // Char constants
 107         TOKcharv, TOKwcharv, TOKdcharv,
 108
 109         // Leaf operators
 110         TOKidentifier,  TOKstring,
 111         TOKthis,        TOKsuper,
 112         TOKhalt,        TOKtuple,
 113
 114         // Basic types
 115         TOKvoid,
 116         TOKint8, TOKuns8,
 117         TOKint16, TOKuns16,
 118         TOKint32, TOKuns32,
 119         TOKint64, TOKuns64,
 120         TOKfloat32, TOKfloat64, TOKfloat80,
 121         TOKimaginary32, TOKimaginary64, TOKimaginary80,
 122         TOKcomplex32, TOKcomplex64, TOKcomplex80,
 123         TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
 124         TOKcent, TOKucent,
 125
 126         // Aggregates
 127         TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
 128         TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,
 129         TOKmixin,
 130
 131         TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
 132         TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
 133         TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
 134         TOKauto, TOKpackage, TOKmanifest,
 135
 136         // Statements
 137         TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
 138         TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
 139         TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
 140         TOKasm, TOKforeach, TOKforeach_reverse,
 141         TOKscope,
 142         TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,
 143
 144         // Contracts
 145         TOKbody, TOKinvariant,
 146
 147         // Testing
 148         TOKunittest,
 149
 150         // Added after 1.0
 151         TOKref,
 152         TOKmacro,
 153 #if V2
 154         TOKtraits,
 155         TOKoverloadset,
 156         TOKpure,
 157         TOKnothrow,
 158         TOKtls,
 159         TOKline,
 160         TOKfile,
 161 #endif
 162
 163         TOKMAX
 164 };
 165
 166 #define CASE_BASIC_TYPES                        \
 167         case TOKwchar: case TOKdchar:           \
 168         case TOKbit: case TOKbool: case TOKchar:        \
 169         case TOKint8: case TOKuns8:             \
 170         case TOKint16: case TOKuns16:           \
 171         case TOKint32: case TOKuns32:           \
 172         case TOKint64: case TOKuns64:           \
 173         case TOKfloat32: case TOKfloat64: case TOKfloat80:              \
 174         case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:  \
 175         case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:        \
 176         case TOKvoid
 177
 178 #define CASE_BASIC_TYPES_X(t)                                   \
 179         case TOKvoid:    t = Type::tvoid;  goto LabelX;         \
 180         case TOKint8:    t = Type::tint8;  goto LabelX;         \
 181         case TOKuns8:    t = Type::tuns8;  goto LabelX;         \
 182         case TOKint16:   t = Type::tint16; goto LabelX;         \
 183         case TOKuns16:   t = Type::tuns16; goto LabelX;         \
 184         case TOKint32:   t = Type::tint32; goto LabelX;         \
 185         case TOKuns32:   t = Type::tuns32; goto LabelX;         \
 186         case TOKint64:   t = Type::tint64; goto LabelX;         \
 187         case TOKuns64:   t = Type::tuns64; goto LabelX;         \
 188         case TOKfloat32: t = Type::tfloat32; goto LabelX;       \
 189         case TOKfloat64: t = Type::tfloat64; goto LabelX;       \
 190         case TOKfloat80: t = Type::tfloat80; goto LabelX;       \
 191         case TOKimaginary32: t = Type::timaginary32; goto LabelX;       \
 192         case TOKimaginary64: t = Type::timaginary64; goto LabelX;       \
 193         case TOKimaginary80: t = Type::timaginary80; goto LabelX;       \
 194         case TOKcomplex32: t = Type::tcomplex32; goto LabelX;   \
 195         case TOKcomplex64: t = Type::tcomplex64; goto LabelX;   \
 196         case TOKcomplex80: t = Type::tcomplex80; goto LabelX;   \
 197         case TOKbit:     t = Type::tbit;     goto LabelX;       \
 198         case TOKbool:    t = Type::tbool;    goto LabelX;       \
 199         case TOKchar:    t = Type::tchar;    goto LabelX;       \
 200         case TOKwchar:   t = Type::twchar; goto LabelX; \
 201         case TOKdchar:   t = Type::tdchar; goto LabelX; \
 202         LabelX
 203
 204 struct Token
 205 {
 206     Token *next;
 207     unsigned char *ptr;         // pointer to first character of this token within buffer
 208     enum TOK value;
 209     unsigned char *blockComment; // doc comment string prior to this token
 210     unsigned char *lineComment;  // doc comment for previous token
 211     union
 212     {
 213         // Integers
 214         d_int32 int32value;
 215         d_uns32 uns32value;
 216         d_int64 int64value;
 217         d_uns64 uns64value;
 218
 219         // Floats
 220 #ifdef IN_GCC
 221         // real_t float80value; // can't use this in a union!
 222 #else
 223         d_float80 float80value;
 224 #endif
 225
 226         struct
 227         {   unsigned char *ustring;     // UTF8 string
 228             unsigned len;
 229             unsigned char postfix;      // 'c', 'w', 'd'
 230         };
 231
 232         Identifier *ident;
 233     };
 234 #ifdef IN_GCC
 235     real_t float80value; // can't use this in a union!
 236 #endif
 237
 238     static char *tochars[TOKMAX];
 239     static void *operator new(size_t sz);
 240
 241     int isKeyword();
 242     void print();
 243     char *toChars();
 244     static char *toChars(enum TOK);
 245 };
 246
 247 struct Lexer
 248 {
 249     static StringTable stringtable;
 250     static OutBuffer stringbuffer;
 251     static Token *freelist;
 252
 253     Loc loc;                    // for error messages
 254
 255     unsigned char *base;        // pointer to start of buffer
 256     unsigned char *end;         // past end of buffer
 257     unsigned char *p;           // current character
 258     Token token;
 259     Module *mod;
 260     int doDocComment;           // collect doc comment information
 261     int anyToken;               // !=0 means seen at least one token
 262     int commentToken;           // !=0 means comments are TOKcomment's
 263
 264     Lexer(Module *mod,
 265         unsigned char *base, unsigned begoffset, unsigned endoffset,
 266         int doDocComment, int commentToken);
 267
 268     static void initKeywords();
 269     static Identifier *idPool(const char *s);
 270     static Identifier *uniqueId(const char *s);
 271     static Identifier *uniqueId(const char *s, int num);
 272
 273     TOK nextToken();
 274     void scan(Token *t);
 275     Token *peek(Token *t);
 276     Token *peekPastParen(Token *t);
 277     unsigned escapeSequence();
 278     TOK wysiwygStringConstant(Token *t, int tc);
 279     TOK hexStringConstant(Token *t);
 280 #if V2
 281     TOK delimitedStringConstant(Token *t);
 282     TOK tokenStringConstant(Token *t);
 283 #endif
 284     TOK escapeStringConstant(Token *t, int wide);
 285     TOK charConstant(Token *t, int wide);
 286     void stringPostfix(Token *t);
 287     unsigned wchar(unsigned u);
 288     TOK number(Token *t);
 289     TOK inreal(Token *t);
 290     void error(const char *format, ...);
 291     void error(Loc loc, const char *format, ...);
 292     void pragma();
 293     unsigned decodeUTF();
 294     void getDocComment(Token *t, unsigned lineComment);
 295
 296     static int isValidIdentifier(char *p);
 297     static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);
 298 };
 299
 300 #endif /* DMD_LEXER_H */