gcc/d/dmd/tokens.d

   1 /**
   2  * Defines lexical tokens.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
   5  *
   6  * Copyright:   Copyright (C) 1999-2024 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_tokens.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
  12  */
  13
  14 module dmd.tokens;
  15
  16 import core.stdc.ctype;
  17 import core.stdc.stdio;
  18 import core.stdc.string;
  19 import dmd.identifier;
  20 import dmd.location;
  21 import dmd.root.ctfloat;
  22 import dmd.common.outbuffer;
  23 import dmd.root.rmem;
  24 import dmd.root.utf;
  25
  26 enum TOK : ubyte
  27 {
  28     reserved,
  29
  30     // Other
  31     leftParenthesis,
  32     rightParenthesis,
  33     leftBracket,
  34     rightBracket,
  35     leftCurly,
  36     rightCurly,
  37     colon,
  38     semicolon,
  39     dotDotDot,
  40     endOfFile,
  41     cast_,
  42     null_,
  43     assert_,
  44     true_,
  45     false_,
  46     throw_,
  47     new_,
  48     delete_,
  49     variable,
  50     slice,
  51     version_,
  52     module_,
  53     dollar,
  54     template_,
  55     typeof_,
  56     pragma_,
  57     typeid_,
  58     comment,
  59
  60     // Operators
  61     lessThan,
  62     greaterThan,
  63     lessOrEqual,
  64     greaterOrEqual,
  65     equal,
  66     notEqual,
  67     identity,
  68     notIdentity,
  69     is_,
  70
  71     leftShift,
  72     rightShift,
  73     leftShiftAssign,
  74     rightShiftAssign,
  75     unsignedRightShift,
  76     unsignedRightShiftAssign,
  77     concatenateAssign, // ~=
  78     add,
  79     min,
  80     addAssign,
  81     minAssign,
  82     mul,
  83     div,
  84     mod,
  85     mulAssign,
  86     divAssign,
  87     modAssign,
  88     and,
  89     or,
  90     xor,
  91     andAssign,
  92     orAssign,
  93     xorAssign,
  94     assign,
  95     not,
  96     tilde,
  97     plusPlus,
  98     minusMinus,
  99     dot,
 100     comma,
 101     question,
 102     andAnd,
 103     orOr,
 104
 105     // Numeric literals
 106     int32Literal,
 107     uns32Literal,
 108     int64Literal,
 109     uns64Literal,
 110     int128Literal,
 111     uns128Literal,
 112     float32Literal,
 113     float64Literal,
 114     float80Literal,
 115     imaginary32Literal,
 116     imaginary64Literal,
 117     imaginary80Literal,
 118
 119     // Char constants
 120     charLiteral,
 121     wcharLiteral,
 122     dcharLiteral,
 123
 124     // Leaf operators
 125     identifier,
 126     string_,
 127     interpolated,
 128     hexadecimalString,
 129     this_,
 130     super_,
 131     error,
 132
 133     // Basic types
 134     void_,
 135     int8,
 136     uns8,
 137     int16,
 138     uns16,
 139     int32,
 140     uns32,
 141     int64,
 142     uns64,
 143     int128,
 144     uns128,
 145     float32,
 146     float64,
 147     float80,
 148     imaginary32,
 149     imaginary64,
 150     imaginary80,
 151     complex32,
 152     complex64,
 153     complex80,
 154     char_,
 155     wchar_,
 156     dchar_,
 157     bool_,
 158
 159     // Aggregates
 160     struct_,
 161     class_,
 162     interface_,
 163     union_,
 164     enum_,
 165     import_,
 166     alias_,
 167     override_,
 168     delegate_,
 169     function_,
 170     mixin_,
 171     align_,
 172     extern_,
 173     private_,
 174     protected_,
 175     public_,
 176     export_,
 177     static_,
 178     final_,
 179     const_,
 180     abstract_,
 181     debug_,
 182     deprecated_,
 183     in_,
 184     out_,
 185     inout_,
 186     lazy_,
 187     auto_,
 188     package_,
 189     immutable_,
 190
 191     // Statements
 192     if_,
 193     else_,
 194     while_,
 195     for_,
 196     do_,
 197     switch_,
 198     case_,
 199     default_,
 200     break_,
 201     continue_,
 202     with_,
 203     synchronized_,
 204     return_,
 205     goto_,
 206     try_,
 207     catch_,
 208     finally_,
 209     asm_,
 210     foreach_,
 211     foreach_reverse_,
 212     scope_,
 213     onScopeExit,
 214     onScopeFailure,
 215     onScopeSuccess,
 216
 217     // Contracts
 218     invariant_,
 219
 220     // Testing
 221     unittest_,
 222
 223     // Added after 1.0
 224     argumentTypes,
 225     ref_,
 226     macro_,
 227
 228     parameters,
 229     traits,
 230     pure_,
 231     nothrow_,
 232     gshared,
 233     line,
 234     file,
 235     fileFullPath,
 236     moduleString,   // __MODULE__
 237     functionString, // __FUNCTION__
 238     prettyFunction, // __PRETTY_FUNCTION__
 239     shared_,
 240     at,
 241     pow,
 242     powAssign,
 243     goesTo,
 244     vector,
 245     pound,
 246
 247     arrow,      // ->
 248     colonColon, // ::
 249     wchar_tLiteral,
 250     endOfLine,  // \n, \r, \u2028, \u2029
 251     whitespace,
 252
 253     // C only keywords
 254     inline,
 255     register,
 256     restrict,
 257     signed,
 258     sizeof_,
 259     typedef_,
 260     unsigned,
 261     volatile,
 262     _Alignas,
 263     _Alignof,
 264     _Atomic,
 265     _Bool,
 266     _Complex,
 267     _Generic,
 268     _Imaginary,
 269     _Noreturn,
 270     _Static_assert,
 271     _Thread_local,
 272
 273     // C only extended keywords
 274     _assert,
 275     _import,
 276     __cdecl,
 277     __declspec,
 278     __stdcall,
 279     __thread,
 280     __pragma,
 281     __int128,
 282     __attribute__,
 283 }
 284
 285 /// Expression nodes
 286 enum EXP : ubyte
 287 {
 288     reserved,
 289
 290     // Other
 291     negate,
 292     cast_,
 293     null_,
 294     assert_,
 295     array,
 296     call,
 297     address,
 298     type,
 299     throw_,
 300     new_,
 301     delete_,
 302     star,
 303     symbolOffset,
 304     variable,
 305     dotVariable,
 306     dotIdentifier,
 307     dotTemplateInstance,
 308     dotType,
 309     slice,
 310     arrayLength,
 311     dollar,
 312     template_,
 313     dotTemplateDeclaration,
 314     declaration,
 315     dSymbol,
 316     typeid_,
 317     uadd,
 318     remove,
 319     newAnonymousClass,
 320     arrayLiteral,
 321     assocArrayLiteral,
 322     structLiteral,
 323     classReference,
 324     thrownException,
 325     delegatePointer,
 326     delegateFunctionPointer,
 327
 328     // Operators
 329     lessThan,
 330     greaterThan,
 331     lessOrEqual,
 332     greaterOrEqual,
 333     equal,
 334     notEqual,
 335     identity,
 336     notIdentity,
 337     index,
 338     is_,
 339
 340     leftShift,
 341     rightShift,
 342     leftShiftAssign,
 343     rightShiftAssign,
 344     unsignedRightShift,
 345     unsignedRightShiftAssign,
 346     concatenate,
 347     concatenateAssign, // ~=
 348     concatenateElemAssign,
 349     concatenateDcharAssign,
 350     add,
 351     min,
 352     addAssign,
 353     minAssign,
 354     mul,
 355     div,
 356     mod,
 357     mulAssign,
 358     divAssign,
 359     modAssign,
 360     and,
 361     or,
 362     xor,
 363     andAssign,
 364     orAssign,
 365     xorAssign,
 366     assign,
 367     not,
 368     tilde,
 369     plusPlus,
 370     minusMinus,
 371     construct,
 372     blit,
 373     dot,
 374     comma,
 375     question,
 376     andAnd,
 377     orOr,
 378     prePlusPlus,
 379     preMinusMinus,
 380
 381     // Leaf operators
 382     identifier,
 383     string_,
 384     interpolated,
 385     this_,
 386     super_,
 387     halt,
 388     tuple,
 389     error,
 390
 391     // Basic types
 392     void_,
 393     int64,
 394     float64,
 395     complex80,
 396     import_,
 397     delegate_,
 398     function_,
 399     mixin_,
 400     in_,
 401     break_,
 402     continue_,
 403     goto_,
 404     scope_,
 405
 406     traits,
 407     overloadSet,
 408     line,
 409     file,
 410     fileFullPath,
 411     moduleString,   // __MODULE__
 412     functionString, // __FUNCTION__
 413     prettyFunction, // __PRETTY_FUNCTION__
 414     pow,
 415     powAssign,
 416     vector,
 417
 418     voidExpression,
 419     cantExpression,
 420     showCtfeContext,
 421     objcClassReference,
 422     vectorArray,
 423     compoundLiteral, // ( type-name ) { initializer-list }
 424     _Generic,
 425     interval,
 426
 427     loweredAssignExp,
 428 }
 429
 430 enum FirstCKeyword = TOK.inline;
 431
 432 // Assert that all token enum members have consecutive values and
 433 // that none of them overlap
 434 static assert(() {
 435     foreach (idx, enumName; __traits(allMembers, TOK)) {
 436        static if (idx != __traits(getMember, TOK, enumName)) {
 437            pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
 438            static assert(0);
 439        }
 440     }
 441     return true;
 442 }());
 443
 444 /****************************************
 445  */
 446
 447 private immutable TOK[] keywords =
 448 [
 449     TOK.this_,
 450     TOK.super_,
 451     TOK.assert_,
 452     TOK.null_,
 453     TOK.true_,
 454     TOK.false_,
 455     TOK.cast_,
 456     TOK.new_,
 457     TOK.delete_,
 458     TOK.throw_,
 459     TOK.module_,
 460     TOK.pragma_,
 461     TOK.typeof_,
 462     TOK.typeid_,
 463     TOK.template_,
 464     TOK.void_,
 465     TOK.int8,
 466     TOK.uns8,
 467     TOK.int16,
 468     TOK.uns16,
 469     TOK.int32,
 470     TOK.uns32,
 471     TOK.int64,
 472     TOK.uns64,
 473     TOK.int128,
 474     TOK.uns128,
 475     TOK.float32,
 476     TOK.float64,
 477     TOK.float80,
 478     TOK.bool_,
 479     TOK.char_,
 480     TOK.wchar_,
 481     TOK.dchar_,
 482     TOK.imaginary32,
 483     TOK.imaginary64,
 484     TOK.imaginary80,
 485     TOK.complex32,
 486     TOK.complex64,
 487     TOK.complex80,
 488     TOK.delegate_,
 489     TOK.function_,
 490     TOK.is_,
 491     TOK.if_,
 492     TOK.else_,
 493     TOK.while_,
 494     TOK.for_,
 495     TOK.do_,
 496     TOK.switch_,
 497     TOK.case_,
 498     TOK.default_,
 499     TOK.break_,
 500     TOK.continue_,
 501     TOK.synchronized_,
 502     TOK.return_,
 503     TOK.goto_,
 504     TOK.try_,
 505     TOK.catch_,
 506     TOK.finally_,
 507     TOK.with_,
 508     TOK.asm_,
 509     TOK.foreach_,
 510     TOK.foreach_reverse_,
 511     TOK.scope_,
 512     TOK.struct_,
 513     TOK.class_,
 514     TOK.interface_,
 515     TOK.union_,
 516     TOK.enum_,
 517     TOK.import_,
 518     TOK.mixin_,
 519     TOK.static_,
 520     TOK.final_,
 521     TOK.const_,
 522     TOK.alias_,
 523     TOK.override_,
 524     TOK.abstract_,
 525     TOK.debug_,
 526     TOK.deprecated_,
 527     TOK.in_,
 528     TOK.out_,
 529     TOK.inout_,
 530     TOK.lazy_,
 531     TOK.auto_,
 532     TOK.align_,
 533     TOK.extern_,
 534     TOK.private_,
 535     TOK.package_,
 536     TOK.protected_,
 537     TOK.public_,
 538     TOK.export_,
 539     TOK.invariant_,
 540     TOK.unittest_,
 541     TOK.version_,
 542     TOK.argumentTypes,
 543     TOK.parameters,
 544     TOK.ref_,
 545     TOK.macro_,
 546     TOK.pure_,
 547     TOK.nothrow_,
 548     TOK.gshared,
 549     TOK.traits,
 550     TOK.vector,
 551     TOK.file,
 552     TOK.fileFullPath,
 553     TOK.line,
 554     TOK.moduleString,
 555     TOK.functionString,
 556     TOK.prettyFunction,
 557     TOK.shared_,
 558     TOK.immutable_,
 559
 560     // C only keywords
 561     TOK.inline,
 562     TOK.register,
 563     TOK.restrict,
 564     TOK.signed,
 565     TOK.sizeof_,
 566     TOK.typedef_,
 567     TOK.unsigned,
 568     TOK.volatile,
 569     TOK._Alignas,
 570     TOK._Alignof,
 571     TOK._Atomic,
 572     TOK._Bool,
 573     TOK._Complex,
 574     TOK._Generic,
 575     TOK._Imaginary,
 576     TOK._Noreturn,
 577     TOK._Static_assert,
 578     TOK._Thread_local,
 579
 580     // C only extended keywords
 581     TOK._assert,
 582     TOK._import,
 583     TOK.__cdecl,
 584     TOK.__declspec,
 585     TOK.__stdcall,
 586     TOK.__thread,
 587     TOK.__pragma,
 588     TOK.__int128,
 589     TOK.__attribute__,
 590 ];
 591
 592 // Initialize the identifier pool
 593 shared static this() nothrow
 594 {
 595     Identifier.initTable();
 596     foreach (kw; keywords)
 597     {
 598         //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr);
 599         Identifier.idPool(Token.tochars[kw], kw);
 600     }
 601 }
 602
 603 /************************************
 604  * This is used to pick the C keywords out of the tokens.
 605  * If it's not a C keyword, then it's an identifier.
 606  */
 607 static immutable TOK[TOK.max + 1] Ckeywords =
 608 () {
 609     with (TOK)
 610     {
 611         TOK[TOK.max + 1] tab = identifier;  // default to identifier
 612         enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
 613                        enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
 614                        restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
 615                        union_, unsigned, void_, volatile, while_, asm_, typeof_,
 616                        _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
 617                        _Static_assert, _Thread_local,
 618                        _import, __cdecl, __declspec, __stdcall, __thread, __pragma, __int128, __attribute__,
 619                        _assert ];
 620
 621         foreach (kw; Ckwds)
 622             tab[kw] = cast(TOK) kw;
 623
 624         return tab;
 625     }
 626 } ();
 627
 628 struct InterpolatedSet {
 629     // all strings in the parts are zero terminated at length+1
 630     string[] parts;
 631 }
 632
 633 /***********************************************************
 634  */
 635 extern (C++) struct Token
 636 {
 637     Token* next;
 638     Loc loc;
 639     const(char)* ptr; // pointer to first character of this token within buffer
 640     TOK value;
 641     const(char)[] blockComment; // doc comment string prior to this token
 642     const(char)[] lineComment; // doc comment for previous token
 643
 644     union
 645     {
 646         // Integers
 647         long intvalue;
 648         ulong unsvalue;
 649         // Floats
 650         real_t floatvalue;
 651
 652         struct
 653         {
 654             union
 655             {
 656                 const(char)* ustring; // UTF8 string
 657                 InterpolatedSet* interpolatedSet;
 658             }
 659             uint len;
 660             ubyte postfix; // 'c', 'w', 'd'
 661         }
 662
 663         Identifier ident;
 664     }
 665
 666     extern (D) private static immutable string[TOK.max + 1] tochars =
 667     [
 668         // Keywords
 669         TOK.this_: "this",
 670         TOK.super_: "super",
 671         TOK.assert_: "assert",
 672         TOK.null_: "null",
 673         TOK.true_: "true",
 674         TOK.false_: "false",
 675         TOK.cast_: "cast",
 676         TOK.new_: "new",
 677         TOK.delete_: "delete",
 678         TOK.throw_: "throw",
 679         TOK.module_: "module",
 680         TOK.pragma_: "pragma",
 681         TOK.typeof_: "typeof",
 682         TOK.typeid_: "typeid",
 683         TOK.template_: "template",
 684         TOK.void_: "void",
 685         TOK.int8: "byte",
 686         TOK.uns8: "ubyte",
 687         TOK.int16: "short",
 688         TOK.uns16: "ushort",
 689         TOK.int32: "int",
 690         TOK.uns32: "uint",
 691         TOK.int64: "long",
 692         TOK.uns64: "ulong",
 693         TOK.int128: "cent",
 694         TOK.uns128: "ucent",
 695         TOK.float32: "float",
 696         TOK.float64: "double",
 697         TOK.float80: "real",
 698         TOK.bool_: "bool",
 699         TOK.char_: "char",
 700         TOK.wchar_: "wchar",
 701         TOK.dchar_: "dchar",
 702         TOK.imaginary32: "ifloat",
 703         TOK.imaginary64: "idouble",
 704         TOK.imaginary80: "ireal",
 705         TOK.complex32: "cfloat",
 706         TOK.complex64: "cdouble",
 707         TOK.complex80: "creal",
 708         TOK.delegate_: "delegate",
 709         TOK.function_: "function",
 710         TOK.is_: "is",
 711         TOK.if_: "if",
 712         TOK.else_: "else",
 713         TOK.while_: "while",
 714         TOK.for_: "for",
 715         TOK.do_: "do",
 716         TOK.switch_: "switch",
 717         TOK.case_: "case",
 718         TOK.default_: "default",
 719         TOK.break_: "break",
 720         TOK.continue_: "continue",
 721         TOK.synchronized_: "synchronized",
 722         TOK.return_: "return",
 723         TOK.goto_: "goto",
 724         TOK.try_: "try",
 725         TOK.catch_: "catch",
 726         TOK.finally_: "finally",
 727         TOK.with_: "with",
 728         TOK.asm_: "asm",
 729         TOK.foreach_: "foreach",
 730         TOK.foreach_reverse_: "foreach_reverse",
 731         TOK.scope_: "scope",
 732         TOK.struct_: "struct",
 733         TOK.class_: "class",
 734         TOK.interface_: "interface",
 735         TOK.union_: "union",
 736         TOK.enum_: "enum",
 737         TOK.import_: "import",
 738         TOK.mixin_: "mixin",
 739         TOK.static_: "static",
 740         TOK.final_: "final",
 741         TOK.const_: "const",
 742         TOK.alias_: "alias",
 743         TOK.override_: "override",
 744         TOK.abstract_: "abstract",
 745         TOK.debug_: "debug",
 746         TOK.deprecated_: "deprecated",
 747         TOK.in_: "in",
 748         TOK.out_: "out",
 749         TOK.inout_: "inout",
 750         TOK.lazy_: "lazy",
 751         TOK.auto_: "auto",
 752         TOK.align_: "align",
 753         TOK.extern_: "extern",
 754         TOK.private_: "private",
 755         TOK.package_: "package",
 756         TOK.protected_: "protected",
 757         TOK.public_: "public",
 758         TOK.export_: "export",
 759         TOK.invariant_: "invariant",
 760         TOK.unittest_: "unittest",
 761         TOK.version_: "version",
 762         TOK.argumentTypes: "__argTypes",
 763         TOK.parameters: "__parameters",
 764         TOK.ref_: "ref",
 765         TOK.macro_: "macro",
 766         TOK.pure_: "pure",
 767         TOK.nothrow_: "nothrow",
 768         TOK.gshared: "__gshared",
 769         TOK.traits: "__traits",
 770         TOK.vector: "__vector",
 771         TOK.file: "__FILE__",
 772         TOK.fileFullPath: "__FILE_FULL_PATH__",
 773         TOK.line: "__LINE__",
 774         TOK.moduleString: "__MODULE__",
 775         TOK.functionString: "__FUNCTION__",
 776         TOK.prettyFunction: "__PRETTY_FUNCTION__",
 777         TOK.shared_: "shared",
 778         TOK.immutable_: "immutable",
 779
 780         TOK.endOfFile: "End of File",
 781         TOK.leftCurly: "{",
 782         TOK.rightCurly: "}",
 783         TOK.leftParenthesis: "(",
 784         TOK.rightParenthesis: ")",
 785         TOK.leftBracket: "[",
 786         TOK.rightBracket: "]",
 787         TOK.semicolon: ";",
 788         TOK.colon: ":",
 789         TOK.comma: ",",
 790         TOK.dot: ".",
 791         TOK.xor: "^",
 792         TOK.xorAssign: "^=",
 793         TOK.assign: "=",
 794         TOK.lessThan: "<",
 795         TOK.greaterThan: ">",
 796         TOK.lessOrEqual: "<=",
 797         TOK.greaterOrEqual: ">=",
 798         TOK.equal: "==",
 799         TOK.notEqual: "!=",
 800         TOK.not: "!",
 801         TOK.leftShift: "<<",
 802         TOK.rightShift: ">>",
 803         TOK.unsignedRightShift: ">>>",
 804         TOK.add: "+",
 805         TOK.min: "-",
 806         TOK.mul: "*",
 807         TOK.div: "/",
 808         TOK.mod: "%",
 809         TOK.slice: "..",
 810         TOK.dotDotDot: "...",
 811         TOK.and: "&",
 812         TOK.andAnd: "&&",
 813         TOK.or: "|",
 814         TOK.orOr: "||",
 815         TOK.tilde: "~",
 816         TOK.dollar: "$",
 817         TOK.plusPlus: "++",
 818         TOK.minusMinus: "--",
 819         TOK.question: "?",
 820         TOK.variable: "var",
 821         TOK.addAssign: "+=",
 822         TOK.minAssign: "-=",
 823         TOK.mulAssign: "*=",
 824         TOK.divAssign: "/=",
 825         TOK.modAssign: "%=",
 826         TOK.leftShiftAssign: "<<=",
 827         TOK.rightShiftAssign: ">>=",
 828         TOK.unsignedRightShiftAssign: ">>>=",
 829         TOK.andAssign: "&=",
 830         TOK.orAssign: "|=",
 831         TOK.concatenateAssign: "~=",
 832         TOK.identity: "is",
 833         TOK.notIdentity: "!is",
 834         TOK.identifier: "identifier",
 835         TOK.at: "@",
 836         TOK.pow: "^^",
 837         TOK.powAssign: "^^=",
 838         TOK.goesTo: "=>",
 839         TOK.pound: "#",
 840         TOK.arrow: "->",
 841         TOK.colonColon: "::",
 842
 843         // For debugging
 844         TOK.error: "error",
 845         TOK.string_: "string",
 846         TOK.interpolated: "interpolated string",
 847         TOK.onScopeExit: "scope(exit)",
 848         TOK.onScopeSuccess: "scope(success)",
 849         TOK.onScopeFailure: "scope(failure)",
 850
 851         // Finish up
 852         TOK.reserved: "reserved",
 853         TOK.comment: "comment",
 854         TOK.int32Literal: "int32v",
 855         TOK.uns32Literal: "uns32v",
 856         TOK.int64Literal: "int64v",
 857         TOK.uns64Literal: "uns64v",
 858         TOK.int128Literal: "int128v",
 859         TOK.uns128Literal: "uns128v",
 860         TOK.float32Literal: "float32v",
 861         TOK.float64Literal: "float64v",
 862         TOK.float80Literal: "float80v",
 863         TOK.imaginary32Literal: "imaginary32v",
 864         TOK.imaginary64Literal: "imaginary64v",
 865         TOK.imaginary80Literal: "imaginary80v",
 866         TOK.charLiteral: "charv",
 867         TOK.wcharLiteral: "wcharv",
 868         TOK.dcharLiteral: "dcharv",
 869         TOK.wchar_tLiteral: "wchar_tv",
 870         TOK.hexadecimalString: "xstring",
 871         TOK.endOfLine: "\\n",
 872         TOK.whitespace: "whitespace",
 873
 874         // C only keywords
 875         TOK.inline    : "inline",
 876         TOK.register  : "register",
 877         TOK.restrict  : "restrict",
 878         TOK.signed    : "signed",
 879         TOK.sizeof_   : "sizeof",
 880         TOK.typedef_  : "typedef",
 881         TOK.unsigned  : "unsigned",
 882         TOK.volatile  : "volatile",
 883         TOK._Alignas  : "_Alignas",
 884         TOK._Alignof  : "_Alignof",
 885         TOK._Atomic   : "_Atomic",
 886         TOK._Bool     : "_Bool",
 887         TOK._Complex  : "_Complex",
 888         TOK._Generic  : "_Generic",
 889         TOK._Imaginary: "_Imaginary",
 890         TOK._Noreturn : "_Noreturn",
 891         TOK._Static_assert : "_Static_assert",
 892         TOK._Thread_local  : "_Thread_local",
 893
 894         // C only extended keywords
 895         TOK._assert       : "__check",
 896         TOK._import       : "__import",
 897         TOK.__cdecl        : "__cdecl",
 898         TOK.__declspec     : "__declspec",
 899         TOK.__stdcall      : "__stdcall",
 900         TOK.__thread       : "__thread",
 901         TOK.__pragma       : "__pragma",
 902         TOK.__int128       : "__int128",
 903         TOK.__attribute__  : "__attribute__",
 904     ];
 905
 906     static assert(() {
 907         foreach (s; tochars)
 908             assert(s.length);
 909         return true;
 910     }());
 911
 912 nothrow:
 913
 914     extern (D) int isKeyword() pure const @safe @nogc
 915     {
 916         foreach (kw; keywords)
 917         {
 918             if (kw == value)
 919                 return 1;
 920         }
 921         return 0;
 922     }
 923
 924     extern(D) void appendInterpolatedPart(const ref OutBuffer buf) {
 925         appendInterpolatedPart(cast(const(char)*)buf[].ptr, buf.length);
 926     }
 927     extern(D) void appendInterpolatedPart(const(char)[] str) {
 928         appendInterpolatedPart(str.ptr, str.length);
 929     }
 930     extern(D) void appendInterpolatedPart(const(char)* ptr, size_t length) {
 931         assert(value == TOK.interpolated);
 932         if (interpolatedSet is null)
 933             interpolatedSet = new InterpolatedSet;
 934
 935         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
 936         memcpy(s, ptr, length);
 937         s[length] = 0;
 938
 939         interpolatedSet.parts ~= cast(string) s[0 .. length];
 940     }
 941
 942     /****
 943      * Set to contents of ptr[0..length]
 944      * Params:
 945      *  ptr = pointer to string
 946      *  length = length of string
 947      */
 948     void setString(const(char)* ptr, size_t length)
 949     {
 950         value = TOK.string_;
 951         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
 952         memcpy(s, ptr, length);
 953         s[length] = 0;
 954         ustring = s;
 955         len = cast(uint)length;
 956         postfix = 0;
 957     }
 958
 959     /****
 960      * Set to contents of buf
 961      * Params:
 962      *  buf = string (not zero terminated)
 963      */
 964     void setString(const ref OutBuffer buf)
 965     {
 966         setString(cast(const(char)*)buf[].ptr, buf.length);
 967     }
 968
 969     /****
 970      * Set to empty string
 971      */
 972     void setString()
 973     {
 974         value = TOK.string_;
 975         ustring = "";
 976         len = 0;
 977         postfix = 0;
 978     }
 979
 980     extern (C++) const(char)* toChars() const
 981     {
 982         return toString().ptr;
 983     }
 984
 985     /*********************************
 986      * Returns:
 987      *  a zero-terminated string representation of the token,
 988      *  sometimes reusing a static buffer, sometimes leaking memory
 989      */
 990     extern (D) const(char)[] toString() const
 991     {
 992         const bufflen = 3 + 3 * floatvalue.sizeof + 1;
 993         __gshared char[bufflen + 2] buffer;     // extra 2 for suffixes
 994         char* p = &buffer[0];
 995         switch (value)
 996         {
 997         case TOK.int32Literal:
 998             const length = snprintf(p, bufflen, "%d", cast(int)intvalue);
 999             return p[0 .. length];
1000
1001         case TOK.uns32Literal:
1002         case TOK.wchar_tLiteral:
1003             const length = snprintf(p, bufflen, "%uU", cast(uint)unsvalue);
1004             return p[0 .. length];
1005
1006         case TOK.wcharLiteral:
1007         case TOK.dcharLiteral:
1008         case TOK.charLiteral:
1009             OutBuffer buf;
1010             buf.writeSingleCharLiteral(cast(dchar) intvalue);
1011             return buf.extractSlice(true);
1012
1013         case TOK.int64Literal:
1014             const length = snprintf(p, bufflen, "%lldL", cast(long)intvalue);
1015             return p[0 .. length];
1016
1017         case TOK.uns64Literal:
1018             const length = snprintf(p, bufflen, "%lluUL", cast(ulong)unsvalue);
1019             return p[0 .. length];
1020
1021         case TOK.float32Literal:
1022             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1023             p[length] = 'f';
1024             p[length + 1] = 0;
1025             return p[0 .. length + 1];
1026
1027         case TOK.float64Literal:
1028             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1029             return p[0 .. length];
1030
1031         case TOK.float80Literal:
1032             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1033             p[length] = 'L';
1034             p[length + 1] = 0;
1035             return p[0 .. length + 1];
1036
1037         case TOK.imaginary32Literal:
1038             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1039             p[length    ] = 'f';
1040             p[length + 1] = 'i';
1041             p[length + 2] = 0;
1042             return p[0 .. length + 2];
1043
1044         case TOK.imaginary64Literal:
1045             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1046             p[length] = 'i';
1047             p[length + 1] = 0;
1048             return p[0 .. length + 1];
1049
1050         case TOK.imaginary80Literal:
1051             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1052             p[length    ] = 'L';
1053             p[length + 1] = 'i';
1054             p[length + 2] = 0;
1055             return p[0 .. length + 2];
1056
1057         case TOK.string_:
1058             OutBuffer buf;
1059             buf.writeByte('"');
1060             for (size_t i = 0; i < len;)
1061             {
1062                 dchar c;
1063                 utf_decodeChar(ustring[0 .. len], i, c);
1064                 writeCharLiteral(buf, c);
1065             }
1066             buf.writeByte('"');
1067             if (postfix)
1068                 buf.writeByte(postfix);
1069             return buf.extractSlice(true);
1070
1071         case TOK.hexadecimalString:
1072             OutBuffer buf;
1073             buf.writeByte('x');
1074             buf.writeByte('"');
1075             foreach (size_t i; 0 .. len)
1076             {
1077                 if (i)
1078                     buf.writeByte(' ');
1079                 buf.printf("%02x", ustring[i]);
1080             }
1081             buf.writeByte('"');
1082             if (postfix)
1083                 buf.writeByte(postfix);
1084             return buf.extractSlice(true);
1085
1086         case TOK.identifier:
1087         case TOK.enum_:
1088         case TOK.struct_:
1089         case TOK.import_:
1090         case TOK.wchar_:
1091         case TOK.dchar_:
1092         case TOK.bool_:
1093         case TOK.char_:
1094         case TOK.int8:
1095         case TOK.uns8:
1096         case TOK.int16:
1097         case TOK.uns16:
1098         case TOK.int32:
1099         case TOK.uns32:
1100         case TOK.int64:
1101         case TOK.uns64:
1102         case TOK.int128:
1103         case TOK.uns128:
1104         case TOK.float32:
1105         case TOK.float64:
1106         case TOK.float80:
1107         case TOK.imaginary32:
1108         case TOK.imaginary64:
1109         case TOK.imaginary80:
1110         case TOK.complex32:
1111         case TOK.complex64:
1112         case TOK.complex80:
1113         case TOK.void_:
1114             return ident.toString();
1115
1116         default:
1117             return tochars[value];
1118         }
1119     }
1120
1121     static const(char)* toChars(TOK value)
1122     {
1123         return toString(value).ptr;
1124     }
1125
1126     extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1127     {
1128         return tochars[value];
1129     }
1130 }
1131
1132 /**
1133  * Write a character, using a readable escape sequence if needed
1134  *
1135  * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property
1136  *
1137  * Params:
1138  *   buf = buffer to append character in
1139  *   c = code point to write
1140  */
1141 nothrow
1142 void writeCharLiteral(ref OutBuffer buf, dchar c)
1143 {
1144     switch (c)
1145     {
1146         case '\0':
1147             buf.writestring("\\0");
1148             break;
1149         case '\n':
1150             buf.writestring("\\n");
1151             break;
1152         case '\r':
1153             buf.writestring("\\r");
1154             break;
1155         case '\t':
1156             buf.writestring("\\t");
1157             break;
1158         case '\b':
1159             buf.writestring("\\b");
1160             break;
1161         case '\f':
1162             buf.writestring("\\f");
1163             break;
1164         case '"':
1165         case '\\':
1166             buf.writeByte('\\');
1167             goto default;
1168         default:
1169             if (c <= 0xFF)
1170             {
1171                 if (isprint(c))
1172                     buf.writeByte(c);
1173                 else
1174                     buf.printf("\\x%02x", c);
1175             }
1176             else if (c <= 0xFFFF)
1177                 buf.printf("\\u%04x", c);
1178             else
1179                 buf.printf("\\U%08x", c);
1180             break;
1181     }
1182 }
1183
1184 unittest
1185 {
1186     OutBuffer buf;
1187     foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d)
1188     {
1189         writeCharLiteral(buf, d);
1190     }
1191     assert(buf[] == `a\n\r\t\b\f\0\x11\u7233\U00017233`);
1192 }
1193
1194 /**
1195  * Write a single-quoted character literal
1196  *
1197  * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property
1198  *
1199  * Params:
1200  *   buf = buffer to append character in
1201  *   c = code point to write
1202  */
1203 nothrow
1204 void writeSingleCharLiteral(ref OutBuffer buf, dchar c)
1205 {
1206     buf.writeByte('\'');
1207     if (c == '\'')
1208         buf.writeByte('\\');
1209
1210     if (c == '"')
1211         buf.writeByte('"');
1212     else
1213         writeCharLiteral(buf, c);
1214
1215     buf.writeByte('\'');
1216 }
1217
1218 unittest
1219 {
1220     OutBuffer buf;
1221     writeSingleCharLiteral(buf, '\'');
1222     assert(buf[] == `'\''`);
1223     buf.reset();
1224     writeSingleCharLiteral(buf, '"');
1225     assert(buf[] == `'"'`);
1226     buf.reset();
1227     writeSingleCharLiteral(buf, '\n');
1228     assert(buf[] == `'\n'`);
1229 }