gcc/d/dmd/tokens.d

   1 /**
   2  * Defines lexical tokens.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
   5  *
   6  * Copyright:   Copyright (C) 1999-2023 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_tokens.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
  12  */
  13
  14 module dmd.tokens;
  15
  16 import core.stdc.ctype;
  17 import core.stdc.stdio;
  18 import core.stdc.string;
  19 import dmd.identifier;
  20 import dmd.location;
  21 import dmd.root.ctfloat;
  22 import dmd.common.outbuffer;
  23 import dmd.root.rmem;
  24 import dmd.root.utf;
  25
  26 enum TOK : ubyte
  27 {
  28     reserved,
  29
  30     // Other
  31     leftParenthesis,
  32     rightParenthesis,
  33     leftBracket,
  34     rightBracket,
  35     leftCurly,
  36     rightCurly,
  37     colon,
  38     semicolon,
  39     dotDotDot,
  40     endOfFile,
  41     cast_,
  42     null_,
  43     assert_,
  44     true_,
  45     false_,
  46     throw_,
  47     new_,
  48     delete_,
  49     variable,
  50     slice,
  51     version_,
  52     module_,
  53     dollar,
  54     template_,
  55     typeof_,
  56     pragma_,
  57     typeid_,
  58     comment,
  59
  60     // Operators
  61     lessThan,
  62     greaterThan,
  63     lessOrEqual,
  64     greaterOrEqual,
  65     equal,
  66     notEqual,
  67     identity,
  68     notIdentity,
  69     is_,
  70
  71     leftShift,
  72     rightShift,
  73     leftShiftAssign,
  74     rightShiftAssign,
  75     unsignedRightShift,
  76     unsignedRightShiftAssign,
  77     concatenateAssign, // ~=
  78     add,
  79     min,
  80     addAssign,
  81     minAssign,
  82     mul,
  83     div,
  84     mod,
  85     mulAssign,
  86     divAssign,
  87     modAssign,
  88     and,
  89     or,
  90     xor,
  91     andAssign,
  92     orAssign,
  93     xorAssign,
  94     assign,
  95     not,
  96     tilde,
  97     plusPlus,
  98     minusMinus,
  99     dot,
 100     comma,
 101     question,
 102     andAnd,
 103     orOr,
 104
 105     // Numeric literals
 106     int32Literal,
 107     uns32Literal,
 108     int64Literal,
 109     uns64Literal,
 110     int128Literal,
 111     uns128Literal,
 112     float32Literal,
 113     float64Literal,
 114     float80Literal,
 115     imaginary32Literal,
 116     imaginary64Literal,
 117     imaginary80Literal,
 118
 119     // Char constants
 120     charLiteral,
 121     wcharLiteral,
 122     dcharLiteral,
 123
 124     // Leaf operators
 125     identifier,
 126     string_,
 127     hexadecimalString,
 128     this_,
 129     super_,
 130     error,
 131
 132     // Basic types
 133     void_,
 134     int8,
 135     uns8,
 136     int16,
 137     uns16,
 138     int32,
 139     uns32,
 140     int64,
 141     uns64,
 142     int128,
 143     uns128,
 144     float32,
 145     float64,
 146     float80,
 147     imaginary32,
 148     imaginary64,
 149     imaginary80,
 150     complex32,
 151     complex64,
 152     complex80,
 153     char_,
 154     wchar_,
 155     dchar_,
 156     bool_,
 157
 158     // Aggregates
 159     struct_,
 160     class_,
 161     interface_,
 162     union_,
 163     enum_,
 164     import_,
 165     alias_,
 166     override_,
 167     delegate_,
 168     function_,
 169     mixin_,
 170     align_,
 171     extern_,
 172     private_,
 173     protected_,
 174     public_,
 175     export_,
 176     static_,
 177     final_,
 178     const_,
 179     abstract_,
 180     debug_,
 181     deprecated_,
 182     in_,
 183     out_,
 184     inout_,
 185     lazy_,
 186     auto_,
 187     package_,
 188     immutable_,
 189
 190     // Statements
 191     if_,
 192     else_,
 193     while_,
 194     for_,
 195     do_,
 196     switch_,
 197     case_,
 198     default_,
 199     break_,
 200     continue_,
 201     with_,
 202     synchronized_,
 203     return_,
 204     goto_,
 205     try_,
 206     catch_,
 207     finally_,
 208     asm_,
 209     foreach_,
 210     foreach_reverse_,
 211     scope_,
 212     onScopeExit,
 213     onScopeFailure,
 214     onScopeSuccess,
 215
 216     // Contracts
 217     invariant_,
 218
 219     // Testing
 220     unittest_,
 221
 222     // Added after 1.0
 223     argumentTypes,
 224     ref_,
 225     macro_,
 226
 227     parameters,
 228     traits,
 229     pure_,
 230     nothrow_,
 231     gshared,
 232     line,
 233     file,
 234     fileFullPath,
 235     moduleString,   // __MODULE__
 236     functionString, // __FUNCTION__
 237     prettyFunction, // __PRETTY_FUNCTION__
 238     shared_,
 239     at,
 240     pow,
 241     powAssign,
 242     goesTo,
 243     vector,
 244     pound,
 245
 246     arrow,      // ->
 247     colonColon, // ::
 248     wchar_tLiteral,
 249     endOfLine,  // \n, \r, \u2028, \u2029
 250     whitespace,
 251
 252     // C only keywords
 253     inline,
 254     register,
 255     restrict,
 256     signed,
 257     sizeof_,
 258     typedef_,
 259     unsigned,
 260     volatile,
 261     _Alignas,
 262     _Alignof,
 263     _Atomic,
 264     _Bool,
 265     _Complex,
 266     _Generic,
 267     _Imaginary,
 268     _Noreturn,
 269     _Static_assert,
 270     _Thread_local,
 271
 272     // C only extended keywords
 273     _assert,
 274     _import,
 275     __cdecl,
 276     __declspec,
 277     __stdcall,
 278     __thread,
 279     __pragma,
 280     __int128,
 281     __attribute__,
 282 }
 283
 284 /// Expression nodes
 285 enum EXP : ubyte
 286 {
 287     reserved,
 288
 289     // Other
 290     negate,
 291     cast_,
 292     null_,
 293     assert_,
 294     array,
 295     call,
 296     address,
 297     type,
 298     throw_,
 299     new_,
 300     delete_,
 301     star,
 302     symbolOffset,
 303     variable,
 304     dotVariable,
 305     dotIdentifier,
 306     dotTemplateInstance,
 307     dotType,
 308     slice,
 309     arrayLength,
 310     dollar,
 311     template_,
 312     dotTemplateDeclaration,
 313     declaration,
 314     dSymbol,
 315     typeid_,
 316     uadd,
 317     remove,
 318     newAnonymousClass,
 319     arrayLiteral,
 320     assocArrayLiteral,
 321     structLiteral,
 322     classReference,
 323     thrownException,
 324     delegatePointer,
 325     delegateFunctionPointer,
 326
 327     // Operators
 328     lessThan,
 329     greaterThan,
 330     lessOrEqual,
 331     greaterOrEqual,
 332     equal,
 333     notEqual,
 334     identity,
 335     notIdentity,
 336     index,
 337     is_,
 338
 339     leftShift,
 340     rightShift,
 341     leftShiftAssign,
 342     rightShiftAssign,
 343     unsignedRightShift,
 344     unsignedRightShiftAssign,
 345     concatenate,
 346     concatenateAssign, // ~=
 347     concatenateElemAssign,
 348     concatenateDcharAssign,
 349     add,
 350     min,
 351     addAssign,
 352     minAssign,
 353     mul,
 354     div,
 355     mod,
 356     mulAssign,
 357     divAssign,
 358     modAssign,
 359     and,
 360     or,
 361     xor,
 362     andAssign,
 363     orAssign,
 364     xorAssign,
 365     assign,
 366     not,
 367     tilde,
 368     plusPlus,
 369     minusMinus,
 370     construct,
 371     blit,
 372     dot,
 373     comma,
 374     question,
 375     andAnd,
 376     orOr,
 377     prePlusPlus,
 378     preMinusMinus,
 379
 380     // Leaf operators
 381     identifier,
 382     string_,
 383     this_,
 384     super_,
 385     halt,
 386     tuple,
 387     error,
 388
 389     // Basic types
 390     void_,
 391     int64,
 392     float64,
 393     complex80,
 394     import_,
 395     delegate_,
 396     function_,
 397     mixin_,
 398     in_,
 399     break_,
 400     continue_,
 401     goto_,
 402     scope_,
 403
 404     traits,
 405     overloadSet,
 406     line,
 407     file,
 408     fileFullPath,
 409     moduleString,   // __MODULE__
 410     functionString, // __FUNCTION__
 411     prettyFunction, // __PRETTY_FUNCTION__
 412     pow,
 413     powAssign,
 414     vector,
 415
 416     voidExpression,
 417     cantExpression,
 418     showCtfeContext,
 419     objcClassReference,
 420     vectorArray,
 421     compoundLiteral, // ( type-name ) { initializer-list }
 422     _Generic,
 423     interval,
 424
 425     loweredAssignExp,
 426 }
 427
 428 enum FirstCKeyword = TOK.inline;
 429
 430 // Assert that all token enum members have consecutive values and
 431 // that none of them overlap
 432 static assert(() {
 433     foreach (idx, enumName; __traits(allMembers, TOK)) {
 434        static if (idx != __traits(getMember, TOK, enumName)) {
 435            pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
 436            static assert(0);
 437        }
 438     }
 439     return true;
 440 }());
 441
 442 /****************************************
 443  */
 444
 445 private immutable TOK[] keywords =
 446 [
 447     TOK.this_,
 448     TOK.super_,
 449     TOK.assert_,
 450     TOK.null_,
 451     TOK.true_,
 452     TOK.false_,
 453     TOK.cast_,
 454     TOK.new_,
 455     TOK.delete_,
 456     TOK.throw_,
 457     TOK.module_,
 458     TOK.pragma_,
 459     TOK.typeof_,
 460     TOK.typeid_,
 461     TOK.template_,
 462     TOK.void_,
 463     TOK.int8,
 464     TOK.uns8,
 465     TOK.int16,
 466     TOK.uns16,
 467     TOK.int32,
 468     TOK.uns32,
 469     TOK.int64,
 470     TOK.uns64,
 471     TOK.int128,
 472     TOK.uns128,
 473     TOK.float32,
 474     TOK.float64,
 475     TOK.float80,
 476     TOK.bool_,
 477     TOK.char_,
 478     TOK.wchar_,
 479     TOK.dchar_,
 480     TOK.imaginary32,
 481     TOK.imaginary64,
 482     TOK.imaginary80,
 483     TOK.complex32,
 484     TOK.complex64,
 485     TOK.complex80,
 486     TOK.delegate_,
 487     TOK.function_,
 488     TOK.is_,
 489     TOK.if_,
 490     TOK.else_,
 491     TOK.while_,
 492     TOK.for_,
 493     TOK.do_,
 494     TOK.switch_,
 495     TOK.case_,
 496     TOK.default_,
 497     TOK.break_,
 498     TOK.continue_,
 499     TOK.synchronized_,
 500     TOK.return_,
 501     TOK.goto_,
 502     TOK.try_,
 503     TOK.catch_,
 504     TOK.finally_,
 505     TOK.with_,
 506     TOK.asm_,
 507     TOK.foreach_,
 508     TOK.foreach_reverse_,
 509     TOK.scope_,
 510     TOK.struct_,
 511     TOK.class_,
 512     TOK.interface_,
 513     TOK.union_,
 514     TOK.enum_,
 515     TOK.import_,
 516     TOK.mixin_,
 517     TOK.static_,
 518     TOK.final_,
 519     TOK.const_,
 520     TOK.alias_,
 521     TOK.override_,
 522     TOK.abstract_,
 523     TOK.debug_,
 524     TOK.deprecated_,
 525     TOK.in_,
 526     TOK.out_,
 527     TOK.inout_,
 528     TOK.lazy_,
 529     TOK.auto_,
 530     TOK.align_,
 531     TOK.extern_,
 532     TOK.private_,
 533     TOK.package_,
 534     TOK.protected_,
 535     TOK.public_,
 536     TOK.export_,
 537     TOK.invariant_,
 538     TOK.unittest_,
 539     TOK.version_,
 540     TOK.argumentTypes,
 541     TOK.parameters,
 542     TOK.ref_,
 543     TOK.macro_,
 544     TOK.pure_,
 545     TOK.nothrow_,
 546     TOK.gshared,
 547     TOK.traits,
 548     TOK.vector,
 549     TOK.file,
 550     TOK.fileFullPath,
 551     TOK.line,
 552     TOK.moduleString,
 553     TOK.functionString,
 554     TOK.prettyFunction,
 555     TOK.shared_,
 556     TOK.immutable_,
 557
 558     // C only keywords
 559     TOK.inline,
 560     TOK.register,
 561     TOK.restrict,
 562     TOK.signed,
 563     TOK.sizeof_,
 564     TOK.typedef_,
 565     TOK.unsigned,
 566     TOK.volatile,
 567     TOK._Alignas,
 568     TOK._Alignof,
 569     TOK._Atomic,
 570     TOK._Bool,
 571     TOK._Complex,
 572     TOK._Generic,
 573     TOK._Imaginary,
 574     TOK._Noreturn,
 575     TOK._Static_assert,
 576     TOK._Thread_local,
 577
 578     // C only extended keywords
 579     TOK._assert,
 580     TOK._import,
 581     TOK.__cdecl,
 582     TOK.__declspec,
 583     TOK.__stdcall,
 584     TOK.__thread,
 585     TOK.__pragma,
 586     TOK.__int128,
 587     TOK.__attribute__,
 588 ];
 589
 590 // Initialize the identifier pool
 591 shared static this() nothrow
 592 {
 593     Identifier.initTable();
 594     foreach (kw; keywords)
 595     {
 596         //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr);
 597         Identifier.idPool(Token.tochars[kw], kw);
 598     }
 599 }
 600
 601 /************************************
 602  * This is used to pick the C keywords out of the tokens.
 603  * If it's not a C keyword, then it's an identifier.
 604  */
 605 static immutable TOK[TOK.max + 1] Ckeywords =
 606 () {
 607     with (TOK)
 608     {
 609         TOK[TOK.max + 1] tab = identifier;  // default to identifier
 610         enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
 611                        enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
 612                        restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
 613                        union_, unsigned, void_, volatile, while_, asm_, typeof_,
 614                        _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
 615                        _Static_assert, _Thread_local,
 616                        _import, __cdecl, __declspec, __stdcall, __thread, __pragma, __int128, __attribute__,
 617                        _assert ];
 618
 619         foreach (kw; Ckwds)
 620             tab[kw] = cast(TOK) kw;
 621
 622         return tab;
 623     }
 624 } ();
 625
 626
 627 /***********************************************************
 628  */
 629 extern (C++) struct Token
 630 {
 631     Token* next;
 632     Loc loc;
 633     const(char)* ptr; // pointer to first character of this token within buffer
 634     TOK value;
 635     const(char)[] blockComment; // doc comment string prior to this token
 636     const(char)[] lineComment; // doc comment for previous token
 637
 638     union
 639     {
 640         // Integers
 641         long intvalue;
 642         ulong unsvalue;
 643         // Floats
 644         real_t floatvalue;
 645
 646         struct
 647         {
 648             const(char)* ustring; // UTF8 string
 649             uint len;
 650             ubyte postfix; // 'c', 'w', 'd'
 651         }
 652
 653         Identifier ident;
 654     }
 655
 656     extern (D) private static immutable string[TOK.max + 1] tochars =
 657     [
 658         // Keywords
 659         TOK.this_: "this",
 660         TOK.super_: "super",
 661         TOK.assert_: "assert",
 662         TOK.null_: "null",
 663         TOK.true_: "true",
 664         TOK.false_: "false",
 665         TOK.cast_: "cast",
 666         TOK.new_: "new",
 667         TOK.delete_: "delete",
 668         TOK.throw_: "throw",
 669         TOK.module_: "module",
 670         TOK.pragma_: "pragma",
 671         TOK.typeof_: "typeof",
 672         TOK.typeid_: "typeid",
 673         TOK.template_: "template",
 674         TOK.void_: "void",
 675         TOK.int8: "byte",
 676         TOK.uns8: "ubyte",
 677         TOK.int16: "short",
 678         TOK.uns16: "ushort",
 679         TOK.int32: "int",
 680         TOK.uns32: "uint",
 681         TOK.int64: "long",
 682         TOK.uns64: "ulong",
 683         TOK.int128: "cent",
 684         TOK.uns128: "ucent",
 685         TOK.float32: "float",
 686         TOK.float64: "double",
 687         TOK.float80: "real",
 688         TOK.bool_: "bool",
 689         TOK.char_: "char",
 690         TOK.wchar_: "wchar",
 691         TOK.dchar_: "dchar",
 692         TOK.imaginary32: "ifloat",
 693         TOK.imaginary64: "idouble",
 694         TOK.imaginary80: "ireal",
 695         TOK.complex32: "cfloat",
 696         TOK.complex64: "cdouble",
 697         TOK.complex80: "creal",
 698         TOK.delegate_: "delegate",
 699         TOK.function_: "function",
 700         TOK.is_: "is",
 701         TOK.if_: "if",
 702         TOK.else_: "else",
 703         TOK.while_: "while",
 704         TOK.for_: "for",
 705         TOK.do_: "do",
 706         TOK.switch_: "switch",
 707         TOK.case_: "case",
 708         TOK.default_: "default",
 709         TOK.break_: "break",
 710         TOK.continue_: "continue",
 711         TOK.synchronized_: "synchronized",
 712         TOK.return_: "return",
 713         TOK.goto_: "goto",
 714         TOK.try_: "try",
 715         TOK.catch_: "catch",
 716         TOK.finally_: "finally",
 717         TOK.with_: "with",
 718         TOK.asm_: "asm",
 719         TOK.foreach_: "foreach",
 720         TOK.foreach_reverse_: "foreach_reverse",
 721         TOK.scope_: "scope",
 722         TOK.struct_: "struct",
 723         TOK.class_: "class",
 724         TOK.interface_: "interface",
 725         TOK.union_: "union",
 726         TOK.enum_: "enum",
 727         TOK.import_: "import",
 728         TOK.mixin_: "mixin",
 729         TOK.static_: "static",
 730         TOK.final_: "final",
 731         TOK.const_: "const",
 732         TOK.alias_: "alias",
 733         TOK.override_: "override",
 734         TOK.abstract_: "abstract",
 735         TOK.debug_: "debug",
 736         TOK.deprecated_: "deprecated",
 737         TOK.in_: "in",
 738         TOK.out_: "out",
 739         TOK.inout_: "inout",
 740         TOK.lazy_: "lazy",
 741         TOK.auto_: "auto",
 742         TOK.align_: "align",
 743         TOK.extern_: "extern",
 744         TOK.private_: "private",
 745         TOK.package_: "package",
 746         TOK.protected_: "protected",
 747         TOK.public_: "public",
 748         TOK.export_: "export",
 749         TOK.invariant_: "invariant",
 750         TOK.unittest_: "unittest",
 751         TOK.version_: "version",
 752         TOK.argumentTypes: "__argTypes",
 753         TOK.parameters: "__parameters",
 754         TOK.ref_: "ref",
 755         TOK.macro_: "macro",
 756         TOK.pure_: "pure",
 757         TOK.nothrow_: "nothrow",
 758         TOK.gshared: "__gshared",
 759         TOK.traits: "__traits",
 760         TOK.vector: "__vector",
 761         TOK.file: "__FILE__",
 762         TOK.fileFullPath: "__FILE_FULL_PATH__",
 763         TOK.line: "__LINE__",
 764         TOK.moduleString: "__MODULE__",
 765         TOK.functionString: "__FUNCTION__",
 766         TOK.prettyFunction: "__PRETTY_FUNCTION__",
 767         TOK.shared_: "shared",
 768         TOK.immutable_: "immutable",
 769
 770         TOK.endOfFile: "End of File",
 771         TOK.leftCurly: "{",
 772         TOK.rightCurly: "}",
 773         TOK.leftParenthesis: "(",
 774         TOK.rightParenthesis: ")",
 775         TOK.leftBracket: "[",
 776         TOK.rightBracket: "]",
 777         TOK.semicolon: ";",
 778         TOK.colon: ":",
 779         TOK.comma: ",",
 780         TOK.dot: ".",
 781         TOK.xor: "^",
 782         TOK.xorAssign: "^=",
 783         TOK.assign: "=",
 784         TOK.lessThan: "<",
 785         TOK.greaterThan: ">",
 786         TOK.lessOrEqual: "<=",
 787         TOK.greaterOrEqual: ">=",
 788         TOK.equal: "==",
 789         TOK.notEqual: "!=",
 790         TOK.not: "!",
 791         TOK.leftShift: "<<",
 792         TOK.rightShift: ">>",
 793         TOK.unsignedRightShift: ">>>",
 794         TOK.add: "+",
 795         TOK.min: "-",
 796         TOK.mul: "*",
 797         TOK.div: "/",
 798         TOK.mod: "%",
 799         TOK.slice: "..",
 800         TOK.dotDotDot: "...",
 801         TOK.and: "&",
 802         TOK.andAnd: "&&",
 803         TOK.or: "|",
 804         TOK.orOr: "||",
 805         TOK.tilde: "~",
 806         TOK.dollar: "$",
 807         TOK.plusPlus: "++",
 808         TOK.minusMinus: "--",
 809         TOK.question: "?",
 810         TOK.variable: "var",
 811         TOK.addAssign: "+=",
 812         TOK.minAssign: "-=",
 813         TOK.mulAssign: "*=",
 814         TOK.divAssign: "/=",
 815         TOK.modAssign: "%=",
 816         TOK.leftShiftAssign: "<<=",
 817         TOK.rightShiftAssign: ">>=",
 818         TOK.unsignedRightShiftAssign: ">>>=",
 819         TOK.andAssign: "&=",
 820         TOK.orAssign: "|=",
 821         TOK.concatenateAssign: "~=",
 822         TOK.identity: "is",
 823         TOK.notIdentity: "!is",
 824         TOK.identifier: "identifier",
 825         TOK.at: "@",
 826         TOK.pow: "^^",
 827         TOK.powAssign: "^^=",
 828         TOK.goesTo: "=>",
 829         TOK.pound: "#",
 830         TOK.arrow: "->",
 831         TOK.colonColon: "::",
 832
 833         // For debugging
 834         TOK.error: "error",
 835         TOK.string_: "string",
 836         TOK.onScopeExit: "scope(exit)",
 837         TOK.onScopeSuccess: "scope(success)",
 838         TOK.onScopeFailure: "scope(failure)",
 839
 840         // Finish up
 841         TOK.reserved: "reserved",
 842         TOK.comment: "comment",
 843         TOK.int32Literal: "int32v",
 844         TOK.uns32Literal: "uns32v",
 845         TOK.int64Literal: "int64v",
 846         TOK.uns64Literal: "uns64v",
 847         TOK.int128Literal: "int128v",
 848         TOK.uns128Literal: "uns128v",
 849         TOK.float32Literal: "float32v",
 850         TOK.float64Literal: "float64v",
 851         TOK.float80Literal: "float80v",
 852         TOK.imaginary32Literal: "imaginary32v",
 853         TOK.imaginary64Literal: "imaginary64v",
 854         TOK.imaginary80Literal: "imaginary80v",
 855         TOK.charLiteral: "charv",
 856         TOK.wcharLiteral: "wcharv",
 857         TOK.dcharLiteral: "dcharv",
 858         TOK.wchar_tLiteral: "wchar_tv",
 859         TOK.hexadecimalString: "xstring",
 860         TOK.endOfLine: "\\n",
 861         TOK.whitespace: "whitespace",
 862
 863         // C only keywords
 864         TOK.inline    : "inline",
 865         TOK.register  : "register",
 866         TOK.restrict  : "restrict",
 867         TOK.signed    : "signed",
 868         TOK.sizeof_   : "sizeof",
 869         TOK.typedef_  : "typedef",
 870         TOK.unsigned  : "unsigned",
 871         TOK.volatile  : "volatile",
 872         TOK._Alignas  : "_Alignas",
 873         TOK._Alignof  : "_Alignof",
 874         TOK._Atomic   : "_Atomic",
 875         TOK._Bool     : "_Bool",
 876         TOK._Complex  : "_Complex",
 877         TOK._Generic  : "_Generic",
 878         TOK._Imaginary: "_Imaginary",
 879         TOK._Noreturn : "_Noreturn",
 880         TOK._Static_assert : "_Static_assert",
 881         TOK._Thread_local  : "_Thread_local",
 882
 883         // C only extended keywords
 884         TOK._assert       : "__check",
 885         TOK._import       : "__import",
 886         TOK.__cdecl        : "__cdecl",
 887         TOK.__declspec     : "__declspec",
 888         TOK.__stdcall      : "__stdcall",
 889         TOK.__thread       : "__thread",
 890         TOK.__pragma       : "__pragma",
 891         TOK.__int128       : "__int128",
 892         TOK.__attribute__  : "__attribute__",
 893     ];
 894
 895     static assert(() {
 896         foreach (s; tochars)
 897             assert(s.length);
 898         return true;
 899     }());
 900
 901 nothrow:
 902
 903     int isKeyword() pure const @safe @nogc
 904     {
 905         foreach (kw; keywords)
 906         {
 907             if (kw == value)
 908                 return 1;
 909         }
 910         return 0;
 911     }
 912
 913     /****
 914      * Set to contents of ptr[0..length]
 915      * Params:
 916      *  ptr = pointer to string
 917      *  length = length of string
 918      */
 919     void setString(const(char)* ptr, size_t length)
 920     {
 921         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
 922         memcpy(s, ptr, length);
 923         s[length] = 0;
 924         ustring = s;
 925         len = cast(uint)length;
 926         postfix = 0;
 927     }
 928
 929     /****
 930      * Set to contents of buf
 931      * Params:
 932      *  buf = string (not zero terminated)
 933      */
 934     void setString(const ref OutBuffer buf)
 935     {
 936         setString(cast(const(char)*)buf[].ptr, buf.length);
 937     }
 938
 939     /****
 940      * Set to empty string
 941      */
 942     void setString()
 943     {
 944         ustring = "";
 945         len = 0;
 946         postfix = 0;
 947     }
 948
 949     extern (C++) const(char)* toChars() const
 950     {
 951         const bufflen = 3 + 3 * floatvalue.sizeof + 1;
 952         __gshared char[bufflen] buffer;
 953         const(char)* p = &buffer[0];
 954         switch (value)
 955         {
 956         case TOK.int32Literal:
 957             snprintf(&buffer[0], bufflen, "%d", cast(int)intvalue);
 958             break;
 959         case TOK.uns32Literal:
 960         case TOK.wchar_tLiteral:
 961             snprintf(&buffer[0], bufflen, "%uU", cast(uint)unsvalue);
 962             break;
 963         case TOK.wcharLiteral:
 964         case TOK.dcharLiteral:
 965         case TOK.charLiteral:
 966             {
 967                 OutBuffer buf;
 968                 buf.writeSingleCharLiteral(cast(dchar) intvalue);
 969                 buf.writeByte('\0');
 970                 p = buf.extractChars();
 971             }
 972             break;
 973         case TOK.int64Literal:
 974             snprintf(&buffer[0], bufflen, "%lldL", cast(long)intvalue);
 975             break;
 976         case TOK.uns64Literal:
 977             snprintf(&buffer[0], bufflen, "%lluUL", cast(ulong)unsvalue);
 978             break;
 979         case TOK.float32Literal:
 980             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
 981             strcat(&buffer[0], "f");
 982             break;
 983         case TOK.float64Literal:
 984             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
 985             break;
 986         case TOK.float80Literal:
 987             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
 988             strcat(&buffer[0], "L");
 989             break;
 990         case TOK.imaginary32Literal:
 991             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
 992             strcat(&buffer[0], "fi");
 993             break;
 994         case TOK.imaginary64Literal:
 995             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
 996             strcat(&buffer[0], "i");
 997             break;
 998         case TOK.imaginary80Literal:
 999             CTFloat.sprint(&buffer[0], bufflen, 'g', floatvalue);
1000             strcat(&buffer[0], "Li");
1001             break;
1002         case TOK.string_:
1003             {
1004                 OutBuffer buf;
1005                 buf.writeByte('"');
1006                 for (size_t i = 0; i < len;)
1007                 {
1008                     dchar c;
1009                     utf_decodeChar(ustring[0 .. len], i, c);
1010                     writeCharLiteral(buf, c);
1011                 }
1012                 buf.writeByte('"');
1013                 if (postfix)
1014                     buf.writeByte(postfix);
1015                 buf.writeByte(0);
1016                 p = buf.extractChars();
1017             }
1018             break;
1019         case TOK.hexadecimalString:
1020             {
1021                 OutBuffer buf;
1022                 buf.writeByte('x');
1023                 buf.writeByte('"');
1024                 foreach (size_t i; 0 .. len)
1025                 {
1026                     if (i)
1027                         buf.writeByte(' ');
1028                     buf.printf("%02x", ustring[i]);
1029                 }
1030                 buf.writeByte('"');
1031                 if (postfix)
1032                     buf.writeByte(postfix);
1033                 buf.writeByte(0);
1034                 p = buf.extractData();
1035                 break;
1036             }
1037         case TOK.identifier:
1038         case TOK.enum_:
1039         case TOK.struct_:
1040         case TOK.import_:
1041         case TOK.wchar_:
1042         case TOK.dchar_:
1043         case TOK.bool_:
1044         case TOK.char_:
1045         case TOK.int8:
1046         case TOK.uns8:
1047         case TOK.int16:
1048         case TOK.uns16:
1049         case TOK.int32:
1050         case TOK.uns32:
1051         case TOK.int64:
1052         case TOK.uns64:
1053         case TOK.int128:
1054         case TOK.uns128:
1055         case TOK.float32:
1056         case TOK.float64:
1057         case TOK.float80:
1058         case TOK.imaginary32:
1059         case TOK.imaginary64:
1060         case TOK.imaginary80:
1061         case TOK.complex32:
1062         case TOK.complex64:
1063         case TOK.complex80:
1064         case TOK.void_:
1065             p = ident.toChars();
1066             break;
1067         default:
1068             p = toChars(value);
1069             break;
1070         }
1071         return p;
1072     }
1073
1074     static const(char)* toChars(TOK value)
1075     {
1076         return toString(value).ptr;
1077     }
1078
1079     extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1080     {
1081         return tochars[value];
1082     }
1083 }
1084
1085 /**
1086  * Write a character, using a readable escape sequence if needed
1087  *
1088  * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property
1089  *
1090  * Params:
1091  *   buf = buffer to append character in
1092  *   c = code point to write
1093  */
1094 nothrow
1095 void writeCharLiteral(ref OutBuffer buf, dchar c)
1096 {
1097     switch (c)
1098     {
1099         case '\0':
1100             buf.writestring("\\0");
1101             break;
1102         case '\n':
1103             buf.writestring("\\n");
1104             break;
1105         case '\r':
1106             buf.writestring("\\r");
1107             break;
1108         case '\t':
1109             buf.writestring("\\t");
1110             break;
1111         case '\b':
1112             buf.writestring("\\b");
1113             break;
1114         case '\f':
1115             buf.writestring("\\f");
1116             break;
1117         case '"':
1118         case '\\':
1119             buf.writeByte('\\');
1120             goto default;
1121         default:
1122             if (c <= 0xFF)
1123             {
1124                 if (isprint(c))
1125                     buf.writeByte(c);
1126                 else
1127                     buf.printf("\\x%02x", c);
1128             }
1129             else if (c <= 0xFFFF)
1130                 buf.printf("\\u%04x", c);
1131             else
1132                 buf.printf("\\U%08x", c);
1133             break;
1134     }
1135 }
1136
1137 unittest
1138 {
1139     OutBuffer buf;
1140     foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d)
1141     {
1142         writeCharLiteral(buf, d);
1143     }
1144     assert(buf[] == `a\n\r\t\b\f\0\x11\u7233\U00017233`);
1145 }
1146
1147 /**
1148  * Write a single-quoted character literal
1149  *
1150  * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property
1151  *
1152  * Params:
1153  *   buf = buffer to append character in
1154  *   c = code point to write
1155  */
1156 nothrow
1157 void writeSingleCharLiteral(ref OutBuffer buf, dchar c)
1158 {
1159     buf.writeByte('\'');
1160     if (c == '\'')
1161         buf.writeByte('\\');
1162
1163     if (c == '"')
1164         buf.writeByte('"');
1165     else
1166         writeCharLiteral(buf, c);
1167
1168     buf.writeByte('\'');
1169 }
1170
1171 unittest
1172 {
1173     OutBuffer buf;
1174     writeSingleCharLiteral(buf, '\'');
1175     assert(buf[] == `'\''`);
1176     buf.reset();
1177     writeSingleCharLiteral(buf, '"');
1178     assert(buf[] == `'"'`);
1179     buf.reset();
1180     writeSingleCharLiteral(buf, '\n');
1181     assert(buf[] == `'\n'`);
1182 }