gcc/d/dmd/tokens.d

   1 /**
   2  * Defines lexical tokens.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
   5  *
   6  * Copyright:   Copyright (C) 1999-2024 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_tokens.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
  12  */
  13
  14 module dmd.tokens;
  15
  16 import core.stdc.ctype;
  17 import core.stdc.stdio;
  18 import core.stdc.string;
  19 import dmd.identifier;
  20 import dmd.location;
  21 import dmd.root.ctfloat;
  22 import dmd.common.outbuffer;
  23 import dmd.root.rmem;
  24 import dmd.root.utf;
  25
  26 enum TOK : ubyte
  27 {
  28     reserved,
  29
  30     // Other
  31     leftParenthesis,
  32     rightParenthesis,
  33     leftBracket,
  34     rightBracket,
  35     leftCurly,
  36     rightCurly,
  37     colon,
  38     semicolon,
  39     dotDotDot,
  40     endOfFile,
  41     cast_,
  42     null_,
  43     assert_,
  44     true_,
  45     false_,
  46     throw_,
  47     new_,
  48     delete_,
  49     variable,
  50     slice,
  51     version_,
  52     module_,
  53     dollar,
  54     template_,
  55     typeof_,
  56     pragma_,
  57     typeid_,
  58     comment,
  59
  60     // Operators
  61     lessThan,
  62     greaterThan,
  63     lessOrEqual,
  64     greaterOrEqual,
  65     equal,
  66     notEqual,
  67     identity,
  68     notIdentity,
  69     is_,
  70
  71     leftShift,
  72     rightShift,
  73     leftShiftAssign,
  74     rightShiftAssign,
  75     unsignedRightShift,
  76     unsignedRightShiftAssign,
  77     concatenateAssign, // ~=
  78     add,
  79     min,
  80     addAssign,
  81     minAssign,
  82     mul,
  83     div,
  84     mod,
  85     mulAssign,
  86     divAssign,
  87     modAssign,
  88     and,
  89     or,
  90     xor,
  91     andAssign,
  92     orAssign,
  93     xorAssign,
  94     assign,
  95     not,
  96     tilde,
  97     plusPlus,
  98     minusMinus,
  99     dot,
 100     comma,
 101     question,
 102     andAnd,
 103     orOr,
 104
 105     // Numeric literals
 106     int32Literal,
 107     uns32Literal,
 108     int64Literal,
 109     uns64Literal,
 110     int128Literal,
 111     uns128Literal,
 112     float32Literal,
 113     float64Literal,
 114     float80Literal,
 115     imaginary32Literal,
 116     imaginary64Literal,
 117     imaginary80Literal,
 118
 119     // Char constants
 120     charLiteral,
 121     wcharLiteral,
 122     dcharLiteral,
 123
 124     // Leaf operators
 125     identifier,
 126     string_,
 127     hexadecimalString,
 128     this_,
 129     super_,
 130     error,
 131
 132     // Basic types
 133     void_,
 134     int8,
 135     uns8,
 136     int16,
 137     uns16,
 138     int32,
 139     uns32,
 140     int64,
 141     uns64,
 142     int128,
 143     uns128,
 144     float32,
 145     float64,
 146     float80,
 147     imaginary32,
 148     imaginary64,
 149     imaginary80,
 150     complex32,
 151     complex64,
 152     complex80,
 153     char_,
 154     wchar_,
 155     dchar_,
 156     bool_,
 157
 158     // Aggregates
 159     struct_,
 160     class_,
 161     interface_,
 162     union_,
 163     enum_,
 164     import_,
 165     alias_,
 166     override_,
 167     delegate_,
 168     function_,
 169     mixin_,
 170     align_,
 171     extern_,
 172     private_,
 173     protected_,
 174     public_,
 175     export_,
 176     static_,
 177     final_,
 178     const_,
 179     abstract_,
 180     debug_,
 181     deprecated_,
 182     in_,
 183     out_,
 184     inout_,
 185     lazy_,
 186     auto_,
 187     package_,
 188     immutable_,
 189
 190     // Statements
 191     if_,
 192     else_,
 193     while_,
 194     for_,
 195     do_,
 196     switch_,
 197     case_,
 198     default_,
 199     break_,
 200     continue_,
 201     with_,
 202     synchronized_,
 203     return_,
 204     goto_,
 205     try_,
 206     catch_,
 207     finally_,
 208     asm_,
 209     foreach_,
 210     foreach_reverse_,
 211     scope_,
 212     onScopeExit,
 213     onScopeFailure,
 214     onScopeSuccess,
 215
 216     // Contracts
 217     invariant_,
 218
 219     // Testing
 220     unittest_,
 221
 222     // Added after 1.0
 223     argumentTypes,
 224     ref_,
 225     macro_,
 226
 227     parameters,
 228     traits,
 229     pure_,
 230     nothrow_,
 231     gshared,
 232     line,
 233     file,
 234     fileFullPath,
 235     moduleString,   // __MODULE__
 236     functionString, // __FUNCTION__
 237     prettyFunction, // __PRETTY_FUNCTION__
 238     shared_,
 239     at,
 240     pow,
 241     powAssign,
 242     goesTo,
 243     vector,
 244     pound,
 245
 246     arrow,      // ->
 247     colonColon, // ::
 248     wchar_tLiteral,
 249     endOfLine,  // \n, \r, \u2028, \u2029
 250     whitespace,
 251
 252     // C only keywords
 253     inline,
 254     register,
 255     restrict,
 256     signed,
 257     sizeof_,
 258     typedef_,
 259     unsigned,
 260     volatile,
 261     _Alignas,
 262     _Alignof,
 263     _Atomic,
 264     _Bool,
 265     _Complex,
 266     _Generic,
 267     _Imaginary,
 268     _Noreturn,
 269     _Static_assert,
 270     _Thread_local,
 271
 272     // C only extended keywords
 273     _assert,
 274     _import,
 275     __cdecl,
 276     __declspec,
 277     __stdcall,
 278     __thread,
 279     __pragma,
 280     __int128,
 281     __attribute__,
 282 }
 283
 284 /// Expression nodes
 285 enum EXP : ubyte
 286 {
 287     reserved,
 288
 289     // Other
 290     negate,
 291     cast_,
 292     null_,
 293     assert_,
 294     array,
 295     call,
 296     address,
 297     type,
 298     throw_,
 299     new_,
 300     delete_,
 301     star,
 302     symbolOffset,
 303     variable,
 304     dotVariable,
 305     dotIdentifier,
 306     dotTemplateInstance,
 307     dotType,
 308     slice,
 309     arrayLength,
 310     dollar,
 311     template_,
 312     dotTemplateDeclaration,
 313     declaration,
 314     dSymbol,
 315     typeid_,
 316     uadd,
 317     remove,
 318     newAnonymousClass,
 319     arrayLiteral,
 320     assocArrayLiteral,
 321     structLiteral,
 322     classReference,
 323     thrownException,
 324     delegatePointer,
 325     delegateFunctionPointer,
 326
 327     // Operators
 328     lessThan,
 329     greaterThan,
 330     lessOrEqual,
 331     greaterOrEqual,
 332     equal,
 333     notEqual,
 334     identity,
 335     notIdentity,
 336     index,
 337     is_,
 338
 339     leftShift,
 340     rightShift,
 341     leftShiftAssign,
 342     rightShiftAssign,
 343     unsignedRightShift,
 344     unsignedRightShiftAssign,
 345     concatenate,
 346     concatenateAssign, // ~=
 347     concatenateElemAssign,
 348     concatenateDcharAssign,
 349     add,
 350     min,
 351     addAssign,
 352     minAssign,
 353     mul,
 354     div,
 355     mod,
 356     mulAssign,
 357     divAssign,
 358     modAssign,
 359     and,
 360     or,
 361     xor,
 362     andAssign,
 363     orAssign,
 364     xorAssign,
 365     assign,
 366     not,
 367     tilde,
 368     plusPlus,
 369     minusMinus,
 370     construct,
 371     blit,
 372     dot,
 373     comma,
 374     question,
 375     andAnd,
 376     orOr,
 377     prePlusPlus,
 378     preMinusMinus,
 379
 380     // Leaf operators
 381     identifier,
 382     string_,
 383     this_,
 384     super_,
 385     halt,
 386     tuple,
 387     error,
 388
 389     // Basic types
 390     void_,
 391     int64,
 392     float64,
 393     complex80,
 394     import_,
 395     delegate_,
 396     function_,
 397     mixin_,
 398     in_,
 399     break_,
 400     continue_,
 401     goto_,
 402     scope_,
 403
 404     traits,
 405     overloadSet,
 406     line,
 407     file,
 408     fileFullPath,
 409     moduleString,   // __MODULE__
 410     functionString, // __FUNCTION__
 411     prettyFunction, // __PRETTY_FUNCTION__
 412     pow,
 413     powAssign,
 414     vector,
 415
 416     voidExpression,
 417     cantExpression,
 418     showCtfeContext,
 419     objcClassReference,
 420     vectorArray,
 421     compoundLiteral, // ( type-name ) { initializer-list }
 422     _Generic,
 423     interval,
 424
 425     loweredAssignExp,
 426 }
 427
 428 enum FirstCKeyword = TOK.inline;
 429
 430 // Assert that all token enum members have consecutive values and
 431 // that none of them overlap
 432 static assert(() {
 433     foreach (idx, enumName; __traits(allMembers, TOK)) {
 434        static if (idx != __traits(getMember, TOK, enumName)) {
 435            pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
 436            static assert(0);
 437        }
 438     }
 439     return true;
 440 }());
 441
 442 /****************************************
 443  */
 444
 445 private immutable TOK[] keywords =
 446 [
 447     TOK.this_,
 448     TOK.super_,
 449     TOK.assert_,
 450     TOK.null_,
 451     TOK.true_,
 452     TOK.false_,
 453     TOK.cast_,
 454     TOK.new_,
 455     TOK.delete_,
 456     TOK.throw_,
 457     TOK.module_,
 458     TOK.pragma_,
 459     TOK.typeof_,
 460     TOK.typeid_,
 461     TOK.template_,
 462     TOK.void_,
 463     TOK.int8,
 464     TOK.uns8,
 465     TOK.int16,
 466     TOK.uns16,
 467     TOK.int32,
 468     TOK.uns32,
 469     TOK.int64,
 470     TOK.uns64,
 471     TOK.int128,
 472     TOK.uns128,
 473     TOK.float32,
 474     TOK.float64,
 475     TOK.float80,
 476     TOK.bool_,
 477     TOK.char_,
 478     TOK.wchar_,
 479     TOK.dchar_,
 480     TOK.imaginary32,
 481     TOK.imaginary64,
 482     TOK.imaginary80,
 483     TOK.complex32,
 484     TOK.complex64,
 485     TOK.complex80,
 486     TOK.delegate_,
 487     TOK.function_,
 488     TOK.is_,
 489     TOK.if_,
 490     TOK.else_,
 491     TOK.while_,
 492     TOK.for_,
 493     TOK.do_,
 494     TOK.switch_,
 495     TOK.case_,
 496     TOK.default_,
 497     TOK.break_,
 498     TOK.continue_,
 499     TOK.synchronized_,
 500     TOK.return_,
 501     TOK.goto_,
 502     TOK.try_,
 503     TOK.catch_,
 504     TOK.finally_,
 505     TOK.with_,
 506     TOK.asm_,
 507     TOK.foreach_,
 508     TOK.foreach_reverse_,
 509     TOK.scope_,
 510     TOK.struct_,
 511     TOK.class_,
 512     TOK.interface_,
 513     TOK.union_,
 514     TOK.enum_,
 515     TOK.import_,
 516     TOK.mixin_,
 517     TOK.static_,
 518     TOK.final_,
 519     TOK.const_,
 520     TOK.alias_,
 521     TOK.override_,
 522     TOK.abstract_,
 523     TOK.debug_,
 524     TOK.deprecated_,
 525     TOK.in_,
 526     TOK.out_,
 527     TOK.inout_,
 528     TOK.lazy_,
 529     TOK.auto_,
 530     TOK.align_,
 531     TOK.extern_,
 532     TOK.private_,
 533     TOK.package_,
 534     TOK.protected_,
 535     TOK.public_,
 536     TOK.export_,
 537     TOK.invariant_,
 538     TOK.unittest_,
 539     TOK.version_,
 540     TOK.argumentTypes,
 541     TOK.parameters,
 542     TOK.ref_,
 543     TOK.macro_,
 544     TOK.pure_,
 545     TOK.nothrow_,
 546     TOK.gshared,
 547     TOK.traits,
 548     TOK.vector,
 549     TOK.file,
 550     TOK.fileFullPath,
 551     TOK.line,
 552     TOK.moduleString,
 553     TOK.functionString,
 554     TOK.prettyFunction,
 555     TOK.shared_,
 556     TOK.immutable_,
 557
 558     // C only keywords
 559     TOK.inline,
 560     TOK.register,
 561     TOK.restrict,
 562     TOK.signed,
 563     TOK.sizeof_,
 564     TOK.typedef_,
 565     TOK.unsigned,
 566     TOK.volatile,
 567     TOK._Alignas,
 568     TOK._Alignof,
 569     TOK._Atomic,
 570     TOK._Bool,
 571     TOK._Complex,
 572     TOK._Generic,
 573     TOK._Imaginary,
 574     TOK._Noreturn,
 575     TOK._Static_assert,
 576     TOK._Thread_local,
 577
 578     // C only extended keywords
 579     TOK._assert,
 580     TOK._import,
 581     TOK.__cdecl,
 582     TOK.__declspec,
 583     TOK.__stdcall,
 584     TOK.__thread,
 585     TOK.__pragma,
 586     TOK.__int128,
 587     TOK.__attribute__,
 588 ];
 589
 590 // Initialize the identifier pool
 591 shared static this() nothrow
 592 {
 593     Identifier.initTable();
 594     foreach (kw; keywords)
 595     {
 596         //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr);
 597         Identifier.idPool(Token.tochars[kw], kw);
 598     }
 599 }
 600
 601 /************************************
 602  * This is used to pick the C keywords out of the tokens.
 603  * If it's not a C keyword, then it's an identifier.
 604  */
 605 static immutable TOK[TOK.max + 1] Ckeywords =
 606 () {
 607     with (TOK)
 608     {
 609         TOK[TOK.max + 1] tab = identifier;  // default to identifier
 610         enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
 611                        enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
 612                        restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
 613                        union_, unsigned, void_, volatile, while_, asm_, typeof_,
 614                        _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
 615                        _Static_assert, _Thread_local,
 616                        _import, __cdecl, __declspec, __stdcall, __thread, __pragma, __int128, __attribute__,
 617                        _assert ];
 618
 619         foreach (kw; Ckwds)
 620             tab[kw] = cast(TOK) kw;
 621
 622         return tab;
 623     }
 624 } ();
 625
 626
 627 /***********************************************************
 628  */
 629 extern (C++) struct Token
 630 {
 631     Token* next;
 632     Loc loc;
 633     const(char)* ptr; // pointer to first character of this token within buffer
 634     TOK value;
 635     const(char)[] blockComment; // doc comment string prior to this token
 636     const(char)[] lineComment; // doc comment for previous token
 637
 638     union
 639     {
 640         // Integers
 641         long intvalue;
 642         ulong unsvalue;
 643         // Floats
 644         real_t floatvalue;
 645
 646         struct
 647         {
 648             const(char)* ustring; // UTF8 string
 649             uint len;
 650             ubyte postfix; // 'c', 'w', 'd'
 651         }
 652
 653         Identifier ident;
 654     }
 655
 656     extern (D) private static immutable string[TOK.max + 1] tochars =
 657     [
 658         // Keywords
 659         TOK.this_: "this",
 660         TOK.super_: "super",
 661         TOK.assert_: "assert",
 662         TOK.null_: "null",
 663         TOK.true_: "true",
 664         TOK.false_: "false",
 665         TOK.cast_: "cast",
 666         TOK.new_: "new",
 667         TOK.delete_: "delete",
 668         TOK.throw_: "throw",
 669         TOK.module_: "module",
 670         TOK.pragma_: "pragma",
 671         TOK.typeof_: "typeof",
 672         TOK.typeid_: "typeid",
 673         TOK.template_: "template",
 674         TOK.void_: "void",
 675         TOK.int8: "byte",
 676         TOK.uns8: "ubyte",
 677         TOK.int16: "short",
 678         TOK.uns16: "ushort",
 679         TOK.int32: "int",
 680         TOK.uns32: "uint",
 681         TOK.int64: "long",
 682         TOK.uns64: "ulong",
 683         TOK.int128: "cent",
 684         TOK.uns128: "ucent",
 685         TOK.float32: "float",
 686         TOK.float64: "double",
 687         TOK.float80: "real",
 688         TOK.bool_: "bool",
 689         TOK.char_: "char",
 690         TOK.wchar_: "wchar",
 691         TOK.dchar_: "dchar",
 692         TOK.imaginary32: "ifloat",
 693         TOK.imaginary64: "idouble",
 694         TOK.imaginary80: "ireal",
 695         TOK.complex32: "cfloat",
 696         TOK.complex64: "cdouble",
 697         TOK.complex80: "creal",
 698         TOK.delegate_: "delegate",
 699         TOK.function_: "function",
 700         TOK.is_: "is",
 701         TOK.if_: "if",
 702         TOK.else_: "else",
 703         TOK.while_: "while",
 704         TOK.for_: "for",
 705         TOK.do_: "do",
 706         TOK.switch_: "switch",
 707         TOK.case_: "case",
 708         TOK.default_: "default",
 709         TOK.break_: "break",
 710         TOK.continue_: "continue",
 711         TOK.synchronized_: "synchronized",
 712         TOK.return_: "return",
 713         TOK.goto_: "goto",
 714         TOK.try_: "try",
 715         TOK.catch_: "catch",
 716         TOK.finally_: "finally",
 717         TOK.with_: "with",
 718         TOK.asm_: "asm",
 719         TOK.foreach_: "foreach",
 720         TOK.foreach_reverse_: "foreach_reverse",
 721         TOK.scope_: "scope",
 722         TOK.struct_: "struct",
 723         TOK.class_: "class",
 724         TOK.interface_: "interface",
 725         TOK.union_: "union",
 726         TOK.enum_: "enum",
 727         TOK.import_: "import",
 728         TOK.mixin_: "mixin",
 729         TOK.static_: "static",
 730         TOK.final_: "final",
 731         TOK.const_: "const",
 732         TOK.alias_: "alias",
 733         TOK.override_: "override",
 734         TOK.abstract_: "abstract",
 735         TOK.debug_: "debug",
 736         TOK.deprecated_: "deprecated",
 737         TOK.in_: "in",
 738         TOK.out_: "out",
 739         TOK.inout_: "inout",
 740         TOK.lazy_: "lazy",
 741         TOK.auto_: "auto",
 742         TOK.align_: "align",
 743         TOK.extern_: "extern",
 744         TOK.private_: "private",
 745         TOK.package_: "package",
 746         TOK.protected_: "protected",
 747         TOK.public_: "public",
 748         TOK.export_: "export",
 749         TOK.invariant_: "invariant",
 750         TOK.unittest_: "unittest",
 751         TOK.version_: "version",
 752         TOK.argumentTypes: "__argTypes",
 753         TOK.parameters: "__parameters",
 754         TOK.ref_: "ref",
 755         TOK.macro_: "macro",
 756         TOK.pure_: "pure",
 757         TOK.nothrow_: "nothrow",
 758         TOK.gshared: "__gshared",
 759         TOK.traits: "__traits",
 760         TOK.vector: "__vector",
 761         TOK.file: "__FILE__",
 762         TOK.fileFullPath: "__FILE_FULL_PATH__",
 763         TOK.line: "__LINE__",
 764         TOK.moduleString: "__MODULE__",
 765         TOK.functionString: "__FUNCTION__",
 766         TOK.prettyFunction: "__PRETTY_FUNCTION__",
 767         TOK.shared_: "shared",
 768         TOK.immutable_: "immutable",
 769
 770         TOK.endOfFile: "End of File",
 771         TOK.leftCurly: "{",
 772         TOK.rightCurly: "}",
 773         TOK.leftParenthesis: "(",
 774         TOK.rightParenthesis: ")",
 775         TOK.leftBracket: "[",
 776         TOK.rightBracket: "]",
 777         TOK.semicolon: ";",
 778         TOK.colon: ":",
 779         TOK.comma: ",",
 780         TOK.dot: ".",
 781         TOK.xor: "^",
 782         TOK.xorAssign: "^=",
 783         TOK.assign: "=",
 784         TOK.lessThan: "<",
 785         TOK.greaterThan: ">",
 786         TOK.lessOrEqual: "<=",
 787         TOK.greaterOrEqual: ">=",
 788         TOK.equal: "==",
 789         TOK.notEqual: "!=",
 790         TOK.not: "!",
 791         TOK.leftShift: "<<",
 792         TOK.rightShift: ">>",
 793         TOK.unsignedRightShift: ">>>",
 794         TOK.add: "+",
 795         TOK.min: "-",
 796         TOK.mul: "*",
 797         TOK.div: "/",
 798         TOK.mod: "%",
 799         TOK.slice: "..",
 800         TOK.dotDotDot: "...",
 801         TOK.and: "&",
 802         TOK.andAnd: "&&",
 803         TOK.or: "|",
 804         TOK.orOr: "||",
 805         TOK.tilde: "~",
 806         TOK.dollar: "$",
 807         TOK.plusPlus: "++",
 808         TOK.minusMinus: "--",
 809         TOK.question: "?",
 810         TOK.variable: "var",
 811         TOK.addAssign: "+=",
 812         TOK.minAssign: "-=",
 813         TOK.mulAssign: "*=",
 814         TOK.divAssign: "/=",
 815         TOK.modAssign: "%=",
 816         TOK.leftShiftAssign: "<<=",
 817         TOK.rightShiftAssign: ">>=",
 818         TOK.unsignedRightShiftAssign: ">>>=",
 819         TOK.andAssign: "&=",
 820         TOK.orAssign: "|=",
 821         TOK.concatenateAssign: "~=",
 822         TOK.identity: "is",
 823         TOK.notIdentity: "!is",
 824         TOK.identifier: "identifier",
 825         TOK.at: "@",
 826         TOK.pow: "^^",
 827         TOK.powAssign: "^^=",
 828         TOK.goesTo: "=>",
 829         TOK.pound: "#",
 830         TOK.arrow: "->",
 831         TOK.colonColon: "::",
 832
 833         // For debugging
 834         TOK.error: "error",
 835         TOK.string_: "string",
 836         TOK.onScopeExit: "scope(exit)",
 837         TOK.onScopeSuccess: "scope(success)",
 838         TOK.onScopeFailure: "scope(failure)",
 839
 840         // Finish up
 841         TOK.reserved: "reserved",
 842         TOK.comment: "comment",
 843         TOK.int32Literal: "int32v",
 844         TOK.uns32Literal: "uns32v",
 845         TOK.int64Literal: "int64v",
 846         TOK.uns64Literal: "uns64v",
 847         TOK.int128Literal: "int128v",
 848         TOK.uns128Literal: "uns128v",
 849         TOK.float32Literal: "float32v",
 850         TOK.float64Literal: "float64v",
 851         TOK.float80Literal: "float80v",
 852         TOK.imaginary32Literal: "imaginary32v",
 853         TOK.imaginary64Literal: "imaginary64v",
 854         TOK.imaginary80Literal: "imaginary80v",
 855         TOK.charLiteral: "charv",
 856         TOK.wcharLiteral: "wcharv",
 857         TOK.dcharLiteral: "dcharv",
 858         TOK.wchar_tLiteral: "wchar_tv",
 859         TOK.hexadecimalString: "xstring",
 860         TOK.endOfLine: "\\n",
 861         TOK.whitespace: "whitespace",
 862
 863         // C only keywords
 864         TOK.inline    : "inline",
 865         TOK.register  : "register",
 866         TOK.restrict  : "restrict",
 867         TOK.signed    : "signed",
 868         TOK.sizeof_   : "sizeof",
 869         TOK.typedef_  : "typedef",
 870         TOK.unsigned  : "unsigned",
 871         TOK.volatile  : "volatile",
 872         TOK._Alignas  : "_Alignas",
 873         TOK._Alignof  : "_Alignof",
 874         TOK._Atomic   : "_Atomic",
 875         TOK._Bool     : "_Bool",
 876         TOK._Complex  : "_Complex",
 877         TOK._Generic  : "_Generic",
 878         TOK._Imaginary: "_Imaginary",
 879         TOK._Noreturn : "_Noreturn",
 880         TOK._Static_assert : "_Static_assert",
 881         TOK._Thread_local  : "_Thread_local",
 882
 883         // C only extended keywords
 884         TOK._assert       : "__check",
 885         TOK._import       : "__import",
 886         TOK.__cdecl        : "__cdecl",
 887         TOK.__declspec     : "__declspec",
 888         TOK.__stdcall      : "__stdcall",
 889         TOK.__thread       : "__thread",
 890         TOK.__pragma       : "__pragma",
 891         TOK.__int128       : "__int128",
 892         TOK.__attribute__  : "__attribute__",
 893     ];
 894
 895     static assert(() {
 896         foreach (s; tochars)
 897             assert(s.length);
 898         return true;
 899     }());
 900
 901 nothrow:
 902
 903     extern (D) int isKeyword() pure const @safe @nogc
 904     {
 905         foreach (kw; keywords)
 906         {
 907             if (kw == value)
 908                 return 1;
 909         }
 910         return 0;
 911     }
 912
 913     /****
 914      * Set to contents of ptr[0..length]
 915      * Params:
 916      *  ptr = pointer to string
 917      *  length = length of string
 918      */
 919     void setString(const(char)* ptr, size_t length)
 920     {
 921         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
 922         memcpy(s, ptr, length);
 923         s[length] = 0;
 924         ustring = s;
 925         len = cast(uint)length;
 926         postfix = 0;
 927     }
 928
 929     /****
 930      * Set to contents of buf
 931      * Params:
 932      *  buf = string (not zero terminated)
 933      */
 934     void setString(const ref OutBuffer buf)
 935     {
 936         setString(cast(const(char)*)buf[].ptr, buf.length);
 937     }
 938
 939     /****
 940      * Set to empty string
 941      */
 942     void setString()
 943     {
 944         ustring = "";
 945         len = 0;
 946         postfix = 0;
 947     }
 948
 949     extern (C++) const(char)* toChars() const
 950     {
 951         return toString().ptr;
 952     }
 953
 954     /*********************************
 955      * Returns:
 956      *  a zero-terminated string representation of the token,
 957      *  sometimes reusing a static buffer, sometimes leaking memory
 958      */
 959     extern (D) const(char)[] toString() const
 960     {
 961         const bufflen = 3 + 3 * floatvalue.sizeof + 1;
 962         __gshared char[bufflen + 2] buffer;     // extra 2 for suffixes
 963         char* p = &buffer[0];
 964         switch (value)
 965         {
 966         case TOK.int32Literal:
 967             const length = snprintf(p, bufflen, "%d", cast(int)intvalue);
 968             return p[0 .. length];
 969
 970         case TOK.uns32Literal:
 971         case TOK.wchar_tLiteral:
 972             const length = snprintf(p, bufflen, "%uU", cast(uint)unsvalue);
 973             return p[0 .. length];
 974
 975         case TOK.wcharLiteral:
 976         case TOK.dcharLiteral:
 977         case TOK.charLiteral:
 978             OutBuffer buf;
 979             buf.writeSingleCharLiteral(cast(dchar) intvalue);
 980             return buf.extractSlice(true);
 981
 982         case TOK.int64Literal:
 983             const length = snprintf(p, bufflen, "%lldL", cast(long)intvalue);
 984             return p[0 .. length];
 985
 986         case TOK.uns64Literal:
 987             const length = snprintf(p, bufflen, "%lluUL", cast(ulong)unsvalue);
 988             return p[0 .. length];
 989
 990         case TOK.float32Literal:
 991             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
 992             p[length] = 'f';
 993             p[length + 1] = 0;
 994             return p[0 .. length + 1];
 995
 996         case TOK.float64Literal:
 997             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
 998             return p[0 .. length];
 999
1000         case TOK.float80Literal:
1001             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1002             p[length] = 'L';
1003             p[length + 1] = 0;
1004             return p[0 .. length + 1];
1005
1006         case TOK.imaginary32Literal:
1007             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1008             p[length    ] = 'f';
1009             p[length + 1] = 'i';
1010             p[length + 2] = 0;
1011             return p[0 .. length + 2];
1012
1013         case TOK.imaginary64Literal:
1014             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1015             p[length] = 'i';
1016             p[length + 1] = 0;
1017             return p[0 .. length + 1];
1018
1019         case TOK.imaginary80Literal:
1020             const length = CTFloat.sprint(p, bufflen, 'g', floatvalue);
1021             p[length    ] = 'L';
1022             p[length + 1] = 'i';
1023             p[length + 2] = 0;
1024             return p[0 .. length + 2];
1025
1026         case TOK.string_:
1027             OutBuffer buf;
1028             buf.writeByte('"');
1029             for (size_t i = 0; i < len;)
1030             {
1031                 dchar c;
1032                 utf_decodeChar(ustring[0 .. len], i, c);
1033                 writeCharLiteral(buf, c);
1034             }
1035             buf.writeByte('"');
1036             if (postfix)
1037                 buf.writeByte(postfix);
1038             return buf.extractSlice(true);
1039
1040         case TOK.hexadecimalString:
1041             OutBuffer buf;
1042             buf.writeByte('x');
1043             buf.writeByte('"');
1044             foreach (size_t i; 0 .. len)
1045             {
1046                 if (i)
1047                     buf.writeByte(' ');
1048                 buf.printf("%02x", ustring[i]);
1049             }
1050             buf.writeByte('"');
1051             if (postfix)
1052                 buf.writeByte(postfix);
1053             return buf.extractSlice(true);
1054
1055         case TOK.identifier:
1056         case TOK.enum_:
1057         case TOK.struct_:
1058         case TOK.import_:
1059         case TOK.wchar_:
1060         case TOK.dchar_:
1061         case TOK.bool_:
1062         case TOK.char_:
1063         case TOK.int8:
1064         case TOK.uns8:
1065         case TOK.int16:
1066         case TOK.uns16:
1067         case TOK.int32:
1068         case TOK.uns32:
1069         case TOK.int64:
1070         case TOK.uns64:
1071         case TOK.int128:
1072         case TOK.uns128:
1073         case TOK.float32:
1074         case TOK.float64:
1075         case TOK.float80:
1076         case TOK.imaginary32:
1077         case TOK.imaginary64:
1078         case TOK.imaginary80:
1079         case TOK.complex32:
1080         case TOK.complex64:
1081         case TOK.complex80:
1082         case TOK.void_:
1083             return ident.toString();
1084
1085         default:
1086             return tochars[value];
1087         }
1088     }
1089
1090     static const(char)* toChars(TOK value)
1091     {
1092         return toString(value).ptr;
1093     }
1094
1095     extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1096     {
1097         return tochars[value];
1098     }
1099 }
1100
1101 /**
1102  * Write a character, using a readable escape sequence if needed
1103  *
1104  * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property
1105  *
1106  * Params:
1107  *   buf = buffer to append character in
1108  *   c = code point to write
1109  */
1110 nothrow
1111 void writeCharLiteral(ref OutBuffer buf, dchar c)
1112 {
1113     switch (c)
1114     {
1115         case '\0':
1116             buf.writestring("\\0");
1117             break;
1118         case '\n':
1119             buf.writestring("\\n");
1120             break;
1121         case '\r':
1122             buf.writestring("\\r");
1123             break;
1124         case '\t':
1125             buf.writestring("\\t");
1126             break;
1127         case '\b':
1128             buf.writestring("\\b");
1129             break;
1130         case '\f':
1131             buf.writestring("\\f");
1132             break;
1133         case '"':
1134         case '\\':
1135             buf.writeByte('\\');
1136             goto default;
1137         default:
1138             if (c <= 0xFF)
1139             {
1140                 if (isprint(c))
1141                     buf.writeByte(c);
1142                 else
1143                     buf.printf("\\x%02x", c);
1144             }
1145             else if (c <= 0xFFFF)
1146                 buf.printf("\\u%04x", c);
1147             else
1148                 buf.printf("\\U%08x", c);
1149             break;
1150     }
1151 }
1152
1153 unittest
1154 {
1155     OutBuffer buf;
1156     foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d)
1157     {
1158         writeCharLiteral(buf, d);
1159     }
1160     assert(buf[] == `a\n\r\t\b\f\0\x11\u7233\U00017233`);
1161 }
1162
1163 /**
1164  * Write a single-quoted character literal
1165  *
1166  * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property
1167  *
1168  * Params:
1169  *   buf = buffer to append character in
1170  *   c = code point to write
1171  */
1172 nothrow
1173 void writeSingleCharLiteral(ref OutBuffer buf, dchar c)
1174 {
1175     buf.writeByte('\'');
1176     if (c == '\'')
1177         buf.writeByte('\\');
1178
1179     if (c == '"')
1180         buf.writeByte('"');
1181     else
1182         writeCharLiteral(buf, c);
1183
1184     buf.writeByte('\'');
1185 }
1186
1187 unittest
1188 {
1189     OutBuffer buf;
1190     writeSingleCharLiteral(buf, '\'');
1191     assert(buf[] == `'\''`);
1192     buf.reset();
1193     writeSingleCharLiteral(buf, '"');
1194     assert(buf[] == `'"'`);
1195     buf.reset();
1196     writeSingleCharLiteral(buf, '\n');
1197     assert(buf[] == `'\n'`);
1198 }