gcc/d/dmd/dmodule.d

   1 /**
   2  * Defines a package and module.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/module.html, Modules)
   5  *
   6  * Copyright:   Copyright (C) 1999-2023 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/dmodule.d, _dmodule.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_dmodule.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/dmodule.d
  12  */
  13
  14 module dmd.dmodule;
  15
  16 import core.stdc.stdio;
  17 import core.stdc.stdlib;
  18 import core.stdc.string;
  19 import dmd.aggregate;
  20 import dmd.arraytypes;
  21 import dmd.astcodegen;
  22 import dmd.astenums;
  23 import dmd.compiler;
  24 import dmd.gluelayer;
  25 import dmd.dimport;
  26 import dmd.dmacro;
  27 import dmd.doc;
  28 import dmd.dscope;
  29 import dmd.dsymbol;
  30 import dmd.dsymbolsem;
  31 import dmd.errors;
  32 import dmd.errorsink;
  33 import dmd.expression;
  34 import dmd.expressionsem;
  35 import dmd.file_manager;
  36 import dmd.func;
  37 import dmd.globals;
  38 import dmd.id;
  39 import dmd.identifier;
  40 import dmd.location;
  41 import dmd.parse;
  42 import dmd.cparse;
  43 import dmd.root.array;
  44 import dmd.root.file;
  45 import dmd.root.filename;
  46 import dmd.common.outbuffer;
  47 import dmd.root.port;
  48 import dmd.root.rmem;
  49 import dmd.rootobject;
  50 import dmd.root.string;
  51 import dmd.semantic2;
  52 import dmd.semantic3;
  53 import dmd.target;
  54 import dmd.utils;
  55 import dmd.visitor;
  56
  57 version (IN_GCC) {}
  58 else version (IN_LLVM) {}
  59 else version = MARS;
  60
  61 // function used to call semantic3 on a module's dependencies
  62 void semantic3OnDependencies(Module m)
  63 {
  64     if (!m)
  65         return;
  66
  67     if (m.semanticRun > PASS.semantic3)
  68         return;
  69
  70     m.semantic3(null);
  71
  72     foreach (i; 1 .. m.aimports.length)
  73         semantic3OnDependencies(m.aimports[i]);
  74 }
  75
  76 /**
  77  * Remove generated .di files on error and exit
  78  */
  79 void removeHdrFilesAndFail(ref Param params, ref Modules modules) nothrow
  80 {
  81     if (params.dihdr.doOutput)
  82     {
  83         foreach (m; modules)
  84         {
  85             if (m.filetype == FileType.dhdr)
  86                 continue;
  87             File.remove(m.hdrfile.toChars());
  88         }
  89     }
  90
  91     fatal();
  92 }
  93
  94 /**
  95  * Converts a chain of identifiers to the filename of the module
  96  *
  97  * Params:
  98  *  packages = the names of the "parent" packages
  99  *  ident = the name of the child package or module
 100  *
 101  * Returns:
 102  *  the filename of the child package or module
 103  */
 104 private const(char)[] getFilename(Identifier[] packages, Identifier ident) nothrow
 105 {
 106     const(char)[] filename = ident.toString();
 107
 108     OutBuffer buf;
 109     OutBuffer dotmods;
 110     auto modAliases = &global.params.modFileAliasStrings;
 111
 112     if (packages.length == 0 && modAliases.length == 0)
 113         return filename;
 114
 115     void checkModFileAlias(const(char)[] p)
 116     {
 117         /* Check and replace the contents of buf[] with
 118         * an alias string from global.params.modFileAliasStrings[]
 119         */
 120         dotmods.writestring(p);
 121         foreach_reverse (const m; *modAliases)
 122         {
 123             const q = strchr(m, '=');
 124             assert(q);
 125             if (dotmods.length == q - m && memcmp(dotmods.peekChars(), m, q - m) == 0)
 126             {
 127                 buf.setsize(0);
 128                 auto rhs = q[1 .. strlen(q)];
 129                 if (rhs.length > 0 && (rhs[$ - 1] == '/' || rhs[$ - 1] == '\\'))
 130                     rhs = rhs[0 .. $ - 1]; // remove trailing separator
 131                 buf.writestring(rhs);
 132                 break; // last matching entry in ms[] wins
 133             }
 134         }
 135         dotmods.writeByte('.');
 136     }
 137
 138     foreach (pid; packages)
 139     {
 140         const p = pid.toString();
 141         buf.writestring(p);
 142         if (modAliases.length)
 143             checkModFileAlias(p);
 144         version (Windows)
 145             enum FileSeparator = '\\';
 146         else
 147             enum FileSeparator = '/';
 148         buf.writeByte(FileSeparator);
 149     }
 150     buf.writestring(filename);
 151     if (modAliases.length)
 152         checkModFileAlias(filename);
 153     buf.writeByte(0);
 154     filename = buf.extractSlice()[0 .. $ - 1];
 155
 156     return filename;
 157 }
 158
 159 /***********************************************************
 160  */
 161 extern (C++) class Package : ScopeDsymbol
 162 {
 163     PKG isPkgMod = PKG.unknown;
 164     uint tag;        // auto incremented tag, used to mask package tree in scopes
 165     Module mod;     // !=null if isPkgMod == PKG.module_
 166
 167     final extern (D) this(const ref Loc loc, Identifier ident) nothrow
 168     {
 169         super(loc, ident);
 170         __gshared uint packageTag;
 171         this.tag = packageTag++;
 172     }
 173
 174     override const(char)* kind() const nothrow
 175     {
 176         return "package";
 177     }
 178
 179     override bool equals(const RootObject o) const
 180     {
 181         // custom 'equals' for bug 17441. "package a" and "module a" are not equal
 182         if (this == o)
 183             return true;
 184         auto p = cast(Package)o;
 185         return p && isModule() == p.isModule() && ident.equals(p.ident);
 186     }
 187
 188     /****************************************************
 189      * Input:
 190      *      packages[]      the pkg1.pkg2 of pkg1.pkg2.mod
 191      * Returns:
 192      *      the symbol table that mod should be inserted into
 193      * Output:
 194      *      *pparent        the rightmost package, i.e. pkg2, or NULL if no packages
 195      *      *ppkg           the leftmost package, i.e. pkg1, or NULL if no packages
 196      */
 197     extern (D) static DsymbolTable resolve(Identifier[] packages, Dsymbol* pparent, Package* ppkg)
 198     {
 199         DsymbolTable dst = Module.modules;
 200         Dsymbol parent = null;
 201         //printf("Package::resolve()\n");
 202         if (ppkg)
 203             *ppkg = null;
 204         foreach (pid; packages)
 205         {
 206             Package pkg;
 207             Dsymbol p = dst.lookup(pid);
 208             if (!p)
 209             {
 210                 pkg = new Package(Loc.initial, pid);
 211                 dst.insert(pkg);
 212                 pkg.parent = parent;
 213                 pkg.symtab = new DsymbolTable();
 214             }
 215             else
 216             {
 217                 pkg = p.isPackage();
 218                 assert(pkg);
 219                 // It might already be a module, not a package, but that needs
 220                 // to be checked at a higher level, where a nice error message
 221                 // can be generated.
 222                 // dot net needs modules and packages with same name
 223                 // But we still need a symbol table for it
 224                 if (!pkg.symtab)
 225                     pkg.symtab = new DsymbolTable();
 226             }
 227             parent = pkg;
 228             dst = pkg.symtab;
 229             if (ppkg && !*ppkg)
 230                 *ppkg = pkg;
 231             if (pkg.isModule())
 232             {
 233                 // Return the module so that a nice error message can be generated
 234                 if (ppkg)
 235                     *ppkg = cast(Package)p;
 236                 break;
 237             }
 238         }
 239
 240         if (pparent)
 241             *pparent = parent;
 242         return dst;
 243     }
 244
 245     override final inout(Package) isPackage() inout
 246     {
 247         return this;
 248     }
 249
 250     /**
 251      * Checks if pkg is a sub-package of this
 252      *
 253      * For example, if this qualifies to 'a1.a2' and pkg - to 'a1.a2.a3',
 254      * this function returns 'true'. If it is other way around or qualified
 255      * package paths conflict function returns 'false'.
 256      *
 257      * Params:
 258      *  pkg = possible subpackage
 259      *
 260      * Returns:
 261      *  see description
 262      */
 263     final bool isAncestorPackageOf(const Package pkg) const
 264     {
 265         if (this == pkg)
 266             return true;
 267         if (!pkg || !pkg.parent)
 268             return false;
 269         return isAncestorPackageOf(pkg.parent.isPackage());
 270     }
 271
 272     override void accept(Visitor v)
 273     {
 274         v.visit(this);
 275     }
 276
 277     final Module isPackageMod()
 278     {
 279         if (isPkgMod == PKG.module_)
 280         {
 281             return mod;
 282         }
 283         return null;
 284     }
 285
 286     /**
 287      * Checks for the existence of a package.d to set isPkgMod appropriately
 288      * if isPkgMod == PKG.unknown
 289      */
 290     final void resolvePKGunknown()
 291     {
 292         if (isModule())
 293             return;
 294         if (isPkgMod != PKG.unknown)
 295             return;
 296
 297         Identifier[] packages;
 298         for (Dsymbol s = this.parent; s; s = s.parent)
 299             packages ~= s.ident;
 300         reverse(packages);
 301
 302         if (Module.find(getFilename(packages, ident)))
 303             Module.load(Loc.initial, packages, this.ident);
 304         else
 305             isPkgMod = PKG.package_;
 306     }
 307 }
 308
 309 /***********************************************************
 310  */
 311 extern (C++) final class Module : Package
 312 {
 313     extern (C++) __gshared Module rootModule;
 314     extern (C++) __gshared DsymbolTable modules; // symbol table of all modules
 315     extern (C++) __gshared Modules amodules;     // array of all modules
 316     extern (C++) __gshared Dsymbols deferred;    // deferred Dsymbol's needing semantic() run on them
 317     extern (C++) __gshared Dsymbols deferred2;   // deferred Dsymbol's needing semantic2() run on them
 318     extern (C++) __gshared Dsymbols deferred3;   // deferred Dsymbol's needing semantic3() run on them
 319
 320     static void _init()
 321     {
 322         modules = new DsymbolTable();
 323     }
 324
 325     /**
 326      * Deinitializes the global state of the compiler.
 327      *
 328      * This can be used to restore the state set by `_init` to its original
 329      * state.
 330      */
 331     static void deinitialize()
 332     {
 333         modules = modules.init;
 334     }
 335
 336     extern (C++) __gshared AggregateDeclaration moduleinfo;
 337
 338     const(char)[] arg;           // original argument name
 339     ModuleDeclaration* md;      // if !=null, the contents of the ModuleDeclaration declaration
 340     const FileName srcfile;     // input source file
 341     const FileName objfile;     // output .obj file
 342     const FileName hdrfile;     // 'header' file
 343     FileName docfile;           // output documentation file
 344     const(ubyte)[] src;         /// Raw content of the file
 345     uint errors;                // if any errors in file
 346     uint numlines;              // number of lines in source file
 347     FileType filetype;          // source file type
 348     bool hasAlwaysInlines;      // contains references to functions that must be inlined
 349     bool isPackageFile;         // if it is a package.d
 350     Package pkg;                // if isPackageFile is true, the Package that contains this package.d
 351     Strings contentImportedFiles; // array of files whose content was imported
 352     int needmoduleinfo;
 353     private ThreeState selfimports;
 354     private ThreeState rootimports;
 355     Dsymbol[void*] tagSymTab;   /// ImportC: tag symbols that conflict with other symbols used as the index
 356
 357     private OutBuffer defines;  // collect all the #define lines here
 358
 359
 360     /*************************************
 361      * Return true if module imports itself.
 362      */
 363     bool selfImports()
 364     {
 365         //printf("Module::selfImports() %s\n", toChars());
 366         if (selfimports == ThreeState.none)
 367         {
 368             foreach (Module m; amodules)
 369                 m.insearch = false;
 370             selfimports = imports(this) ? ThreeState.yes : ThreeState.no;
 371             foreach (Module m; amodules)
 372                 m.insearch = false;
 373         }
 374         return selfimports == ThreeState.yes;
 375     }
 376
 377     /*************************************
 378      * Return true if module imports root module.
 379      */
 380     bool rootImports()
 381     {
 382         //printf("Module::rootImports() %s\n", toChars());
 383         if (rootimports == ThreeState.none)
 384         {
 385             foreach (Module m; amodules)
 386                 m.insearch = false;
 387             rootimports = ThreeState.no;
 388             foreach (Module m; amodules)
 389             {
 390                 if (m.isRoot() && imports(m))
 391                 {
 392                     rootimports = ThreeState.yes;
 393                     break;
 394                 }
 395             }
 396             foreach (Module m; amodules)
 397                 m.insearch = false;
 398         }
 399         return rootimports == ThreeState.yes;
 400     }
 401
 402     Identifier searchCacheIdent;
 403     Dsymbol searchCacheSymbol;  // cached value of search
 404     SearchOptFlags searchCacheFlags;       // cached flags
 405     bool insearch;
 406
 407     /**
 408      * A root module is one that will be compiled all the way to
 409      * object code.  This field holds the root module that caused
 410      * this module to be loaded.  If this module is a root module,
 411      * then it will be set to `this`.  This is used to determine
 412      * ownership of template instantiation.
 413      */
 414     Module importedFrom;
 415
 416     Dsymbols* decldefs;         // top level declarations for this Module
 417
 418     Modules aimports;           // all imported modules
 419
 420     uint debuglevel;            // debug level
 421     Identifiers* debugids;      // debug identifiers
 422     Identifiers* debugidsNot;   // forward referenced debug identifiers
 423
 424     uint versionlevel;          // version level
 425     Identifiers* versionids;    // version identifiers
 426     Identifiers* versionidsNot; // forward referenced version identifiers
 427
 428     MacroTable macrotable;      // document comment macros
 429     Escape* _escapetable;       // document comment escapes
 430
 431     size_t nameoffset;          // offset of module name from start of ModuleInfo
 432     size_t namelen;             // length of module name in characters
 433
 434     extern (D) this(const ref Loc loc, const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 435     {
 436         super(loc, ident);
 437         const(char)[] srcfilename;
 438         //printf("Module::Module(filename = '%.*s', ident = '%s')\n", cast(int)filename.length, filename.ptr, ident.toChars());
 439         this.arg = filename;
 440         srcfilename = FileName.defaultExt(filename, mars_ext);
 441         if (target.run_noext && global.params.run &&
 442             !FileName.ext(filename) &&
 443             FileName.exists(srcfilename) == 0 &&
 444             FileName.exists(filename) == 1)
 445         {
 446             FileName.free(srcfilename.ptr);
 447             srcfilename = FileName.removeExt(filename); // just does a mem.strdup(filename)
 448         }
 449         else if (!FileName.equalsExt(srcfilename, mars_ext) &&
 450                  !FileName.equalsExt(srcfilename, hdr_ext) &&
 451                  !FileName.equalsExt(srcfilename, c_ext) &&
 452                  !FileName.equalsExt(srcfilename, i_ext) &&
 453                  !FileName.equalsExt(srcfilename, dd_ext))
 454         {
 455
 456             error(loc, "%s `%s` source file name '%.*s' must have .%.*s extension",
 457                   kind, toPrettyChars,
 458                   cast(int)srcfilename.length, srcfilename.ptr,
 459                   cast(int)mars_ext.length, mars_ext.ptr);
 460             fatal();
 461         }
 462
 463         srcfile = FileName(srcfilename);
 464         objfile = setOutfilename(global.params.objname, global.params.objdir, filename, target.obj_ext);
 465         if (doDocComment)
 466             setDocfile();
 467         if (doHdrGen)
 468             hdrfile = setOutfilename(global.params.dihdr.name, global.params.dihdr.dir, arg, hdr_ext);
 469     }
 470
 471     extern (D) this(const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 472     {
 473         this(Loc.initial, filename, ident, doDocComment, doHdrGen);
 474     }
 475
 476     static Module create(const(char)* filename, Identifier ident, int doDocComment, int doHdrGen)
 477     {
 478         return create(filename.toDString, ident, doDocComment, doHdrGen);
 479     }
 480
 481     extern (D) static Module create(const(char)[] filename, Identifier ident, int doDocComment, int doHdrGen)
 482     {
 483         return new Module(Loc.initial, filename, ident, doDocComment, doHdrGen);
 484     }
 485
 486     static const(char)* find(const(char)* filename)
 487     {
 488         return find(filename.toDString).ptr;
 489     }
 490
 491     extern (D) static const(char)[] find(const(char)[] filename)
 492     {
 493         return global.fileManager.lookForSourceFile(filename, global.path ? (*global.path)[] : null);
 494     }
 495
 496     extern (C++) static Module load(const ref Loc loc, Identifiers* packages, Identifier ident)
 497     {
 498         return load(loc, packages ? (*packages)[] : null, ident);
 499     }
 500
 501     extern (D) static Module load(const ref Loc loc, Identifier[] packages, Identifier ident)
 502     {
 503         //printf("Module::load(ident = '%s')\n", ident.toChars());
 504         // Build module filename by turning:
 505         //  foo.bar.baz
 506         // into:
 507         //  foo\bar\baz
 508         const(char)[] filename = getFilename(packages, ident);
 509         // Look for the source file
 510         if (const result = find(filename))
 511             filename = result; // leaks
 512
 513         auto m = new Module(loc, filename, ident, 0, 0);
 514
 515         if (!m.read(loc))
 516             return null;
 517         if (global.params.v.verbose)
 518         {
 519             OutBuffer buf;
 520             foreach (pid; packages)
 521             {
 522                 buf.writestring(pid.toString());
 523                 buf.writeByte('.');
 524             }
 525             buf.printf("%s\t(%s)", ident.toChars(), m.srcfile.toChars());
 526             message("import    %s", buf.peekChars());
 527         }
 528         if((m = m.parse()) is null) return null;
 529
 530         return m;
 531     }
 532
 533     override const(char)* kind() const
 534     {
 535         return "module";
 536     }
 537
 538     /*********************************************
 539      * Combines things into output file name for .html and .di files.
 540      * Input:
 541      *      name    Command line name given for the file, NULL if none
 542      *      dir     Command line directory given for the file, NULL if none
 543      *      arg     Name of the source file
 544      *      ext     File name extension to use if 'name' is NULL
 545      *      global.params.preservePaths     get output path from arg
 546      *      srcfile Input file - output file name must not match input file
 547      */
 548     extern(D) FileName setOutfilename(const(char)[] name, const(char)[] dir, const(char)[] arg, const(char)[] ext)
 549     {
 550         const(char)[] docfilename;
 551         if (name)
 552         {
 553             docfilename = name;
 554         }
 555         else
 556         {
 557             const(char)[] argdoc;
 558             OutBuffer buf;
 559             if (arg == "__stdin.d")
 560             {
 561                 version (Posix)
 562                     import core.sys.posix.unistd : getpid;
 563                 else version (Windows)
 564                     import core.sys.windows.winbase : getpid = GetCurrentProcessId;
 565                 buf.printf("__stdin_%d.d", getpid());
 566                 arg = buf[];
 567             }
 568             if (global.params.preservePaths)
 569                 argdoc = arg;
 570             else
 571                 argdoc = FileName.name(arg);
 572             // If argdoc doesn't have an absolute path, make it relative to dir
 573             if (!FileName.absolute(argdoc))
 574             {
 575                 //FileName::ensurePathExists(dir);
 576                 argdoc = FileName.combine(dir, argdoc);
 577             }
 578             docfilename = FileName.forceExt(argdoc, ext);
 579         }
 580         if (FileName.equals(docfilename, srcfile.toString()))
 581         {
 582             error(loc, "%s `%s` source file and output file have same name '%s'",
 583                 kind, toPrettyChars, srcfile.toChars());
 584             fatal();
 585         }
 586         return FileName(docfilename);
 587     }
 588
 589     extern (D) void setDocfile()
 590     {
 591         docfile = setOutfilename(global.params.ddoc.name, global.params.ddoc.dir, arg, doc_ext);
 592     }
 593
 594     /**
 595      * Trigger the relevant semantic error when a file cannot be read
 596      *
 597      * We special case `object.d` as a failure is likely to be a rare
 598      * but difficult to diagnose case for the user. Packages also require
 599      * special handling to avoid exposing the compiler's internals.
 600      *
 601      * Params:
 602      *  loc = The location at which the file read originated (e.g. import)
 603      */
 604     private void onFileReadError(const ref Loc loc)
 605     {
 606         if (FileName.equals(srcfile.toString(), "object.d"))
 607         {
 608             .error(loc, "cannot find source code for runtime library file 'object.d'");
 609             version (IN_LLVM)
 610             {
 611                 errorSupplemental(loc, "ldc2 might not be correctly installed.");
 612                 errorSupplemental(loc, "Please check your ldc2.conf configuration file.");
 613                 errorSupplemental(loc, "Installation instructions can be found at http://wiki.dlang.org/LDC.");
 614             }
 615             version (MARS)
 616             {
 617                 errorSupplemental(loc, "dmd might not be correctly installed. Run 'dmd -man' for installation instructions.");
 618                 const dmdConfFile = global.inifilename.length ? FileName.canonicalName(global.inifilename) : "not found";
 619                 errorSupplemental(loc, "config file: %.*s", cast(int)dmdConfFile.length, dmdConfFile.ptr);
 620             }
 621         }
 622         else if (FileName.ext(this.arg) || !loc.isValid())
 623         {
 624             // Modules whose original argument name has an extension, or do not
 625             // have a valid location come from the command-line.
 626             // Error that their file cannot be found and return early.
 627             .error(loc, "cannot find input file `%s`", srcfile.toChars());
 628         }
 629         else
 630         {
 631             // if module is not named 'package' but we're trying to read 'package.d', we're looking for a package module
 632             bool isPackageMod = (strcmp(toChars(), "package") != 0) && isPackageFileName(srcfile);
 633             if (isPackageMod)
 634                 .error(loc, "importing package '%s' requires a 'package.d' file which cannot be found in '%s'", toChars(), srcfile.toChars());
 635             else
 636             {
 637                 .error(loc, "unable to read module `%s`", toChars());
 638                 const pkgfile = FileName.combine(FileName.removeExt(srcfile.toString()), package_d);
 639                 .errorSupplemental(loc, "Expected '%s' or '%s' in one of the following import paths:",
 640                     srcfile.toChars(), pkgfile.ptr);
 641             }
 642         }
 643         if (!global.gag)
 644         {
 645             /* Print path
 646              */
 647             if (global.path)
 648             {
 649                 foreach (i, p; *global.path)
 650                     fprintf(stderr, "import path[%llu] = %s\n", cast(ulong)i, p);
 651             }
 652             else
 653             {
 654                 fprintf(stderr, "Specify path to file '%s' with -I switch\n", srcfile.toChars());
 655             }
 656
 657             removeHdrFilesAndFail(global.params, Module.amodules);
 658         }
 659     }
 660
 661     /**
 662      * Reads the file from `srcfile` and loads the source buffer.
 663      *
 664      * If makefile module dependency is requested, we add this module
 665      * to the list of dependencies from here.
 666      *
 667      * Params:
 668      *  loc = the location
 669      *
 670      * Returns: `true` if successful
 671      */
 672     bool read(const ref Loc loc)
 673     {
 674         if (this.src)
 675             return true; // already read
 676
 677         //printf("Module::read('%s') file '%s'\n", toChars(), srcfile.toChars());
 678
 679         /* Preprocess the file if it's a .c file
 680          */
 681         FileName filename = srcfile;
 682         bool ifile = false;             // did we generate a .i file
 683         scope (exit)
 684         {
 685             if (ifile)
 686                 File.remove(filename.toChars());        // remove generated file
 687         }
 688
 689         if (global.preprocess &&
 690             FileName.equalsExt(srcfile.toString(), c_ext) &&
 691             FileName.exists(srcfile.toString()))
 692         {
 693             filename = global.preprocess(srcfile, loc, ifile, &defines);  // run C preprocessor
 694         }
 695
 696         if (auto result = global.fileManager.lookup(filename))
 697         {
 698             this.src = result;
 699             if (global.params.makeDeps.doOutput)
 700                 global.params.makeDeps.files.push(srcfile.toChars());
 701             return true;
 702         }
 703
 704         this.onFileReadError(loc);
 705         return false;
 706     }
 707
 708     /// syntactic parse
 709     Module parse()
 710     {
 711         return parseModule!ASTCodegen();
 712     }
 713
 714     /// ditto
 715     extern (D) Module parseModule(AST)()
 716     {
 717         const(char)* srcname = srcfile.toChars();
 718         //printf("Module::parse(srcname = '%s')\n", srcname);
 719         isPackageFile = isPackageFileName(srcfile);
 720         const(char)[] buf = processSource(src, this);
 721         // an error happened on UTF conversion
 722         if (buf is null) return null;
 723
 724         /* If it starts with the string "Ddoc", then it's a documentation
 725          * source file.
 726          */
 727         if (buf.length>= 4 && buf[0..4] == "Ddoc")
 728         {
 729             comment = buf.ptr + 4;
 730             filetype = FileType.ddoc;
 731             if (!docfile)
 732                 setDocfile();
 733             return this;
 734         }
 735         /* If it has the extension ".dd", it is also a documentation
 736          * source file. Documentation source files may begin with "Ddoc"
 737          * but do not have to if they have the .dd extension.
 738          * https://issues.dlang.org/show_bug.cgi?id=15465
 739          */
 740         if (FileName.equalsExt(arg, dd_ext))
 741         {
 742             comment = buf.ptr; // the optional Ddoc, if present, is handled above.
 743             filetype = FileType.ddoc;
 744             if (!docfile)
 745                 setDocfile();
 746             return this;
 747         }
 748         /* If it has the extension ".di", it is a "header" file.
 749          */
 750         if (FileName.equalsExt(arg, hdr_ext))
 751             filetype = FileType.dhdr;
 752
 753         /// Promote `this` to a root module if requested via `-i`
 754         void checkCompiledImport()
 755         {
 756             if (!this.isRoot() && Compiler.onImport(this))
 757                 this.importedFrom = this;
 758         }
 759
 760         DsymbolTable dst;
 761         Package ppack = null;
 762
 763         /* If it has the extension ".c", it is a "C" file.
 764          * If it has the extension ".i", it is a preprocessed "C" file.
 765          */
 766         if (FileName.equalsExt(arg, c_ext) || FileName.equalsExt(arg, i_ext))
 767         {
 768             filetype = FileType.c;
 769
 770             global.compileEnv.masm = target.os == Target.OS.Windows && !target.omfobj; // Microsoft inline assembler format
 771             scope p = new CParser!AST(this, buf, cast(bool) docfile, global.errorSink, target.c, &defines, &global.compileEnv);
 772             global.compileEnv.masm = false;
 773             p.nextToken();
 774             checkCompiledImport();
 775             members = p.parseModule();
 776             assert(!p.md); // C doesn't have module declarations
 777             numlines = p.scanloc.linnum;
 778         }
 779         else
 780         {
 781             const bool doUnittests = global.params.parsingUnittestsRequired();
 782             scope p = new Parser!AST(this, buf, cast(bool) docfile, global.errorSink, &global.compileEnv, doUnittests);
 783             p.transitionIn = global.params.v.vin;
 784             p.nextToken();
 785             p.parseModuleDeclaration();
 786             md = p.md;
 787
 788             if (md)
 789             {
 790                 /* A ModuleDeclaration, md, was provided.
 791                 * The ModuleDeclaration sets the packages this module appears in, and
 792                 * the name of this module.
 793                 */
 794                 this.ident = md.id;
 795                 dst = Package.resolve(md.packages, &this.parent, &ppack);
 796             }
 797
 798             // Done after parsing the module header because `module x.y.z` may override the file name
 799             checkCompiledImport();
 800
 801             members = p.parseModuleContent();
 802             numlines = p.scanloc.linnum;
 803         }
 804
 805         /* The symbol table into which the module is to be inserted.
 806          */
 807
 808         if (md)
 809         {
 810             // Mark the package path as accessible from the current module
 811             // https://issues.dlang.org/show_bug.cgi?id=21661
 812             // Code taken from Import.addPackageAccess()
 813             if (md.packages.length > 0)
 814             {
 815                 // module a.b.c.d;
 816                 auto p = ppack; // a
 817                 addAccessiblePackage(p, Visibility(Visibility.Kind.private_));
 818                 foreach (id; md.packages[1 .. $]) // [b, c]
 819                 {
 820                     p = cast(Package) p.symtab.lookup(id);
 821                     if (p is null)
 822                         break;
 823                     addAccessiblePackage(p, Visibility(Visibility.Kind.private_));
 824                 }
 825             }
 826             assert(dst);
 827             Module m = ppack ? ppack.isModule() : null;
 828             if (m && !isPackageFileName(m.srcfile))
 829             {
 830                 .error(md.loc, "package name '%s' conflicts with usage as a module name in file %s", ppack.toPrettyChars(), m.srcfile.toChars());
 831             }
 832         }
 833         else
 834         {
 835             /* The name of the module is set to the source file name.
 836              * There are no packages.
 837              */
 838             dst = modules; // and so this module goes into global module symbol table
 839             /* Check to see if module name is a valid identifier
 840              */
 841             if (!Identifier.isValidIdentifier(this.ident.toChars()))
 842                 error(loc, "%s `%s` has non-identifier characters in filename, use module declaration instead", kind, toPrettyChars);
 843         }
 844         // Insert module into the symbol table
 845         Dsymbol s = this;
 846         if (isPackageFile)
 847         {
 848             /* If the source tree is as follows:
 849              *     pkg/
 850              *     +- package.d
 851              *     +- common.d
 852              * the 'pkg' will be incorporated to the internal package tree in two ways:
 853              *     import pkg;
 854              * and:
 855              *     import pkg.common;
 856              *
 857              * If both are used in one compilation, 'pkg' as a module (== pkg/package.d)
 858              * and a package name 'pkg' will conflict each other.
 859              *
 860              * To avoid the conflict:
 861              * 1. If preceding package name insertion had occurred by Package::resolve,
 862              *    reuse the previous wrapping 'Package' if it exists
 863              * 2. Otherwise, 'package.d' wrapped by 'Package' is inserted to the internal tree in here.
 864              *
 865              * Then change Package::isPkgMod to PKG.module_ and set Package::mod.
 866              *
 867              * Note that the 'wrapping Package' is the Package that contains package.d and other submodules,
 868              * the one inserted to the symbol table.
 869              */
 870             auto ps = dst.lookup(ident);
 871             Package p = ps ? ps.isPackage() : null;
 872             if (p is null)
 873             {
 874                 p = new Package(Loc.initial, ident);
 875                 p.tag = this.tag; // reuse the same package tag
 876                 p.symtab = new DsymbolTable();
 877             }
 878             this.tag = p.tag; // reuse the 'older' package tag
 879             this.pkg = p;
 880             p.parent = this.parent;
 881             p.isPkgMod = PKG.module_;
 882             p.mod = this;
 883             s = p;
 884         }
 885         if (!dst.insert(s))
 886         {
 887             /* It conflicts with a name that is already in the symbol table.
 888              * Figure out what went wrong, and issue error message.
 889              */
 890             Dsymbol prev = dst.lookup(ident);
 891             assert(prev);
 892             if (Module mprev = prev.isModule())
 893             {
 894                 if (!FileName.equals(srcname, mprev.srcfile.toChars()))
 895                     error(loc, "%s `%s` from file %s conflicts with another module %s from file %s", kind, toPrettyChars, srcname, mprev.toChars(), mprev.srcfile.toChars());
 896                 else if (isRoot() && mprev.isRoot())
 897                     error(loc, "%s `%s` from file %s is specified twice on the command line", kind, toPrettyChars, srcname);
 898                 else
 899                     error(loc, "%s `%s` from file %s must be imported with 'import %s;'", kind, toPrettyChars, srcname, toPrettyChars());
 900                 // https://issues.dlang.org/show_bug.cgi?id=14446
 901                 // Return previously parsed module to avoid AST duplication ICE.
 902                 return mprev;
 903             }
 904             else if (Package pkg = prev.isPackage())
 905             {
 906                 // 'package.d' loaded after a previous 'Package' insertion
 907                 if (isPackageFile)
 908                     amodules.push(this); // Add to global array of all modules
 909                 else
 910                     error(md ? md.loc : loc, "%s `%s` from file %s conflicts with package name %s", kind, toPrettyChars, srcname, pkg.toChars());
 911             }
 912             else
 913                 assert(global.errors);
 914         }
 915         else
 916         {
 917             // Add to global array of all modules
 918             amodules.push(this);
 919         }
 920         Compiler.onParseModule(this);
 921         return this;
 922     }
 923
 924     /**********************************
 925      * Determine if we need to generate an instance of ModuleInfo
 926      * for this Module.
 927      */
 928     int needModuleInfo()
 929     {
 930         //printf("needModuleInfo() %s, %d, %d\n", toChars(), needmoduleinfo, global.params.cov);
 931         return needmoduleinfo || global.params.cov;
 932     }
 933
 934     /*******************************************
 935      * Print deprecation warning if we're deprecated, when
 936      * this module is imported from scope sc.
 937      *
 938      * Params:
 939      *  sc = the scope into which we are imported
 940      *  loc = the location of the import statement
 941      */
 942     void checkImportDeprecation(const ref Loc loc, Scope* sc)
 943     {
 944         if (md && md.isdeprecated && !sc.isDeprecated)
 945         {
 946             Expression msg = md.msg;
 947             if (StringExp se = msg ? msg.toStringExp() : null)
 948             {
 949                 const slice = se.peekString();
 950                 if (slice.length)
 951                 {
 952                     deprecation(loc, "%s `%s` is deprecated - %.*s", kind, toPrettyChars, cast(int)slice.length, slice.ptr);
 953                     return;
 954                 }
 955             }
 956             deprecation(loc, "%s `%s` is deprecated", kind, toPrettyChars);
 957         }
 958     }
 959
 960     override bool isPackageAccessible(Package p, Visibility visibility, SearchOptFlags flags = SearchOpt.all)
 961     {
 962         if (insearch) // don't follow import cycles
 963             return false;
 964         insearch = true;
 965         scope (exit)
 966             insearch = false;
 967         if (flags & SearchOpt.ignorePrivateImports)
 968             visibility = Visibility(Visibility.Kind.public_); // only consider public imports
 969         return super.isPackageAccessible(p, visibility);
 970     }
 971
 972     override Dsymbol symtabInsert(Dsymbol s)
 973     {
 974         searchCacheIdent = null; // symbol is inserted, so invalidate cache
 975         return Package.symtabInsert(s);
 976     }
 977
 978     extern (D) void deleteObjFile()
 979     {
 980         if (global.params.obj)
 981             File.remove(objfile.toChars());
 982         if (docfile)
 983             File.remove(docfile.toChars());
 984     }
 985
 986     /*******************************************
 987      * Can't run semantic on s now, try again later.
 988      */
 989     extern (D) static void addDeferredSemantic(Dsymbol s)
 990     {
 991         //printf("Module::addDeferredSemantic('%s')\n", s.toChars());
 992         if (!deferred.contains(s))
 993             deferred.push(s);
 994     }
 995
 996     extern (D) static void addDeferredSemantic2(Dsymbol s)
 997     {
 998         //printf("Module::addDeferredSemantic2('%s')\n", s.toChars());
 999         if (!deferred2.contains(s))
1000             deferred2.push(s);
1001     }
1002
1003     extern (D) static void addDeferredSemantic3(Dsymbol s)
1004     {
1005         //printf("Module::addDeferredSemantic3('%s')\n", s.toChars());
1006         if (!deferred.contains(s))
1007             deferred3.push(s);
1008     }
1009
1010     /******************************************
1011      * Run semantic() on deferred symbols.
1012      */
1013     static void runDeferredSemantic()
1014     {
1015         __gshared int nested;
1016         if (nested)
1017             return;
1018         //if (deferred.length) printf("+Module::runDeferredSemantic(), len = %ld\n", deferred.length);
1019         nested++;
1020
1021         size_t len;
1022         do
1023         {
1024             len = deferred.length;
1025             if (!len)
1026                 break;
1027
1028             Dsymbol* todo;
1029             Dsymbol* todoalloc = null;
1030             Dsymbol tmp;
1031             if (len == 1)
1032             {
1033                 todo = &tmp;
1034             }
1035             else
1036             {
1037                 todo = cast(Dsymbol*)Mem.check(malloc(len * Dsymbol.sizeof));
1038                 todoalloc = todo;
1039             }
1040             memcpy(todo, deferred.tdata(), len * Dsymbol.sizeof);
1041             deferred.setDim(0);
1042
1043             foreach (i; 0..len)
1044             {
1045                 Dsymbol s = todo[i];
1046                 s.dsymbolSemantic(null);
1047                 //printf("deferred: %s, parent = %s\n", s.toChars(), s.parent.toChars());
1048             }
1049             //printf("\tdeferred.length = %ld, len = %ld\n", deferred.length, len);
1050             if (todoalloc)
1051                 free(todoalloc);
1052         }
1053         while (deferred.length != len); // while making progress
1054         nested--;
1055         //printf("-Module::runDeferredSemantic(), len = %ld\n", deferred.length);
1056     }
1057
1058     static void runDeferredSemantic2()
1059     {
1060         Module.runDeferredSemantic();
1061
1062         Dsymbols* a = &Module.deferred2;
1063         for (size_t i = 0; i < a.length; i++)
1064         {
1065             Dsymbol s = (*a)[i];
1066             //printf("[%d] %s semantic2a\n", i, s.toPrettyChars());
1067             s.semantic2(null);
1068
1069             if (global.errors)
1070                 break;
1071         }
1072         a.setDim(0);
1073     }
1074
1075     static void runDeferredSemantic3()
1076     {
1077         Module.runDeferredSemantic2();
1078
1079         Dsymbols* a = &Module.deferred3;
1080         for (size_t i = 0; i < a.length; i++)
1081         {
1082             Dsymbol s = (*a)[i];
1083             //printf("[%d] %s semantic3a\n", i, s.toPrettyChars());
1084             s.semantic3(null);
1085
1086             if (global.errors)
1087                 break;
1088         }
1089         a.setDim(0);
1090     }
1091
1092     extern (D) static void clearCache() nothrow
1093     {
1094         foreach (Module m; amodules)
1095             m.searchCacheIdent = null;
1096     }
1097
1098     /************************************
1099      * Recursively look at every module this module imports,
1100      * return true if it imports m.
1101      * Can be used to detect circular imports.
1102      */
1103     int imports(Module m) nothrow
1104     {
1105         //printf("%s Module::imports(%s)\n", toChars(), m.toChars());
1106         version (none)
1107         {
1108             foreach (i, Module mi; aimports)
1109                 printf("\t[%d] %s\n", cast(int) i, mi.toChars());
1110         }
1111         foreach (Module mi; aimports)
1112         {
1113             if (mi == m)
1114                 return true;
1115             if (!mi.insearch)
1116             {
1117                 mi.insearch = true;
1118                 int r = mi.imports(m);
1119                 if (r)
1120                     return r;
1121             }
1122         }
1123         return false;
1124     }
1125
1126     bool isRoot() nothrow
1127     {
1128         return this.importedFrom == this;
1129     }
1130
1131     /// Returns: Whether this module is in the `core` package and has name `ident`
1132     bool isCoreModule(Identifier ident) nothrow
1133     {
1134         return this.ident == ident && parent && parent.ident == Id.core && !parent.parent;
1135     }
1136
1137     // Back end
1138     int doppelganger; // sub-module
1139     Symbol* cov; // private uint[] __coverage;
1140     uint[] covb; // bit array of valid code line numbers
1141     Symbol* sictor; // module order independent constructor
1142     Symbol* sctor; // module constructor
1143     Symbol* sdtor; // module destructor
1144     Symbol* ssharedctor; // module shared constructor
1145     Symbol* sshareddtor; // module shared destructor
1146     Symbol* stest; // module unit test
1147     Symbol* sfilename; // symbol for filename
1148
1149     uint[uint] ctfe_cov; /// coverage information from ctfe execution_count[line]
1150
1151     override inout(Module) isModule() inout nothrow
1152     {
1153         return this;
1154     }
1155
1156     override void accept(Visitor v)
1157     {
1158         v.visit(this);
1159     }
1160
1161     /***********************************************
1162      * Writes this module's fully-qualified name to buf
1163      * Params:
1164      *    buf = The buffer to write to
1165      */
1166     void fullyQualifiedName(ref OutBuffer buf) nothrow
1167     {
1168         buf.writestring(ident.toString());
1169
1170         for (auto package_ = parent; package_ !is null; package_ = package_.parent)
1171         {
1172             buf.prependstring(".");
1173             buf.prependstring(package_.ident.toChars());
1174         }
1175     }
1176
1177     /** Lazily initializes and returns the escape table.
1178     Turns out it eats a lot of memory.
1179     */
1180     extern(D) Escape* escapetable() nothrow
1181     {
1182         if (!_escapetable)
1183             _escapetable = new Escape();
1184         return _escapetable;
1185     }
1186
1187     /****************************
1188      * A Singleton that loads core.stdc.config
1189      * Returns:
1190      *  Module of core.stdc.config, null if couldn't find it
1191      */
1192     extern (D) static Module loadCoreStdcConfig()
1193     {
1194         __gshared Module core_stdc_config;
1195         auto pkgids = new Identifier[2];
1196         pkgids[0] = Id.core;
1197         pkgids[1] = Id.stdc;
1198         return loadModuleFromLibrary(core_stdc_config, pkgids, Id.config);
1199     }
1200
1201     /****************************
1202      * A Singleton that loads core.atomic
1203      * Returns:
1204      *  Module of core.atomic, null if couldn't find it
1205      */
1206     extern (D) static Module loadCoreAtomic()
1207     {
1208         __gshared Module core_atomic;
1209         auto pkgids = new Identifier[1];
1210         pkgids[0] = Id.core;
1211         return loadModuleFromLibrary(core_atomic, pkgids, Id.atomic);
1212     }
1213
1214     /****************************
1215      * A Singleton that loads std.math
1216      * Returns:
1217      *  Module of std.math, null if couldn't find it
1218      */
1219     extern (D) static Module loadStdMath()
1220     {
1221         __gshared Module std_math;
1222         auto pkgids = new Identifier[1];
1223         pkgids[0] = Id.std;
1224         return loadModuleFromLibrary(std_math, pkgids, Id.math);
1225     }
1226
1227     /**********************************
1228      * Load a Module from the library.
1229      * Params:
1230      *  mod = cached return value of this call
1231      *  pkgids = package identifiers
1232      *  modid = module id
1233      * Returns:
1234      *  Module loaded, null if cannot load it
1235      */
1236     extern (D) private static Module loadModuleFromLibrary(ref Module mod, Identifier[] pkgids, Identifier modid)
1237     {
1238         if (mod)
1239             return mod;
1240
1241         auto imp = new Import(Loc.initial, pkgids[], modid, null, true);
1242         // Module.load will call fatal() if there's no module available.
1243         // Gag the error here, pushing the error handling to the caller.
1244         const errors = global.startGagging();
1245         imp.load(null);
1246         if (imp.mod)
1247         {
1248             imp.mod.importAll(null);
1249             imp.mod.dsymbolSemantic(null);
1250         }
1251         global.endGagging(errors);
1252         mod = imp.mod;
1253         return mod;
1254     }
1255 }
1256
1257 /***********************************************************
1258  */
1259 extern (C++) struct ModuleDeclaration
1260 {
1261     Loc loc;
1262     Identifier id;
1263     Identifier[] packages;  // array of Identifier's representing packages
1264     bool isdeprecated;      // if it is a deprecated module
1265     Expression msg;
1266
1267     extern (D) this(const ref Loc loc, Identifier[] packages, Identifier id, Expression msg, bool isdeprecated) @safe
1268     {
1269         this.loc = loc;
1270         this.packages = packages;
1271         this.id = id;
1272         this.msg = msg;
1273         this.isdeprecated = isdeprecated;
1274     }
1275
1276     extern (C++) const(char)* toChars() const @safe
1277     {
1278         OutBuffer buf;
1279         foreach (pid; packages)
1280         {
1281             buf.writestring(pid.toString());
1282             buf.writeByte('.');
1283         }
1284         buf.writestring(id.toString());
1285         return buf.extractChars();
1286     }
1287
1288     /// Provide a human readable representation
1289     extern (D) const(char)[] toString() const
1290     {
1291         return this.toChars().toDString;
1292     }
1293 }
1294
1295 /****************************************
1296  * Create array of the local classes in the Module, suitable
1297  * for inclusion in ModuleInfo
1298  * Params:
1299  *      mod = the Module
1300  *      aclasses = array to fill in
1301  * Returns: array of local classes
1302  */
1303 extern (C++) void getLocalClasses(Module mod, ref ClassDeclarations aclasses)
1304 {
1305     //printf("members.length = %d\n", mod.members.length);
1306     int pushAddClassDg(size_t n, Dsymbol sm)
1307     {
1308         if (!sm)
1309             return 0;
1310
1311         if (auto cd = sm.isClassDeclaration())
1312         {
1313             // compatibility with previous algorithm
1314             if (cd.parent && cd.parent.isTemplateMixin())
1315                 return 0;
1316
1317             if (cd.classKind != ClassKind.objc)
1318                 aclasses.push(cd);
1319         }
1320         return 0;
1321     }
1322
1323     _foreach(null, mod.members, &pushAddClassDg);
1324 }
1325
1326
1327 alias ForeachDg = int delegate(size_t idx, Dsymbol s);
1328
1329 /***************************************
1330  * Expands attribute declarations in members in depth first
1331  * order. Calls dg(size_t symidx, Dsymbol *sym) for each
1332  * member.
1333  * If dg returns !=0, stops and returns that value else returns 0.
1334  * Use this function to avoid the O(N + N^2/2) complexity of
1335  * calculating dim and calling N times getNth.
1336  * Returns:
1337  *  last value returned by dg()
1338  */
1339 int _foreach(Scope* sc, Dsymbols* members, scope ForeachDg dg, size_t* pn = null)
1340 {
1341     assert(dg);
1342     if (!members)
1343         return 0;
1344     size_t n = pn ? *pn : 0; // take over index
1345     int result = 0;
1346     foreach (size_t i; 0 .. members.length)
1347     {
1348         import dmd.attrib : AttribDeclaration;
1349         import dmd.dtemplate : TemplateMixin;
1350
1351         Dsymbol s = (*members)[i];
1352         if (AttribDeclaration a = s.isAttribDeclaration())
1353             result = _foreach(sc, a.include(sc), dg, &n);
1354         else if (TemplateMixin tm = s.isTemplateMixin())
1355             result = _foreach(sc, tm.members, dg, &n);
1356         else if (s.isTemplateInstance())
1357         {
1358         }
1359         else if (s.isUnitTestDeclaration())
1360         {
1361         }
1362         else
1363             result = dg(n++, s);
1364         if (result)
1365             break;
1366     }
1367     if (pn)
1368         *pn = n; // update index
1369     return result;
1370 }
1371
1372 /**
1373  * Process the content of a source file
1374  *
1375  * Attempts to find which encoding it is using, if it has BOM,
1376  * and then normalize the source to UTF-8. If no encoding is required,
1377  * a slice of `src` will be returned without extra allocation.
1378  *
1379  * Params:
1380  *  src = Content of the source file to process
1381  *  mod = Module matching `src`, used for error handling
1382  *
1383  * Returns:
1384  *   UTF-8 encoded variant of `src`, stripped of any BOM,
1385  *   or `null` if an error happened.
1386  */
1387 private const(char)[] processSource (const(ubyte)[] src, Module mod)
1388 {
1389     enum SourceEncoding { utf16, utf32}
1390     enum Endian { little, big}
1391
1392     /*
1393      * Convert a buffer from UTF32 to UTF8
1394      * Params:
1395      *    Endian = is the buffer big/little endian
1396      *    buf = buffer of UTF32 data
1397      * Returns:
1398      *    input buffer reencoded as UTF8
1399      */
1400
1401     char[] UTF32ToUTF8(Endian endian)(const(char)[] buf)
1402     {
1403         static if (endian == Endian.little)
1404             alias readNext = Port.readlongLE;
1405         else
1406             alias readNext = Port.readlongBE;
1407
1408         if (buf.length & 3)
1409         {
1410             .error(mod.loc, "%s `%s` odd length of UTF-32 char source %llu",
1411                 mod.kind, mod.toPrettyChars, cast(ulong) buf.length);
1412             return null;
1413         }
1414
1415         const (uint)[] eBuf = cast(const(uint)[])buf;
1416
1417         OutBuffer dbuf;
1418         dbuf.reserve(eBuf.length);
1419
1420         foreach (i; 0 .. eBuf.length)
1421         {
1422             const u = readNext(&eBuf[i]);
1423             if (u & ~0x7F)
1424             {
1425                 if (u > 0x10FFFF)
1426                 {
1427                     .error(mod.loc, "%s `%s` UTF-32 value %08x greater than 0x10FFFF", mod.kind, mod.toPrettyChars, u);
1428                     return null;
1429                 }
1430                 dbuf.writeUTF8(u);
1431             }
1432             else
1433                 dbuf.writeByte(u);
1434         }
1435         dbuf.writeByte(0); //add null terminator
1436         return dbuf.extractSlice();
1437     }
1438
1439     /*
1440      * Convert a buffer from UTF16 to UTF8
1441      * Params:
1442      *    Endian = is the buffer big/little endian
1443      *    buf = buffer of UTF16 data
1444      * Returns:
1445      *    input buffer reencoded as UTF8
1446      */
1447
1448     char[] UTF16ToUTF8(Endian endian)(const(char)[] buf)
1449     {
1450         static if (endian == Endian.little)
1451             alias readNext = Port.readwordLE;
1452         else
1453             alias readNext = Port.readwordBE;
1454
1455         if (buf.length & 1)
1456         {
1457             .error(mod.loc, "%s `%s` odd length of UTF-16 char source %llu", mod.kind, mod.toPrettyChars, cast(ulong) buf.length);
1458             return null;
1459         }
1460
1461         const (ushort)[] eBuf = cast(const(ushort)[])buf;
1462
1463         OutBuffer dbuf;
1464         dbuf.reserve(eBuf.length);
1465
1466         //i will be incremented in the loop for high codepoints
1467         foreach (ref i; 0 .. eBuf.length)
1468         {
1469             uint u = readNext(&eBuf[i]);
1470             if (u & ~0x7F)
1471             {
1472                 if (0xD800 <= u && u < 0xDC00)
1473                 {
1474                     i++;
1475                     if (i >= eBuf.length)
1476                     {
1477                         .error(mod.loc, "%s `%s` surrogate UTF-16 high value %04x at end of file", mod.kind, mod.toPrettyChars, u);
1478                         return null;
1479                     }
1480                     const u2 = readNext(&eBuf[i]);
1481                     if (u2 < 0xDC00 || 0xE000 <= u2)
1482                     {
1483                         .error(mod.loc, "%s `%s` surrogate UTF-16 low value %04x out of range", mod.kind, mod.toPrettyChars, u2);
1484                         return null;
1485                     }
1486                     u = (u - 0xD7C0) << 10;
1487                     u |= (u2 - 0xDC00);
1488                 }
1489                 else if (u >= 0xDC00 && u <= 0xDFFF)
1490                 {
1491                     .error(mod.loc, "%s `%s` unpaired surrogate UTF-16 value %04x", mod.kind, mod.toPrettyChars, u);
1492                     return null;
1493                 }
1494                 else if (u == 0xFFFE || u == 0xFFFF)
1495                 {
1496                     .error(mod.loc, "%s `%s` illegal UTF-16 value %04x", mod.kind, mod.toPrettyChars, u);
1497                     return null;
1498                 }
1499                 dbuf.writeUTF8(u);
1500             }
1501             else
1502                 dbuf.writeByte(u);
1503         }
1504         dbuf.writeByte(0); //add a terminating null byte
1505         return dbuf.extractSlice();
1506     }
1507
1508     const(char)[] buf = cast(const(char)[]) src;
1509
1510     // Assume the buffer is from memory and has not be read from disk. Assume UTF-8.
1511     if (buf.length < 2)
1512         return buf;
1513
1514     /* Convert all non-UTF-8 formats to UTF-8.
1515      * BOM : https://www.unicode.org/faq/utf_bom.html
1516      * 00 00 FE FF  UTF-32BE, big-endian
1517      * FF FE 00 00  UTF-32LE, little-endian
1518      * FE FF        UTF-16BE, big-endian
1519      * FF FE        UTF-16LE, little-endian
1520      * EF BB BF     UTF-8
1521      */
1522     if (buf[0] == 0xFF && buf[1] == 0xFE)
1523     {
1524         if (buf.length >= 4 && buf[2] == 0 && buf[3] == 0)
1525             return UTF32ToUTF8!(Endian.little)(buf[4 .. $]);
1526         return UTF16ToUTF8!(Endian.little)(buf[2 .. $]);
1527     }
1528
1529     if (buf[0] == 0xFE && buf[1] == 0xFF)
1530         return UTF16ToUTF8!(Endian.big)(buf[2 .. $]);
1531
1532     if (buf.length >= 4 && buf[0] == 0 && buf[1] == 0 && buf[2] == 0xFE && buf[3] == 0xFF)
1533         return UTF32ToUTF8!(Endian.big)(buf[4 .. $]);
1534
1535     if (buf.length >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF)
1536         return buf[3 .. $];
1537
1538     /* There is no BOM. Make use of Arcane Jill's insight that
1539      * the first char of D source must be ASCII to
1540      * figure out the encoding.
1541      */
1542     if (buf.length >= 4 && buf[1] == 0 && buf[2] == 0 && buf[3] == 0)
1543         return UTF32ToUTF8!(Endian.little)(buf);
1544     if (buf.length >= 4 && buf[0] == 0 && buf[1] == 0 && buf[2] == 0)
1545         return UTF32ToUTF8!(Endian.big)(buf);
1546     // try to check for UTF-16
1547     if (buf.length >= 2 && buf[1] == 0)
1548         return UTF16ToUTF8!(Endian.little)(buf);
1549     if (buf[0] == 0)
1550         return UTF16ToUTF8!(Endian.big)(buf);
1551
1552     // It's UTF-8
1553     if (buf[0] >= 0x80)
1554     {
1555         auto loc = mod.getLoc();
1556         .error(loc, "%s `%s` source file must start with BOM or ASCII character, not \\x%02X", mod.kind, mod.toPrettyChars, buf[0]);
1557         return null;
1558     }
1559
1560     return buf;
1561 }
1562
1563 /*******************************************
1564  * Look for member of the form:
1565  *      const(MemberInfo)[] getMembers(string);
1566  * Returns NULL if not found
1567  */
1568 extern(C++) FuncDeclaration findGetMembers(ScopeDsymbol dsym)
1569 {
1570     import dmd.opover : search_function;
1571     Dsymbol s = search_function(dsym, Id.getmembers);
1572     FuncDeclaration fdx = s ? s.isFuncDeclaration() : null;
1573     version (none)
1574     {
1575         // Finish
1576         __gshared TypeFunction tfgetmembers;
1577         if (!tfgetmembers)
1578         {
1579             Scope sc;
1580             sc.eSink = global.errorSink;
1581             auto parameters = new Parameters();
1582             Parameters* p = new Parameter(STC.in_, Type.tchar.constOf().arrayOf(), null, null);
1583             parameters.push(p);
1584             Type tret = null;
1585             TypeFunction tf = new TypeFunction(parameters, tret, VarArg.none, LINK.d);
1586             tfgetmembers = tf.dsymbolSemantic(Loc.initial, &sc).isTypeFunction();
1587         }
1588         if (fdx)
1589             fdx = fdx.overloadExactMatch(tfgetmembers);
1590     }
1591     if (fdx && fdx.isVirtual())
1592         fdx = null;
1593     return fdx;
1594 }