rgdc.d

   1 /*
   2  *  Copyright (C) 2008 by Andrei Alexandrescu
   3  *  Written by Andrei Alexandrescu, www.erdani.org
   4  *  Based on an idea by Georg Wrede
   5  *  Featuring improvements suggested by Christopher Wright
   6  *  Windows port using bug fixes and suggestions by Adam Ruppe
   7  *
   8  * Distributed under the Boost Software License, Version 1.0.
   9  *    (See accompanying file LICENSE_1_0.txt or copy at
  10  *          http://www.boost.org/LICENSE_1_0.txt)
  11  */
  12 import
  13   std.c.stdlib,
  14   std.algorithm,
  15   std.array,
  16   std.datetime,
  17   std.digest.md,
  18   std.exception,
  19   std.file,
  20   std.getopt,
  21   std.parallelism,
  22   std.path,
  23   std.process,
  24   std.range,
  25   std.regex,
  26   std.stdio,
  27   std.string,
  28   std.typetuple;
  29
  30
  31 private enum objExt = ".o";
  32 private enum binExt = "";
  33 private enum libExt = ".a";
  34 private enum altDirSeparator = "";
  35
  36 private bool chatty, buildOnly, dryRun, force;
  37 private string exe;
  38 private string[] exclusions = ["std", "etc", "core", "tango"]; // packages that are to be excluded
  39
  40
  41 private string compilerBinary = "gdc";
  42 private string compilerExtraFlags = "-pipe";
  43
  44
  45 private void yap(size_t line=__LINE__, T...) (auto ref T stuff) {
  46   if (!chatty) return;
  47   debug stderr.writeln(line, ": ", stuff);
  48   else stderr.writeln(stuff);
  49 }
  50
  51
  52 private string[] includeDirs; // -I
  53 private string[] importDirs; // -J
  54
  55
  56 private string expandString (string s) {
  57   auto pos = s.indexOf("$(");
  58   if (pos >= 0) {
  59     string res;
  60     while (pos >= 0) {
  61       res ~= s[0..pos];
  62       s = s[pos+2..$];
  63       auto epos = s.indexOf(')');
  64       if (epos < 1) throw new Exception("invalid envvar name");
  65       auto v = environment.get(s[0..epos], "");
  66       res ~= v;
  67       s = s[epos+1..$];
  68       pos = s.indexOf("$(");
  69     }
  70     res ~= s;
  71     return res;
  72   }
  73   return s;
  74 }
  75
  76
  77 private void readRC (string fname) {
  78   auto reader = File(fname);
  79   scope(exit) collectException(reader.close()); // don't care for errors
  80   foreach (string line; lines(reader)) {
  81     if (line.startsWith("include=")) {
  82       auto val = expandString(line[8..$].strip);
  83       if (val.length > 0) {
  84         val = "-I"~val;
  85         if (!includeDirs.canFind(val)) includeDirs ~= val;
  86       }
  87     } else if (line.startsWith("import=")) {
  88       auto val = expandString(line[7..$].strip);
  89       if (val.length > 0) {
  90         val = "-J"~val;
  91         if (!importDirs.canFind(val)) importDirs ~= val;
  92       }
  93     }
  94   }
  95 }
  96
  97
  98 private File lockFile;
  99
 100 private void lockWorkPath (string workPath) {
 101   string lockFileName = buildPath(workPath, "rgdc.lock");
 102   if (!dryRun) lockFile.open(lockFileName, "w");
 103   yap("lock ", lockFile.name);
 104   if (!dryRun) lockFile.lock();
 105 }
 106
 107
 108 private void unlockWorkPath () {
 109   yap("unlock ", lockFile.name);
 110   if (!dryRun) {
 111     lockFile.unlock();
 112     lockFile.close();
 113   }
 114 }
 115
 116
 117 private bool inALibrary (string source, string object) {
 118   if (object.endsWith(".di") || source == "object" || source == "gcstats") return true;
 119   foreach (string exclusion; exclusions) if (source.startsWith(exclusion~'.')) return true;
 120   return false;
 121 }
 122
 123
 124 private @property string myOwnTmpDir () {
 125   import core.sys.posix.unistd;
 126   auto tmpRoot = format("/tmp/.rgdc-%d", getuid());
 127   yap("mkdirRecurse ", tmpRoot);
 128   if (!dryRun) mkdirRecurse(tmpRoot);
 129   return tmpRoot;
 130 }
 131
 132
 133 private string getWorkPath (in string root, in string[] compilerFlags) {
 134   static string workPath;
 135   if (workPath) return workPath;
 136   enum string[] irrelevantSwitches = ["--help", "-ignore", "-quiet", "-v"];
 137   MD5 context;
 138   context.start();
 139   context.put(getcwd().representation);
 140   context.put(root.representation);
 141   foreach (flag; compilerFlags) {
 142     if (irrelevantSwitches.canFind(flag)) continue;
 143     context.put(flag.representation);
 144   }
 145   auto digest = context.finish();
 146   string hash = toHexString(digest);
 147   const tmpRoot = myOwnTmpDir;
 148   workPath = buildPath(tmpRoot, "rgdc-"~baseName(root)~'-'~hash);
 149   yap("mkdirRecurse ", workPath);
 150   if (!dryRun) mkdirRecurse(workPath);
 151   return workPath;
 152 }
 153
 154
 155 // Rebuild the executable fullExe starting from modules in myDeps
 156 // passing the compiler flags compilerFlags. Generates one large
 157 // object file.
 158 private int rebuild (string root, string fullExe,
 159                      string workDir, string objDir, in string[string] myDeps,
 160                      string[] compilerFlags, bool addStubMain)
 161 {
 162   // Delete the old executable before we start building.
 163   yap("stat ", fullExe);
 164   if (!dryRun && exists(fullExe)) {
 165     yap("rm ", fullExe);
 166     try {
 167       remove(fullExe);
 168     } catch (FileException e) {
 169       // This can occur on Windows if the executable is locked.
 170       // Although we can't delete the file, we can still rename it.
 171       auto oldExe = "%s.%s-%s.old".format(fullExe, Clock.currTime.stdTime, thisProcessID);
 172       yap("mv ", fullExe, " ", oldExe);
 173       rename(fullExe, oldExe);
 174     }
 175   }
 176   auto fullExeTemp = fullExe~".tmp";
 177
 178   string[] buildTodo () {
 179     string outExe = (fullExeTemp[0] != '/' ? objDir~"/"~fullExeTemp : fullExeTemp);
 180     auto todo =
 181       compilerFlags~
 182       ["-o", outExe]~
 183       ["-I"~dirName(root)]~
 184       ["-J"~dirName(root)]~
 185       includeDirs~
 186       importDirs/*~
 187       [root]*/;
 188     foreach (k, objectFile; myDeps) if (objectFile !is null) todo ~= [k];
 189     // Need to add void main(){}?
 190     if (addStubMain) {
 191       auto stubMain = buildPath(myOwnTmpDir, "stubmain.d");
 192       std.file.write(stubMain, "void main(){}");
 193       todo ~= [stubMain];
 194     }
 195     return todo;
 196   }
 197   auto todo = buildTodo();
 198   // Different shells and OS functions have different limits,
 199   // but 1024 seems to be the smallest maximum outside of MS-DOS.
 200   enum maxLength = 1024;
 201   auto commandLength = escapeShellCommand(todo).length;
 202   if (commandLength+compilerBinary.length+compilerExtraFlags.length > 32760) throw new Exception("SHIT!");
 203   immutable result = run([compilerBinary, compilerExtraFlags]~todo);
 204   if (result) {
 205     // build failed
 206     if (exists(fullExeTemp)) remove(fullExeTemp);
 207     return result;
 208   }
 209   // clean up the dir containing the object file, just not in dry
 210   // run mode because we haven't created any!
 211   if (!dryRun) {
 212     yap("stat ", objDir);
 213     if (objDir.exists && objDir.startsWith(workDir)) {
 214       yap("rmdirRecurse ", objDir);
 215       // We swallow the exception because of a potential race: two
 216       // concurrently-running scripts may attempt to remove this
 217       // directory. One will fail.
 218       collectException(rmdirRecurse(objDir));
 219     }
 220     yap("mv ", fullExeTemp, " ", fullExe);
 221     rename(fullExeTemp, fullExe);
 222   }
 223   return 0;
 224 }
 225
 226
 227 // Run a program optionally writing the command line first
 228 // If "replace" is true and the OS supports it, replace the current process.
 229 private int run (string[] args, string output=null, bool replace=false) {
 230   import std.conv;
 231   yap(replace ? "exec " : "spawn ", args.text);
 232   if (dryRun) return 0;
 233   if (replace && !output) {
 234     import std.c.process;
 235     auto argv = args.map!toStringz.chain(null.only).array;
 236     return execv(argv[0], argv.ptr);
 237   }
 238   File outputFile;
 239   if (output) {
 240     outputFile = File(output, "wb");
 241   } else {
 242     outputFile = stdout;
 243   }
 244   auto process = spawnProcess(args, stdin, outputFile);
 245   return process.wait();
 246 }
 247
 248
 249 private int runSilent (string[] args) {
 250   import std.conv;
 251   yap("spawn ", args.text);
 252   if (dryRun) return 0;
 253   auto process = spawnProcess(args);
 254   return process.wait();
 255 }
 256
 257
 258 private int exec (string[] args) {
 259   return run(args, null, true);
 260 }
 261
 262
 263 // Given module rootModule, returns a mapping of all dependees .d
 264 // source filenames to their corresponding .o files sitting in
 265 // directory workDir. The mapping is obtained by running dmd -v against
 266 // rootModule.
 267 private string[string] getDependencies (string rootModule, string workDir, string objDir, string[] compilerFlags) {
 268   immutable depsFilename = buildPath(workDir, "rgdc.deps");
 269
 270   string[string] readDepsFile (/*string depsFilename, string objDir="."*/) {
 271     string d2obj (string dfile) { return buildPath(objDir, dfile.baseName.chomp(".d")~objExt); }
 272
 273     string findLib (string libName) {
 274       // This can't be 100% precise without knowing exactly where the linker
 275       // will look for libraries (which requires, but is not limited to,
 276       // parsing the linker's command line (as specified in dmd.conf/sc.ini).
 277       // Go for best-effort instead.
 278       string[] dirs = ["."];
 279       foreach (envVar; ["LIB", "LIBRARY_PATH", "LD_LIBRARY_PATH"]) dirs ~= environment.get(envVar, "").split(pathSeparator);
 280       string[] names = ["lib"~libName~".so", "lib"~libName~".a"];
 281       dirs ~= ["/lib", "/usr/lib", "/usr/local/lib"];
 282       foreach (dir; dirs) {
 283         foreach (name; names) {
 284           auto path = buildPath(dir, name);
 285           if (path.exists) {
 286             if (path.extension == ".so") return "-l"~path.baseName.stripExtension[3..$];
 287             return absolutePath(path);
 288           }
 289         }
 290       }
 291       return null;
 292     }
 293
 294     yap("read ", depsFilename);
 295     auto depsReader = File(depsFilename);
 296     scope(exit) collectException(depsReader.close()); // don't care for errors
 297     // Fetch all dependencies and append them to myDeps
 298     static auto modInfoRE = ctRegex!(r"^(.+?)\s+\(([^)]+)\)");
 299     //std.stdio (/opt/gdc/include/d/4.9.1/std/stdio.d) : private : object (/opt/gdc/include/d/4.9.1/object.di)
 300     static auto modImportRE = ctRegex!(r"^(.+?)\s+\(([^)]+)\)\s+:\s+string\s+:\s+[^(]+\(([^)]+)\)");
 301     //hello (hello.d) : string : zm (/mnt/tigerclaw/D/prj/rgdc/rgdc_native/zm)
 302     string[string] result;
 303     foreach (string line; lines(depsReader)) {
 304       string modName, modPath, filePath;
 305       auto modImportMatch = match(line, modImportRE);
 306       if (modImportMatch.empty) {
 307         auto modInfoMatch = match(line, modInfoRE);
 308         if (modInfoMatch.empty) continue;
 309         auto modInfoCaps = modInfoMatch.captures;
 310         // [1]: module name
 311         // [2]: module path
 312         modName = modInfoCaps[1];
 313         modPath = modInfoCaps[2];
 314       } else {
 315         auto modImportCaps = modImportMatch.captures;
 316         // [1]: module name
 317         // [2]: module path
 318         // [3]: file path
 319         modName = modImportCaps[1];
 320         modPath = modImportCaps[2];
 321         filePath = modImportCaps[3];
 322       }
 323       if (filePath.length) result[filePath] = null;
 324       if (inALibrary(modName, modPath)) continue;
 325       result[modPath] = d2obj(modPath);
 326     }
 327     return result;
 328   }
 329
 330   // Check if the old dependency file is fine
 331   if (!force) {
 332     yap("stat ", depsFilename);
 333     if (exists(depsFilename)) {
 334       // See if the deps file is still in good shape
 335       auto deps = readDepsFile();
 336       auto allDeps = chain(rootModule.only, deps.byKey).array;
 337       bool mustRebuildDeps = allDeps.anyNewerThan(timeLastModified(depsFilename));
 338       if (!mustRebuildDeps) return deps; // Cool, we're in good shape
 339       // Fall through to rebuilding the deps file
 340     }
 341   }
 342
 343   immutable rootDir = dirName(rootModule);
 344
 345   // Collect dependencies
 346   auto depsGetter =
 347       // "cd "~shellQuote(rootDir)~" && "
 348       [compilerBinary, compilerExtraFlags]~compilerFlags~
 349       [
 350        "-c",
 351        "-o", "/dev/null",
 352        "-fdeps="~depsFilename,
 353        rootModule,
 354        "-I"~rootDir,
 355        "-J"~rootDir
 356       ]~includeDirs~importDirs;
 357
 358   scope(failure) {
 359     // Delete the deps file on failure, we don't want to be fooled
 360     // by it next time we try
 361     collectException(std.file.remove(depsFilename));
 362   }
 363
 364   immutable depsExitCode = runSilent(depsGetter);
 365   if (depsExitCode) {
 366     stderr.writefln("Failed: %s", depsGetter);
 367     collectException(std.file.remove(depsFilename));
 368     exit(depsExitCode);
 369   }
 370
 371   return (dryRun ? null : readDepsFile());
 372 }
 373
 374
 375 // Is any file newer than the given file?
 376 private bool anyNewerThan (in string[] files, in string file) {
 377   yap("stat ", file);
 378   return files.anyNewerThan(file.timeLastModified);
 379 }
 380
 381
 382 // Is any file newer than the given file?
 383 private bool anyNewerThan (in string[] files, SysTime t) {
 384   // Experimental: running newerThan in separate threads, one per file
 385   if (false) {
 386     foreach (source; files) if (source.newerThan(t)) return true;
 387     return false;
 388   } else {
 389     bool result;
 390     foreach (source; taskPool.parallel(files)) if (!result && source.newerThan(t)) result = true;
 391     return result;
 392   }
 393 }
 394
 395
 396 /*
 397  * If force is true, returns true. Otherwise, if source and target both
 398  * exist, returns true iff source's timeLastModified is strictly greater
 399  * than target's. Otherwise, returns true.
 400  */
 401 private bool newerThan (string source, string target) {
 402   if (force) return true;
 403   yap("stat ", target);
 404   return source.newerThan(timeLastModified(target, SysTime(0)));
 405 }
 406
 407
 408 private bool newerThan (string source, SysTime target) {
 409   if (force) return true;
 410   try {
 411     yap("stat ", source);
 412     return DirEntry(source).timeLastModified > target;
 413   } catch (Exception) {
 414     // File not there, consider it newer
 415     return true;
 416   }
 417 }
 418
 419
 420 private @property string helpString () {
 421     return
 422 "rgdc build "~thisVersion~"
 423 Usage: rgdc [RDMD AND DMD OPTIONS]... program [PROGRAM OPTIONS]...
 424 Builds (with dependents) and runs a D program.
 425 Example: rgdc -release myprog --myprogparm 5
 426
 427 Any option to be passed to the compiler must occur before the program name. In
 428 addition to compiler options, rgdc recognizes the following options:
 429   --build-only       just build the executable, don't run it
 430   --chatty           write compiler commands to stdout before executing them
 431   --compiler=comp    use the specified compiler
 432   --dry-run          do not compile, just show what commands would be run (implies --chatty)
 433   --exclude=package  exclude a package from the build (multiple --exclude allowed)
 434   --force            force a rebuild even if apparently not necessary
 435   --help             this message
 436   --main             add a stub main program to the mix (e.g. for unittesting)
 437   --shebang          rgdc is in a shebang line (put as first argument)
 438 ";
 439 }
 440
 441
 442 private @property string thisVersion () {
 443   enum d = __DATE__;
 444   enum month = d[0..3],
 445       day = d[4] == ' ' ? "0"~d[5] : d[4..6],
 446       year = d[7..$];
 447   enum monthNum
 448       = month == "Jan" ? "01"
 449       : month == "Feb" ? "02"
 450       : month == "Mar" ? "03"
 451       : month == "Apr" ? "04"
 452       : month == "May" ? "05"
 453       : month == "Jun" ? "06"
 454       : month == "Jul" ? "07"
 455       : month == "Aug" ? "08"
 456       : month == "Sep" ? "09"
 457       : month == "Oct" ? "10"
 458       : month == "Nov" ? "11"
 459       : month == "Dec" ? "12"
 460       : "";
 461   static assert(month != "", "Unknown month "~month);
 462   return year[0]~year[1..$]~monthNum~day;
 463 }
 464
 465
 466 private string which (string path) {
 467   yap("which ", path);
 468   if (path.canFind(dirSeparator) || altDirSeparator != "" && path.canFind(altDirSeparator)) return path;
 469   string[] extensions = [""];
 470   foreach (extension; extensions) {
 471     foreach (envPath; environment["PATH"].splitter(pathSeparator)) {
 472       string absPath = buildPath(envPath, path~extension);
 473       yap("stat ", absPath);
 474       if (exists(absPath) && isFile(absPath)) return absPath;
 475     }
 476   }
 477   throw new FileException(path, "File not found in PATH");
 478 }
 479
 480
 481 private size_t indexOfProgram (string[] args) {
 482   foreach(i, arg; args[1..$]) {
 483     if (!arg.startsWith('-', '@') && !arg.endsWith(".o", ".a")) return i+1;
 484   }
 485   return args.length;
 486 }
 487
 488
 489 int main(string[] args) {
 490   //writeln("Invoked with: ", args);
 491   if (args.length > 1 && args[1].startsWith("--shebang ", "--shebang=")) {
 492     // multiple options wrapped in one
 493     auto a = args[1]["--shebang ".length..$];
 494     args = args[0..1]~std.string.split(a)~args[2..$];
 495   }
 496
 497   // Continue parsing the command line; now get rgdc's own arguments
 498   auto programPos = indexOfProgram(args);
 499   assert(programPos > 0);
 500   auto argsBeforeProgram = args[0..programPos];
 501
 502   bool bailout; // bailout set by functions called in getopt if program should exit
 503   bool addStubMain; // set by --main
 504   getopt(argsBeforeProgram,
 505     std.getopt.config.caseSensitive,
 506     std.getopt.config.passThrough,
 507     "build-only", &buildOnly,
 508     "chatty", &chatty,
 509     "compiler", &compilerBinary,
 510     "dry-run", &dryRun,
 511     "exclude", &exclusions,
 512     "force", &force,
 513     "help", { writeln(helpString); bailout = true; },
 514     "main", &addStubMain
 515   );
 516   if (bailout) return 0;
 517   if (dryRun) chatty = true; // dry-run implies chatty
 518
 519   // no code on command line => require a source file
 520   if (programPos == args.length) {
 521     write(helpString);
 522     return 1;
 523   }
 524
 525   auto
 526     root = args[programPos].chomp(".d")~".d",
 527     exeBasename = root.baseName(".d"),
 528     exeDirname = root.dirName,
 529     programArgs = args[programPos+1..$];
 530
 531   assert(argsBeforeProgram.length >= 1);
 532   auto compilerFlags = argsBeforeProgram[1..$];
 533
 534   bool lib = compilerFlags.canFind("-lib");
 535   string outExt = (lib ? libExt : binExt);
 536
 537   //collectException(readRC(thisExePath.setExtension(".rc")));
 538   collectException(readRC("rgdc.rc"));
 539
 540   // --build-only implies the user would like a binary in the program's directory
 541   if (buildOnly && !exe) exe = exeDirname~dirSeparator;
 542
 543   if (exe && exe.endsWith(dirSeparator)) {
 544     // user specified a directory, complete it to a file
 545     exe = buildPath(exe, exeBasename)~outExt;
 546   }
 547
 548   // Compute the object directory and ensure it exists
 549   immutable workDir = getWorkPath(root, compilerFlags);
 550   lockWorkPath(workDir); // will be released by the OS on process exit
 551   string objDir = buildPath(workDir, "objs");
 552   yap("mkdirRecurse ", objDir);
 553   if (!dryRun) mkdirRecurse(objDir);
 554
 555   if (lib) {
 556     // When building libraries, DMD does not generate object files.
 557     // Instead, it uses the -od parameter as the location for the library file.
 558     // Thus, override objDir (which is normally a temporary directory)
 559     // to be the target output directory.
 560     objDir = exe.dirName;
 561   }
 562
 563   // Fetch dependencies
 564   const myDeps = getDependencies(root, workDir, objDir, compilerFlags);
 565
 566   // Compute executable name, check for freshness, rebuild
 567   /*
 568     We need to be careful about using -o. Normally the generated
 569     executable is hidden in the unique directory workDir. But if the
 570     user forces generation in a specific place by using -od or -of,
 571     the time of the binary can't be used to check for freshness
 572     because the user may change e.g. the compile option from one run
 573     to the next, yet the generated binary's datetime stays the
 574     same. In those cases, we'll use a dedicated file called ".built"
 575     and placed in workDir. Upon a successful build, ".built" will be
 576     touched. See also
 577     http://d.puremagic.com/issues/show_bug.cgi?id=4814
 578    */
 579   string buildWitness;
 580   SysTime lastBuildTime = SysTime.min;
 581   if (exe) {
 582     // user-specified exe name
 583     buildWitness = buildPath(workDir, ".built");
 584     if (!exe.newerThan(buildWitness)) {
 585       // Both exe and buildWitness exist, and exe is older than
 586       // buildWitness. This is the only situation in which we
 587       // may NOT need to recompile.
 588       lastBuildTime = buildWitness.timeLastModified(SysTime.min);
 589     }
 590   } else {
 591     exe = buildPath(workDir, exeBasename)~outExt;
 592     buildWitness = exe;
 593     lastBuildTime = buildWitness.timeLastModified(SysTime.min);
 594   }
 595
 596   // Have at it
 597   if (chain(root.only, myDeps.byKey).array.anyNewerThan(lastBuildTime)) {
 598     immutable result = rebuild(root, exe, workDir, objDir, myDeps, compilerFlags, addStubMain);
 599     if (result) return result;
 600     // Touch the build witness to track the build time
 601     if (buildWitness != exe) {
 602       yap("touch ", buildWitness);
 603       std.file.write(buildWitness, "");
 604     }
 605   }
 606
 607   if (buildOnly) {
 608     // Pretty much done!
 609     return 0;
 610   }
 611
 612   // release lock on workDir before launching the user's program
 613   unlockWorkPath();
 614
 615   // run
 616   return exec(exe~programArgs);
 617 }