acceptance-tests/profiler-stress/runner.cs

   1 using System;
   2 using System.Diagnostics;
   3 using System.IO;
   4 using System.Linq;
   5 using Newtonsoft.Json;
   6
   7 // Shut up CLS compliance warnings from Json.NET.
   8 [assembly: CLSCompliant (true)]
   9
  10 namespace Mono.Profiling.Tests.Stress {
  11
  12         // https://github.com/xamarin/benchmarker/blob/master/tools/libdbmodel/Benchmark.cs
  13         class Benchmark {
  14                 public string Name { get; set; }
  15                 public string TestDirectory { get; set; }
  16                 public bool OnlyExplicit { get; set; }
  17                 public string[] CommandLine { get; set; }
  18                 public string[] ClientCommandLine { get; set; }
  19                 public string[] AOTAssemblies { get; set; }
  20
  21                 public static Benchmark Load (string file)
  22                 {
  23                         return JsonConvert.DeserializeObject<Benchmark> (File.ReadAllText (file));
  24                 }
  25         }
  26
  27         static class Program {
  28
  29                 static int Main ()
  30                 {
  31                         var depDir = Path.Combine ("..", "external", "benchmarker");
  32                         var benchDir = Path.Combine (depDir, "benchmarks");
  33                         var testDir = Path.Combine (depDir, "tests");
  34
  35                         var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
  36                                          .Select (Benchmark.Load)
  37                                          .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null)
  38                                          .OrderBy (b => b.Name)
  39                                          .ToArray ();
  40
  41                         var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
  42                         var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));
  43
  44                         var rand = new Random ();
  45                         var cpus = Environment.ProcessorCount;
  46
  47                         var successes = 0;
  48                         var failures = 0;
  49
  50                         var sw = Stopwatch.StartNew ();
  51
  52                         for (var i = 0; i < benchmarks.Length; i++) {
  53                                 var bench = benchmarks [i];
  54
  55                                 var sampleFreq = rand.Next (0, 1001);
  56                                 var sampleMode = rand.Next (0, 2) == 1 ? "real" : "process";
  57                                 var maxSamples = rand.Next (0, cpus * 2000 + 1);
  58                                 var heapShotFreq = rand.Next (0, 11);
  59                                 var maxFrames = rand.Next (0, 33);
  60                                 var allocMode = rand.Next (0, 2) == 1 ? "alloc" : "noalloc";
  61
  62                                 var profOptions = $"sample=cycles/{sampleFreq},sampling-{sampleMode},maxsamples={maxSamples},heapshot={heapShotFreq}gc,maxframes={maxFrames},{allocMode},output=/dev/null";
  63
  64                                 var info = new ProcessStartInfo {
  65                                         UseShellExecute = false,
  66                                         WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
  67                                         FileName = monoPath,
  68                                         Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
  69                                 };
  70
  71                                 info.EnvironmentVariables.Clear ();
  72                                 info.EnvironmentVariables.Add ("MONO_PATH", classDir);
  73
  74                                 var progress = $"({i + 1}/{benchmarks.Length})";
  75
  76                                 Console.ForegroundColor = ConsoleColor.Blue;
  77                                 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
  78                                 Console.ResetColor ();
  79
  80                                 var sw2 = Stopwatch.StartNew ();
  81
  82                                 using (var proc = Process.Start (info)) {
  83                                         proc.WaitForExit ();
  84                                         sw2.Stop ();
  85
  86                                         Console.WriteLine ();
  87
  88                                         if (proc.ExitCode != 0)
  89                                                 failures++;
  90                                         else
  91                                                 successes++;
  92
  93                                         Console.ForegroundColor = proc.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
  94                                         Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {sw2.Elapsed.ToString ("G")} and exited with code: {proc.ExitCode}");
  95                                         Console.ResetColor ();
  96                                 }
  97                         }
  98
  99                         sw.Stop ();
 100
 101                         Console.ForegroundColor = failures != 0 ? ConsoleColor.Red : ConsoleColor.Green;
 102                         Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{benchmarks.Length} passing tests");
 103                         Console.ResetColor ();
 104
 105                         return failures;
 106                 }
 107         }
 108 }