acceptance-tests/profiler-stress/runner.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.Diagnostics;
   4 using System.Globalization;
   5 using System.IO;
   6 using System.Linq;
   7 using System.Text;
   8 using System.Text.RegularExpressions;
   9 using System.Threading;
  10 using System.Xml;
  11 using Mono.Unix.Native;
  12 using Newtonsoft.Json;
  13
  14 // Shut up CLS compliance warnings from Json.NET.
  15 [assembly: CLSCompliant (true)]
  16
  17 namespace Mono.Profiling.Tests.Stress {
  18
  19         // https://github.com/xamarin/benchmarker/blob/master/tools/libdbmodel/Benchmark.cs
  20         sealed class Benchmark {
  21
  22                 public string Name { get; set; }
  23                 public string TestDirectory { get; set; }
  24                 public bool OnlyExplicit { get; set; }
  25                 public string[] CommandLine { get; set; }
  26                 public string[] ClientCommandLine { get; set; }
  27                 public string[] AOTAssemblies { get; set; }
  28
  29                 public static Benchmark Load (string file)
  30                 {
  31                         return JsonConvert.DeserializeObject<Benchmark> (File.ReadAllText (file));
  32                 }
  33         }
  34
  35         sealed class TestResult {
  36
  37                 public Benchmark Benchmark { get; set; }
  38                 public ProcessStartInfo StartInfo { get; set; }
  39                 public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
  40                 public int? ExitCode { get; set; }
  41                 public StringBuilder StandardOutput { get; set; } = new StringBuilder ();
  42                 public StringBuilder StandardError { get; set; } = new StringBuilder ();
  43         }
  44
  45         static class Program {
  46
  47                 static readonly TimeSpan _timeout = TimeSpan.FromHours (6);
  48
  49                 static string FilterInvalidXmlChars (string text) {
  50                         return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
  51                 }
  52
  53                 static int Main ()
  54                 {
  55                         var depDir = Path.Combine ("..", "external", "benchmarker");
  56                         var benchDir = Path.Combine (depDir, "benchmarks");
  57                         var testDir = Path.Combine (depDir, "tests");
  58
  59                         var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
  60                                          .Select (Benchmark.Load)
  61                                          .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null)
  62                                          .OrderBy (b => b.Name)
  63                                          .ToArray ();
  64
  65                         var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
  66                         var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));
  67
  68                         var rand = new Random ();
  69                         var cpus = Environment.ProcessorCount;
  70
  71                         var results = new List<TestResult> (benchmarks.Length);
  72
  73                         var sw = Stopwatch.StartNew ();
  74
  75                         for (var i = 0; i < benchmarks.Length; i++) {
  76                                 var bench = benchmarks [i];
  77
  78                                 var sampleFreq = rand.Next (0, 1001);
  79                                 var sampleMode = rand.Next (0, 2) == 1 ? "real" : "process";
  80                                 var maxSamples = rand.Next (0, cpus * 2000 + 1);
  81                                 var heapShotFreq = rand.Next (0, 11);
  82                                 var maxFrames = rand.Next (0, 33);
  83                                 var allocMode = rand.Next (0, 2) == 1 ? "alloc" : "noalloc";
  84
  85                                 var profOptions = $"sample=cycles/{sampleFreq},sampling-{sampleMode},maxsamples={maxSamples},heapshot={heapShotFreq}gc,maxframes={maxFrames},{allocMode},output=/dev/null";
  86
  87                                 var info = new ProcessStartInfo {
  88                                         UseShellExecute = false,
  89                                         WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
  90                                         FileName = monoPath,
  91                                         Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
  92                                         RedirectStandardOutput = true,
  93                                         RedirectStandardError = true,
  94                                 };
  95
  96                                 info.EnvironmentVariables.Clear ();
  97                                 info.EnvironmentVariables.Add ("MONO_PATH", classDir);
  98
  99                                 var progress = $"({i + 1}/{benchmarks.Length})";
 100
 101                                 Console.ForegroundColor = ConsoleColor.Blue;
 102                                 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
 103                                 Console.ResetColor ();
 104
 105                                 var result = new TestResult {
 106                                         Benchmark = bench,
 107                                         StartInfo = info,
 108                                 };
 109
 110                                 using (var proc = new Process ()) {
 111                                         proc.StartInfo = info;
 112
 113                                         proc.OutputDataReceived += (sender, args) => {
 114                                                 if (args.Data != null)
 115                                                         result.StandardOutput.AppendLine (args.Data);
 116                                         };
 117
 118                                         proc.ErrorDataReceived += (sender, args) => {
 119                                                 if (args.Data != null)
 120                                                         result.StandardError.AppendLine (args.Data);
 121                                         };
 122
 123                                         result.Stopwatch.Start ();
 124
 125                                         proc.Start ();
 126
 127                                         proc.BeginOutputReadLine ();
 128                                         proc.BeginErrorReadLine ();
 129
 130                                         if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
 131                                                 // Force a thread dump.
 132                                                 Syscall.kill (proc.Id, Signum.SIGQUIT);
 133                                                 Thread.Sleep (1000);
 134
 135                                                 try {
 136                                                         proc.Kill ();
 137                                                 } catch (Exception) {
 138                                                 }
 139                                         } else
 140                                                 result.ExitCode = proc.ExitCode;
 141
 142                                         result.Stopwatch.Stop ();
 143                                 }
 144
 145                                 var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";
 146
 147                                 Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
 148                                 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
 149                                 Console.ResetColor ();
 150
 151                                 if (result.ExitCode != 0) {
 152                                         Console.ForegroundColor = ConsoleColor.Red;
 153                                         Console.WriteLine ("===== stdout =====");
 154                                         Console.ResetColor ();
 155
 156                                         Console.WriteLine (result.StandardOutput.ToString ());
 157
 158                                         Console.ForegroundColor = ConsoleColor.Red;
 159                                         Console.WriteLine ("===== stderr =====");
 160                                         Console.ResetColor ();
 161
 162                                         Console.WriteLine (result.StandardError.ToString ());
 163                                 }
 164
 165                                 results.Add (result);
 166                         }
 167
 168                         sw.Stop ();
 169
 170                         var successes = results.Count (r => r.ExitCode == 0);
 171                         var failures = results.Count (r => r.ExitCode != null && r.ExitCode != 0);
 172                         var timeouts = results.Count (r => r.ExitCode == null);
 173
 174                         var settings = new XmlWriterSettings {
 175                                 NewLineOnAttributes = true,
 176                                 Indent = true,
 177                         };
 178
 179                         using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
 180                                 writer.WriteStartDocument ();
 181                                 writer.WriteComment ("This file represents the results of running a test suite");
 182
 183                                 writer.WriteStartElement ("test-results");
 184                                 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
 185                                 writer.WriteAttributeString ("total", results.Count.ToString ());
 186                                 writer.WriteAttributeString ("failures", failures.ToString ());
 187                                 writer.WriteAttributeString ("not-run", "0");
 188                                 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 189                                 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 190
 191                                 writer.WriteStartElement ("environment");
 192                                 writer.WriteAttributeString ("nunit-version", "2.4.8.0");
 193                                 writer.WriteAttributeString ("clr-version", Environment.Version.ToString ());
 194                                 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString ());
 195                                 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString ());
 196                                 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory);
 197                                 writer.WriteAttributeString ("machine-name", Environment.MachineName);
 198                                 writer.WriteAttributeString ("user", Environment.UserName);
 199                                 writer.WriteAttributeString ("user-domain", Environment.UserDomainName);
 200                                 writer.WriteEndElement ();
 201
 202                                 writer.WriteStartElement ("culture-info");
 203                                 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name);
 204                                 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name);
 205                                 writer.WriteEndElement ();
 206
 207                                 writer.WriteStartElement ("test-suite");
 208                                 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
 209                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 210                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 211                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 212                                 writer.WriteStartElement ("results");
 213
 214                                 writer.WriteStartElement ("test-suite");
 215                                 writer.WriteAttributeString ("name", "MonoTests");
 216                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 217                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 218                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 219                                 writer.WriteStartElement ("results");
 220
 221                                 writer.WriteStartElement ("test-suite");
 222                                 writer.WriteAttributeString ("name", "profiler-stress");
 223                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 224                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 225                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 226                                 writer.WriteStartElement ("results");
 227
 228                                 foreach (var result in results) {
 229                                         var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;
 230
 231                                         writer.WriteStartElement ("test-case");
 232                                         writer.WriteAttributeString ("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
 233                                         writer.WriteAttributeString ("executed", "True");
 234                                         writer.WriteAttributeString ("success", (result.ExitCode == 0).ToString ());
 235                                         writer.WriteAttributeString ("time", ((int) result.Stopwatch.Elapsed.TotalSeconds).ToString ());
 236                                         writer.WriteAttributeString ("asserts", result.ExitCode == 0 ? "0" : "1");
 237
 238                                         if (result.ExitCode != 0) {
 239                                                 writer.WriteStartElement ("failure");
 240
 241                                                 writer.WriteStartElement ("message");
 242                                                 writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput.ToString ()));
 243                                                 writer.WriteEndElement ();
 244
 245                                                 writer.WriteStartElement ("stack-trace");
 246                                                 writer.WriteCData (FilterInvalidXmlChars (result.StandardError.ToString ()));
 247                                                 writer.WriteEndElement ();
 248
 249                                                 writer.WriteEndElement ();
 250                                         }
 251
 252                                         writer.WriteEndElement ();
 253                                 }
 254
 255                                 writer.WriteEndElement ();
 256                                 writer.WriteEndElement ();
 257
 258                                 writer.WriteEndElement ();
 259                                 writer.WriteEndElement ();
 260
 261                                 writer.WriteEndElement ();
 262                                 writer.WriteEndElement ();
 263
 264                                 writer.WriteEndElement ();
 265
 266                                 writer.WriteEndDocument ();
 267                         }
 268
 269                         var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;
 270
 271                         Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
 272                         Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
 273                         Console.ResetColor ();
 274
 275                         return failures + timeouts;
 276                 }
 277         }
 278 }