acceptance-tests/profiler-stress/runner.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.Diagnostics;
   4 using System.Globalization;
   5 using System.IO;
   6 using System.Linq;
   7 using System.Text;
   8 using System.Text.RegularExpressions;
   9 using System.Threading;
  10 using System.Xml;
  11 using Mono.Unix.Native;
  12 using Newtonsoft.Json;
  13
  14 // Shut up CLS compliance warnings from Json.NET.
  15 [assembly: CLSCompliant (true)]
  16
  17 namespace Mono.Profiling.Tests.Stress {
  18
  19         // https://github.com/xamarin/benchmarker/blob/master/tools/libdbmodel/Benchmark.cs
  20         sealed class Benchmark {
  21
  22                 public string Name { get; set; }
  23                 public string TestDirectory { get; set; }
  24                 public bool OnlyExplicit { get; set; }
  25                 public string[] CommandLine { get; set; }
  26                 public string[] ClientCommandLine { get; set; }
  27                 public string[] AOTAssemblies { get; set; }
  28
  29                 public static Benchmark Load (string file)
  30                 {
  31                         return JsonConvert.DeserializeObject<Benchmark> (File.ReadAllText (file));
  32                 }
  33         }
  34
  35         sealed class TestResult {
  36
  37                 public Benchmark Benchmark { get; set; }
  38                 public ProcessStartInfo StartInfo { get; set; }
  39                 public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
  40                 public int? ExitCode { get; set; }
  41                 public string StandardOutput { get; set; }
  42                 public string StandardError { get; set; }
  43         }
  44
  45         static class Program {
  46
  47                 static readonly string[] _options = new [] {
  48                         "exception",
  49                         "monitor",
  50                         "gc",
  51                         "gcalloc",
  52                         "gcmove",
  53                         "gcroot",
  54                         "gchandle",
  55                         "finalization",
  56                         "counter",
  57                         "jit",
  58                 };
  59
  60                 static readonly TimeSpan _timeout = TimeSpan.FromHours (6);
  61
  62                 const RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.Singleline;
  63
  64                 static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
  65                         { "ironjs-v8", FilterNotOnArm },
  66                         { "msbiology", FilterNever },
  67                 };
  68
  69                 static bool FilterNever (Benchmark benchmark)
  70                 {
  71                         return false;
  72                 }
  73
  74                 static bool FilterNotOnArm (Benchmark benchmark)
  75                 {
  76 #if ARCH_arm || ARCH_arm64
  77                         return false;
  78 #else
  79                         return true;
  80 #endif
  81                 }
  82
  83                 static bool IsSupported (Benchmark benchmark)
  84                 {
  85                         return _filters.TryGetValue (benchmark.Name, out var filter) ? filter (benchmark) : true;
  86                 }
  87
  88                 static string ReplaceInvalidXmlChars (string text) {
  89                         return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
  90                 }
  91
  92                 static int Main ()
  93                 {
  94                         var regex = new Regex (".*", _regexOptions);
  95
  96                         if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_REGEX") is string envRegex)
  97                                 regex = new Regex (envRegex, _regexOptions);
  98
  99                         var depDir = Path.Combine ("..", "external", "benchmarker");
 100                         var benchDir = Path.Combine (depDir, "benchmarks");
 101                         var testDir = Path.Combine (depDir, "tests");
 102
 103                         var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
 104                                          .Select (Benchmark.Load)
 105                                          .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b) && regex.IsMatch (b.Name))
 106                                          .OrderBy (b => b.Name)
 107                                          .ToArray ();
 108
 109                         var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
 110                         var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));
 111
 112                         var seed = Environment.TickCount;
 113
 114                         if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_SEED") is string envSeed)
 115                                 seed = int.Parse (envSeed);
 116
 117                         var timeout = (int) _timeout.TotalMilliseconds;
 118
 119                         if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_TIMEOUT") is string envTimeout)
 120                                 timeout = (int) TimeSpan.Parse (envTimeout).TotalMilliseconds;
 121
 122                         string options = null;
 123
 124                         if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_OPTIONS") is string envOptions)
 125                                 options = envOptions;
 126
 127                         var suspend = false;
 128
 129                         if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_SUSPEND") is string envSuspend)
 130                                 suspend = bool.Parse (envSuspend);
 131
 132                         var rand = new Random (seed);
 133                         var cpus = Environment.ProcessorCount;
 134
 135                         var sw = Stopwatch.StartNew ();
 136
 137                         Console.ForegroundColor = ConsoleColor.Magenta;
 138                         Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Starting test session with seed: {seed}");
 139                         Console.ResetColor ();
 140
 141                         var results = new List<TestResult> (benchmarks.Length);
 142
 143                         for (var i = 0; i < benchmarks.Length; i++) {
 144                                 var bench = benchmarks [i];
 145
 146                                 var sampleFreq = rand.Next (-1000, 1001);
 147                                 var sampleMode = rand.Next (0, 2) == 1 ? "-real" : string.Empty;
 148                                 var maxSamples = rand.Next (0, cpus * 2000 + 1);
 149                                 var heapShotFreq = rand.Next (-10, 11);
 150                                 var maxFrames = rand.Next (0, 33);
 151                                 var flags = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
 152                                                     .Select (x => (x.Value ? string.Empty : "no") + x.Key)
 153                                                     .ToArray ();
 154
 155                                 var profOptions = "nodefaults,output=/dev/null,";
 156
 157                                 if (options == null) {
 158                                         profOptions += $"maxframes={maxFrames},";
 159
 160                                         if (sampleFreq > 0)
 161                                                 profOptions += $"sample{sampleMode}={sampleFreq},maxsamples={maxSamples},";
 162
 163                                         if (heapShotFreq > 0)
 164                                                 profOptions += $"heapshot={heapShotFreq}gc,";
 165
 166                                         profOptions += string.Join (",", flags);
 167                                 } else
 168                                         profOptions += options;
 169
 170                                 var info = new ProcessStartInfo {
 171                                         UseShellExecute = false,
 172                                         WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
 173                                         FileName = monoPath,
 174                                         Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
 175                                         RedirectStandardOutput = true,
 176                                         RedirectStandardError = true,
 177                                 };
 178
 179                                 info.EnvironmentVariables.Clear ();
 180                                 info.EnvironmentVariables.Add ("MONO_PATH", classDir);
 181
 182                                 if (suspend)
 183                                         info.EnvironmentVariables.Add ("MONO_DEBUG", "suspend-on-native-crash,suspend-on-unhandled");
 184
 185                                 var progress = $"({i + 1}/{benchmarks.Length})";
 186
 187                                 Console.ForegroundColor = ConsoleColor.Blue;
 188                                 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
 189                                 Console.ResetColor ();
 190
 191                                 var result = new TestResult {
 192                                         Benchmark = bench,
 193                                         StartInfo = info,
 194                                 };
 195
 196                                 using (var proc = new Process ()) {
 197                                         proc.StartInfo = info;
 198
 199                                         var stdout = new StringBuilder ();
 200                                         var stderr = new StringBuilder ();
 201
 202                                         proc.OutputDataReceived += (sender, args) => {
 203                                                 if (args.Data != null)
 204                                                         lock (result)
 205                                                                 stdout.AppendLine (args.Data);
 206                                         };
 207
 208                                         proc.ErrorDataReceived += (sender, args) => {
 209                                                 if (args.Data != null)
 210                                                         lock (result)
 211                                                                 stderr.AppendLine (args.Data);
 212                                         };
 213
 214                                         result.Stopwatch.Start ();
 215
 216                                         proc.Start ();
 217
 218                                         proc.BeginOutputReadLine ();
 219                                         proc.BeginErrorReadLine ();
 220
 221                                         if (!proc.WaitForExit (timeout)) {
 222                                                 // Force a thread dump.
 223                                                 Syscall.kill (proc.Id, Signum.SIGQUIT);
 224                                                 Thread.Sleep (1000);
 225
 226                                                 try {
 227                                                         proc.Kill ();
 228                                                 } catch (Exception) {
 229                                                 }
 230                                         } else
 231                                                 result.ExitCode = proc.ExitCode;
 232
 233                                         result.Stopwatch.Stop ();
 234
 235                                         lock (result) {
 236                                                 result.StandardOutput = stdout.ToString ();
 237                                                 result.StandardError = stderr.ToString ();
 238                                         }
 239                                 }
 240
 241                                 var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";
 242
 243                                 Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
 244                                 Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
 245                                 Console.ResetColor ();
 246
 247                                 if (result.ExitCode != 0) {
 248                                         Console.ForegroundColor = ConsoleColor.Red;
 249                                         Console.WriteLine ("===== stdout =====");
 250                                         Console.ResetColor ();
 251
 252                                         Console.WriteLine (result.StandardOutput);
 253
 254                                         Console.ForegroundColor = ConsoleColor.Red;
 255                                         Console.WriteLine ("===== stderr =====");
 256                                         Console.ResetColor ();
 257
 258                                         Console.WriteLine (result.StandardError);
 259                                 }
 260
 261                                 results.Add (result);
 262                         }
 263
 264                         sw.Stop ();
 265
 266                         var successes = results.Count (r => r.ExitCode == 0);
 267                         var failures = results.Count (r => r.ExitCode != null && r.ExitCode != 0);
 268                         var timeouts = results.Count (r => r.ExitCode == null);
 269
 270                         var settings = new XmlWriterSettings {
 271                                 NewLineOnAttributes = true,
 272                                 Indent = true,
 273                         };
 274
 275                         using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
 276                                 writer.WriteStartDocument ();
 277                                 writer.WriteComment ($"This file represents the results of running a test suite (seed: {seed})");
 278
 279                                 writer.WriteStartElement ("test-results");
 280                                 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
 281                                 writer.WriteAttributeString ("total", results.Count.ToString ());
 282                                 writer.WriteAttributeString ("failures", failures.ToString ());
 283                                 writer.WriteAttributeString ("not-run", "0");
 284                                 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
 285                                 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
 286
 287                                 writer.WriteStartElement ("environment");
 288                                 writer.WriteAttributeString ("nunit-version", "2.4.8.0");
 289                                 writer.WriteAttributeString ("clr-version", Environment.Version.ToString ());
 290                                 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString ());
 291                                 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString ());
 292                                 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory);
 293                                 writer.WriteAttributeString ("machine-name", Environment.MachineName);
 294                                 writer.WriteAttributeString ("user", Environment.UserName);
 295                                 writer.WriteAttributeString ("user-domain", Environment.UserDomainName);
 296                                 writer.WriteEndElement ();
 297
 298                                 writer.WriteStartElement ("culture-info");
 299                                 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name);
 300                                 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name);
 301                                 writer.WriteEndElement ();
 302
 303                                 writer.WriteStartElement ("test-suite");
 304                                 writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
 305                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 306                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 307                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 308                                 writer.WriteStartElement ("results");
 309
 310                                 writer.WriteStartElement ("test-suite");
 311                                 writer.WriteAttributeString ("name", "MonoTests");
 312                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 313                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 314                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 315                                 writer.WriteStartElement ("results");
 316
 317                                 writer.WriteStartElement ("test-suite");
 318                                 writer.WriteAttributeString ("name", "profiler-stress");
 319                                 writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
 320                                 writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
 321                                 writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
 322                                 writer.WriteStartElement ("results");
 323
 324                                 foreach (var result in results) {
 325                                         var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;
 326
 327                                         writer.WriteStartElement ("test-case");
 328                                         writer.WriteAttributeString ("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
 329                                         writer.WriteAttributeString ("executed", "True");
 330                                         writer.WriteAttributeString ("success", (result.ExitCode == 0).ToString ());
 331                                         writer.WriteAttributeString ("time", ((int) result.Stopwatch.Elapsed.TotalSeconds).ToString ());
 332                                         writer.WriteAttributeString ("asserts", result.ExitCode == 0 ? "0" : "1");
 333
 334                                         if (result.ExitCode != 0) {
 335                                                 writer.WriteStartElement ("failure");
 336
 337                                                 writer.WriteStartElement ("message");
 338                                                 writer.WriteCData (ReplaceInvalidXmlChars (result.StandardOutput));
 339                                                 writer.WriteEndElement ();
 340
 341                                                 writer.WriteStartElement ("stack-trace");
 342                                                 writer.WriteCData (ReplaceInvalidXmlChars (result.StandardError));
 343                                                 writer.WriteEndElement ();
 344
 345                                                 writer.WriteEndElement ();
 346                                         }
 347
 348                                         writer.WriteEndElement ();
 349                                 }
 350
 351                                 writer.WriteEndElement ();
 352                                 writer.WriteEndElement ();
 353
 354                                 writer.WriteEndElement ();
 355                                 writer.WriteEndElement ();
 356
 357                                 writer.WriteEndElement ();
 358                                 writer.WriteEndElement ();
 359
 360                                 writer.WriteEndElement ();
 361
 362                                 writer.WriteEndDocument ();
 363                         }
 364
 365                         var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;
 366
 367                         Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
 368                         Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
 369                         Console.ResetColor ();
 370
 371                         return failures + timeouts;
 372                 }
 373         }
 374 }