5 // Zoltan Varga (vargaz@gmail.com)
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 using System
.Threading
;
14 using System
.Diagnostics
;
15 using System
.Collections
.Generic
;
16 using System
.Globalization
;
19 using System
.Text
.RegularExpressions
;
22 #if !FULL_AOT_DESKTOP && !MOBILE
23 using Mono
.Unix
.Native
;
27 // This is a simple test runner with support for parallel execution
30 public class TestRunner
32 const string TEST_TIME_FORMAT
= "mm\\:ss\\.fff";
33 const string ENV_TIMEOUT
= "TEST_DRIVER_TIMEOUT_SEC";
34 const string MONO_PATH
= "MONO_PATH";
35 const string MONO_GAC_PREFIX
= "MONO_GAC_PREFIX";
39 public StringBuilder stdout
, stderr
;
40 public object stdoutLock
= new object (), stderrLock
= new object ();
41 public string stdoutName
, stderrName
;
42 public TimeSpan duration
;
46 public string test
, opt_set
;
49 public static int Main (String
[] args
) {
52 int timeout
= 2 * 60; // in seconds
53 int expectedExitCode
= 0;
55 string testsuiteName
= null;
56 string inputFile
= null;
59 string disabled_tests
= null;
60 string runtime
= "mono";
62 string mono_path
= null;
63 string runtime_args
= null;
64 string mono_gac_prefix
= null;
65 var opt_sets
= new List
<string> ();
69 while (i
< args
.Length
) {
70 if (args
[i
].StartsWith ("-")) {
71 if (args
[i
] == "-j") {
72 if (i
+ 1 >= args
.Length
) {
73 Console
.WriteLine ("Missing argument to -j command line option.");
76 if (args
[i
+ 1] == "a")
77 concurrency
= Environment
.ProcessorCount
;
79 concurrency
= Int32
.Parse (args
[i
+ 1]);
81 } else if (args
[i
] == "--timeout") {
82 if (i
+ 1 >= args
.Length
) {
83 Console
.WriteLine ("Missing argument to --timeout command line option.");
86 timeout
= Int32
.Parse (args
[i
+ 1]);
88 } else if (args
[i
] == "--disabled") {
89 if (i
+ 1 >= args
.Length
) {
90 Console
.WriteLine ("Missing argument to --disabled command line option.");
93 disabled_tests
= args
[i
+ 1];
95 } else if (args
[i
] == "--runtime") {
96 if (i
+ 1 >= args
.Length
) {
97 Console
.WriteLine ("Missing argument to --runtime command line option.");
100 runtime
= args
[i
+ 1];
102 } else if (args
[i
] == "--runtime-args") {
103 if (i
+ 1 >= args
.Length
) {
104 Console
.WriteLine ("Missing argument to --runtime-args command line option.");
107 runtime_args
= (runtime_args
?? "") + " " + args
[i
+ 1];
109 } else if (args
[i
] == "--config") {
110 if (i
+ 1 >= args
.Length
) {
111 Console
.WriteLine ("Missing argument to --config command line option.");
114 config
= args
[i
+ 1];
116 } else if (args
[i
] == "--opt-sets") {
117 if (i
+ 1 >= args
.Length
) {
118 Console
.WriteLine ("Missing argument to --opt-sets command line option.");
121 foreach (var s
in args
[i
+ 1].Split ())
124 } else if (args
[i
] == "--expected-exit-code") {
125 if (i
+ 1 >= args
.Length
) {
126 Console
.WriteLine ("Missing argument to --expected-exit-code command line option.");
129 expectedExitCode
= Int32
.Parse (args
[i
+ 1]);
131 } else if (args
[i
] == "--testsuite-name") {
132 if (i
+ 1 >= args
.Length
) {
133 Console
.WriteLine ("Missing argument to --testsuite-name command line option.");
136 testsuiteName
= args
[i
+ 1];
138 } else if (args
[i
] == "--input-file") {
139 if (i
+ 1 >= args
.Length
) {
140 Console
.WriteLine ("Missing argument to --input-file command line option.");
143 inputFile
= args
[i
+ 1];
145 } else if (args
[i
] == "--mono-path") {
146 if (i
+ 1 >= args
.Length
) {
147 Console
.WriteLine ("Missing argument to --mono-path command line option.");
150 mono_path
= args
[i
+ 1].Substring(0, args
[i
+ 1].Length
);
153 } else if (args
[i
] == "--mono-gac-prefix") {
154 if (i
+ 1 >= args
.Length
) {
155 Console
.WriteLine ("Missing argument to --mono-gac-prefix command line option.");
158 mono_gac_prefix
= args
[i
+ 1];
160 } else if (args
[i
] == "--verbose") {
163 } else if (args
[i
] == "--repeat") {
164 if (i
+ 1 >= args
.Length
) {
165 Console
.WriteLine ("Missing argument to --repeat command line option.");
168 repeat
= Int32
.Parse (args
[i
+ 1]);
170 Console
.WriteLine ("Invalid argument to --repeat command line option, should be > 1");
175 Console
.WriteLine ("Unknown command line option: '" + args
[i
] + "'.");
183 if (String
.IsNullOrEmpty (testsuiteName
)) {
184 Console
.WriteLine ("Missing the required --testsuite-name command line option.");
188 var disabled
= new Dictionary
<string, string> ();
190 if (disabled_tests
!= null) {
191 foreach (string test
in disabled_tests
.Split ())
192 disabled
[test
] = test
;
195 var tests
= new List
<string> ();
197 if (!String
.IsNullOrEmpty (inputFile
)) {
198 foreach (string l
in File
.ReadAllLines (inputFile
)) {
199 for (int r
= 0; r
< repeat
; ++r
)
203 // The remaining arguments are the tests
204 for (int j
= i
; j
< args
.Length
; ++j
)
205 if (!disabled
.ContainsKey (args
[j
])) {
206 for (int r
= 0; r
< repeat
; ++r
)
207 tests
.Add (args
[j
]);
212 Console
.WriteLine ("No tests selected, exiting.");
216 /* If tests are repeated, we don't want the same test to run consecutively, so we need to randomise their order.
217 * But to ease reproduction of certain order-based bugs (if and only if test A and B execute at the same time),
218 * we want to use a constant seed so the tests always run in the same order. */
219 var random
= new Random (0);
220 tests
= tests
.OrderBy (t
=> random
.Next ()).ToList ();
222 var passed
= new List
<ProcessData
> ();
223 var failed
= new List
<ProcessData
> ();
224 var timedout
= new List
<ProcessData
> ();
226 object monitor
= new object ();
228 Console
.WriteLine ("Running tests: ");
230 var test_info
= new Queue
<TestInfo
> ();
231 if (opt_sets
.Count
== 0) {
232 foreach (string s
in tests
)
233 test_info
.Enqueue (new TestInfo { test = s }
);
235 foreach (string opt
in opt_sets
) {
236 foreach (string s
in tests
)
237 test_info
.Enqueue (new TestInfo { test = s, opt_set = opt }
);
241 /* compute the max length of test names, to have an optimal output width */
242 int output_width
= -1;
243 foreach (TestInfo ti
in test_info
) {
244 if (ti
.test
.Length
> output_width
)
245 output_width
= Math
.Min (120, ti
.test
.Length
);
248 List
<Thread
> threads
= new List
<Thread
> (concurrency
);
250 DateTime test_start_time
= DateTime
.UtcNow
;
252 for (int j
= 0; j
< concurrency
; ++j
) {
253 Thread thread
= new Thread (() => {
258 if (test_info
.Count
== 0)
260 ti
= test_info
.Dequeue ();
263 var output
= new StringWriter ();
265 string test
= ti
.test
;
266 string opt_set
= ti
.opt_set
;
269 output
.Write (String
.Format ("{{0,-{0}}} ", output_width
), test
);
274 /* Spawn a new process */
276 string process_args
= "";
279 process_args
+= " -O=" + opt_set
;
280 if (runtime_args
!= null)
281 process_args
+= " " + runtime_args
;
283 process_args
+= " " + test
;
285 ProcessStartInfo info
= new ProcessStartInfo (runtime
, process_args
);
286 info
.UseShellExecute
= false;
287 info
.RedirectStandardOutput
= true;
288 info
.RedirectStandardError
= true;
289 info
.EnvironmentVariables
[ENV_TIMEOUT
] = timeout
.ToString();
291 info
.EnvironmentVariables
["MONO_CONFIG"] = config
;
292 if (mono_path
!= null)
293 info
.EnvironmentVariables
[MONO_PATH
] = mono_path
;
294 if (mono_gac_prefix
!= null)
295 info
.EnvironmentVariables
[MONO_GAC_PREFIX
] = mono_gac_prefix
;
296 Process p
= new Process ();
299 ProcessData data
= new ProcessData ();
302 string log_prefix
= "";
304 log_prefix
= "." + opt_set
.Replace ("-", "no").Replace (",", "_");
306 data
.stdoutName
= test
+ log_prefix
+ ".stdout";
307 data
.stdout
= new StringBuilder ();
309 data
.stderrName
= test
+ log_prefix
+ ".stderr";
310 data
.stderr
= new StringBuilder ();
312 p
.OutputDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
313 lock (data
.stdoutLock
) {
315 data
.stdout
.AppendLine (e
.Data
);
319 p
.ErrorDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
320 lock (data
.stderrLock
) {
322 data
.stderr
.AppendLine (e
.Data
);
326 var start
= DateTime
.UtcNow
;
330 p
.BeginOutputReadLine ();
331 p
.BeginErrorReadLine ();
333 if (!p
.WaitForExit (timeout
* 1000)) {
334 var end
= DateTime
.UtcNow
;
335 data
.duration
= end
- start
;
341 // Force the process to print a thread dump
342 TryThreadDump (p
.Id
, data
);
345 output
.Write ($"timed out ({timeout}s)");
352 } else if (p
.ExitCode
!= expectedExitCode
) {
353 var end
= DateTime
.UtcNow
;
354 data
.duration
= end
- start
;
361 output
.Write ("failed, time: {0}, exit code: {1}", data
.duration
.ToString (TEST_TIME_FORMAT
), p
.ExitCode
);
363 var end
= DateTime
.UtcNow
;
364 data
.duration
= end
- start
;
371 output
.Write ("passed, time: {0}", data
.duration
.ToString (TEST_TIME_FORMAT
));
378 Console
.WriteLine (output
.ToString ());
385 threads
.Add (thread
);
388 for (int j
= 0; j
< threads
.Count
; ++j
)
391 TimeSpan test_time
= DateTime
.UtcNow
- test_start_time
;
393 int npassed
= passed
.Count
;
394 int nfailed
= failed
.Count
;
395 int ntimedout
= timedout
.Count
;
397 XmlWriterSettings xmlWriterSettings
= new XmlWriterSettings ();
398 xmlWriterSettings
.NewLineOnAttributes
= true;
399 xmlWriterSettings
.Indent
= true;
401 string xmlPath
= String
.Format ("TestResult-{0}.xml", testsuiteName
);
402 using (XmlWriter writer
= XmlWriter
.Create (xmlPath
, xmlWriterSettings
)) {
403 // <?xml version="1.0" encoding="utf-8" standalone="no"?>
404 writer
.WriteStartDocument ();
405 // <!--This file represents the results of running a test suite-->
406 writer
.WriteComment ("This file represents the results of running a test suite");
407 // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
408 writer
.WriteStartElement ("test-results");
409 writer
.WriteAttributeString ("name", String
.Format ("{0}-tests.dummy", testsuiteName
));
410 writer
.WriteAttributeString ("total", (npassed
+ nfailed
+ ntimedout
).ToString());
411 writer
.WriteAttributeString ("failures", (nfailed
+ ntimedout
).ToString());
412 writer
.WriteAttributeString ("not-run", "0");
413 writer
.WriteAttributeString ("date", DateTime
.Now
.ToString ("yyyy-MM-dd"));
414 writer
.WriteAttributeString ("time", DateTime
.Now
.ToString ("HH:mm:ss"));
415 // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
416 writer
.WriteStartElement ("environment");
417 writer
.WriteAttributeString ("nunit-version", "2.4.8.0" );
418 writer
.WriteAttributeString ("clr-version", Environment
.Version
.ToString() );
419 writer
.WriteAttributeString ("os-version", Environment
.OSVersion
.ToString() );
420 writer
.WriteAttributeString ("platform", Environment
.OSVersion
.Platform
.ToString() );
421 writer
.WriteAttributeString ("cwd", Environment
.CurrentDirectory
);
422 writer
.WriteAttributeString ("machine-name", Environment
.MachineName
);
423 writer
.WriteAttributeString ("user", Environment
.UserName
);
424 writer
.WriteAttributeString ("user-domain", Environment
.UserDomainName
);
425 writer
.WriteEndElement ();
426 // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
427 writer
.WriteStartElement ("culture-info");
428 writer
.WriteAttributeString ("current-culture", CultureInfo
.CurrentCulture
.Name
);
429 writer
.WriteAttributeString ("current-uiculture", CultureInfo
.CurrentUICulture
.Name
);
430 writer
.WriteEndElement ();
431 // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
432 writer
.WriteStartElement ("test-suite");
433 writer
.WriteAttributeString ("name", String
.Format ("{0}-tests.dummy", testsuiteName
));
434 writer
.WriteAttributeString ("success", (nfailed
+ ntimedout
== 0).ToString());
435 writer
.WriteAttributeString ("time", test_time
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
436 writer
.WriteAttributeString ("asserts", (nfailed
+ ntimedout
).ToString());
438 writer
.WriteStartElement ("results");
439 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
440 writer
.WriteStartElement ("test-suite");
441 writer
.WriteAttributeString ("name","MonoTests");
442 writer
.WriteAttributeString ("success", (nfailed
+ ntimedout
== 0).ToString());
443 writer
.WriteAttributeString ("time", test_time
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
444 writer
.WriteAttributeString ("asserts", (nfailed
+ ntimedout
).ToString());
446 writer
.WriteStartElement ("results");
447 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
448 writer
.WriteStartElement ("test-suite");
449 writer
.WriteAttributeString ("name", testsuiteName
);
450 writer
.WriteAttributeString ("success", (nfailed
+ ntimedout
== 0).ToString());
451 writer
.WriteAttributeString ("time", test_time
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
452 writer
.WriteAttributeString ("asserts", (nfailed
+ ntimedout
).ToString());
454 writer
.WriteStartElement ("results");
455 // Dump all passing tests first
456 foreach (ProcessData pd
in passed
) {
457 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
458 writer
.WriteStartElement ("test-case");
459 writer
.WriteAttributeString ("name", String
.Format ("MonoTests.{0}.{1}", testsuiteName
, pd
.test
));
460 writer
.WriteAttributeString ("executed", "True");
461 writer
.WriteAttributeString ("success", "True");
462 writer
.WriteAttributeString ("time", pd
.duration
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
463 writer
.WriteAttributeString ("asserts", "0");
464 writer
.WriteEndElement ();
466 // Now dump all failing tests
467 foreach (ProcessData pd
in failed
) {
468 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
469 writer
.WriteStartElement ("test-case");
470 writer
.WriteAttributeString ("name", String
.Format ("MonoTests.{0}.{1}", testsuiteName
, pd
.test
));
471 writer
.WriteAttributeString ("executed", "True");
472 writer
.WriteAttributeString ("success", "False");
473 writer
.WriteAttributeString ("time", pd
.duration
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
474 writer
.WriteAttributeString ("asserts", "1");
475 writer
.WriteStartElement ("failure");
476 writer
.WriteStartElement ("message");
477 writer
.WriteCData (FilterInvalidXmlChars (pd
.stdout
.ToString ()));
478 writer
.WriteEndElement ();
479 writer
.WriteStartElement ("stack-trace");
480 writer
.WriteCData (FilterInvalidXmlChars (pd
.stderr
.ToString ()));
481 writer
.WriteEndElement ();
482 writer
.WriteEndElement ();
483 writer
.WriteEndElement ();
485 // Then dump all timing out tests
486 foreach (ProcessData pd
in timedout
) {
487 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
488 writer
.WriteStartElement ("test-case");
489 writer
.WriteAttributeString ("name", String
.Format ("MonoTests.{0}.{1}_timedout", testsuiteName
, pd
.test
));
490 writer
.WriteAttributeString ("executed", "True");
491 writer
.WriteAttributeString ("success", "False");
492 writer
.WriteAttributeString ("time", pd
.duration
.TotalSeconds
.ToString(CultureInfo
.InvariantCulture
));
493 writer
.WriteAttributeString ("asserts", "1");
494 writer
.WriteStartElement ("failure");
495 writer
.WriteStartElement ("message");
496 writer
.WriteCData (FilterInvalidXmlChars (pd
.stdout
.ToString ()));
497 writer
.WriteEndElement ();
498 writer
.WriteStartElement ("stack-trace");
499 writer
.WriteCData (FilterInvalidXmlChars (pd
.stderr
.ToString ()));
500 writer
.WriteEndElement ();
501 writer
.WriteEndElement ();
502 writer
.WriteEndElement ();
505 writer
.WriteEndElement ();
507 writer
.WriteEndElement ();
509 writer
.WriteEndElement ();
511 writer
.WriteEndElement ();
513 writer
.WriteEndElement ();
515 writer
.WriteEndElement ();
517 writer
.WriteEndElement ();
518 writer
.WriteEndDocument ();
520 string babysitterXmlList
= Environment
.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
521 if (!String
.IsNullOrEmpty(babysitterXmlList
)) {
523 string fullXmlPath
= Path
.GetFullPath(xmlPath
);
524 File
.AppendAllText(babysitterXmlList
, fullXmlPath
+ Environment
.NewLine
);
525 } catch (Exception e
) {
526 Console
.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList
);
532 Console
.WriteLine ();
533 Console
.WriteLine ("Time: {0}", test_time
.ToString (TEST_TIME_FORMAT
));
534 Console
.WriteLine ();
535 Console
.WriteLine ("{0,4} test(s) passed", npassed
);
536 Console
.WriteLine ("{0,4} test(s) failed", nfailed
);
537 Console
.WriteLine ("{0,4} test(s) timed out", ntimedout
);
539 Console
.WriteLine ();
540 Console
.WriteLine (String
.Format ("{0} test(s) passed, {1} test(s) did not pass.", npassed
, nfailed
));
544 Console
.WriteLine ();
545 Console
.WriteLine ("Failed test(s):");
546 foreach (ProcessData pd
in failed
) {
547 Console
.WriteLine ();
548 Console
.WriteLine (pd
.test
);
549 DumpFile (pd
.stdoutName
, pd
.stdout
.ToString ());
550 DumpFile (pd
.stderrName
, pd
.stderr
.ToString ());
555 Console
.WriteLine ();
556 Console
.WriteLine ("Timed out test(s):");
557 foreach (ProcessData pd
in timedout
) {
558 Console
.WriteLine ();
559 Console
.WriteLine (pd
.test
);
560 DumpFile (pd
.stdoutName
, pd
.stdout
.ToString ());
561 DumpFile (pd
.stderrName
, pd
.stderr
.ToString ());
565 return (ntimedout
== 0 && nfailed
== 0) ? 0 : 1;
568 static void DumpFile (string filename
, string text
) {
569 Console
.WriteLine ("=============== {0} ===============", filename
);
570 Console
.WriteLine (text
);
571 Console
.WriteLine ("=============== EOF ===============");
574 static string FilterInvalidXmlChars (string text
) {
575 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
576 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
577 return Regex
.Replace (text
, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
580 static void TryThreadDump (int pid
, ProcessData data
)
588 #if !FULL_AOT_DESKTOP && !MOBILE
589 /* LLDB cannot produce managed stacktraces for all the threads */
591 Syscall
.kill (pid
, Signum
.SIGQUIT
);
604 static void TryLLDB (int pid
, ProcessData data
)
606 string filename
= Path
.GetTempFileName ();
608 using (StreamWriter sw
= new StreamWriter (new FileStream (filename
, FileMode
.Open
, FileAccess
.Write
)))
610 sw
.WriteLine ("process attach --pid " + pid
);
611 sw
.WriteLine ("thread list");
612 sw
.WriteLine ("thread backtrace all");
613 sw
.WriteLine ("detach");
614 sw
.WriteLine ("quit");
617 ProcessStartInfo psi
= new ProcessStartInfo
{
619 Arguments
= "--batch --source \"" + filename
+ "\" --no-lldbinit",
620 UseShellExecute
= false,
621 RedirectStandardError
= true,
622 RedirectStandardOutput
= true,
625 using (Process process
= new Process { StartInfo = psi }
)
627 process
.OutputDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
628 lock (data
.stdoutLock
) {
630 data
.stdout
.AppendLine (e
.Data
);
634 process
.ErrorDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
635 lock (data
.stderrLock
) {
637 data
.stderr
.AppendLine (e
.Data
);
642 process
.BeginOutputReadLine ();
643 process
.BeginErrorReadLine ();
644 if (!process
.WaitForExit (60 * 1000))
650 static void TryGDB (int pid
, ProcessData data
)
652 string filename
= Path
.GetTempFileName ();
654 using (StreamWriter sw
= new StreamWriter (new FileStream (filename
, FileMode
.Open
, FileAccess
.Write
)))
656 sw
.WriteLine ("attach " + pid
);
657 sw
.WriteLine ("info threads");
658 sw
.WriteLine ("thread apply all p mono_print_thread_dump(0)");
659 sw
.WriteLine ("thread apply all backtrace");
662 ProcessStartInfo psi
= new ProcessStartInfo
{
664 Arguments
= "-batch -x \"" + filename
+ "\" -nx",
665 UseShellExecute
= false,
666 RedirectStandardError
= true,
667 RedirectStandardOutput
= true,
670 using (Process process
= new Process { StartInfo = psi }
)
672 process
.OutputDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
673 lock (data
.stdoutLock
) {
675 data
.stdout
.AppendLine (e
.Data
);
679 process
.ErrorDataReceived
+= delegate (object sender
, DataReceivedEventArgs e
) {
680 lock (data
.stderrLock
) {
682 data
.stderr
.AppendLine (e
.Data
);
687 process
.BeginOutputReadLine ();
688 process
.BeginErrorReadLine ();
689 if (!process
.WaitForExit (60 * 1000))