?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[netcore] Implement/fix enum/nullable comparers.
[mono-project.git] / mono / tests / test-runner.cs
blob6c8fd0a9efb7474738ce359bde2f556687391a80
1 //
2 // test-runner.cs
3 //
4 // Author:
5 // Zoltan Varga (vargaz@gmail.com)
6 //
7 // Copyright (C) 2008 Novell, Inc (http://www.novell.com)
8 //
9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
10 //
11 using System;
12 using System.IO;
13 using System.Threading;
14 using System.Diagnostics;
15 using System.Collections.Generic;
16 using System.Globalization;
17 using System.Xml;
18 using System.Text;
19 using System.Text.RegularExpressions;
20 using System.Linq;
21
22 #if !FULL_AOT_DESKTOP && !MOBILE
23 using Mono.Unix.Native;
24 #endif
25
26 //
27 // This is a simple test runner with support for parallel execution
28 //
29
30 public class TestRunner
31 {
32 const string TEST_TIME_FORMAT = "mm\\:ss\\.fff";
33 const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
34 const string MONO_PATH = "MONO_PATH";
35 const string MONO_GAC_PREFIX = "MONO_GAC_PREFIX";
36
37 class ProcessData {
38 public string test;
39 public StringBuilder stdout, stderr;
40 public object stdoutLock = new object (), stderrLock = new object ();
41 public string stdoutName, stderrName;
42 public TimeSpan duration;
43 }
44
45 class TestInfo {
46 public string test, opt_set;
47 }
48
49 public static int Main (String[] args) {
50 // Defaults
51 int concurrency = 1;
52 int timeout = 2 * 60; // in seconds
53 int expectedExitCode = 0;
54 bool verbose = false;
55 string testsuiteName = null;
56 string inputFile = null;
57 int repeat = 1;
58
59 string disabled_tests = null;
60 string runtime = "mono";
61 string config = null;
62 string mono_path = null;
63 string runtime_args = null;
64 string mono_gac_prefix = null;
65 string xunit_output_path = null;
66 var opt_sets = new List<string> ();
67
68 // Process options
69 int i = 0;
70 while (i < args.Length) {
71 if (args [i].StartsWith ("-")) {
72 if (args [i] == "-j") {
73 if (i + 1 >= args.Length) {
74 Console.WriteLine ("Missing argument to -j command line option.");
75 return 1;
76 }
77 if (args [i + 1] == "a")
78 concurrency = Environment.ProcessorCount;
79 else
80 concurrency = Int32.Parse (args [i + 1]);
81 i += 2;
82 } else if (args [i] == "--timeout") {
83 if (i + 1 >= args.Length) {
84 Console.WriteLine ("Missing argument to --timeout command line option.");
85 return 1;
86 }
87 timeout = Int32.Parse (args [i + 1]);
88 i += 2;
89 } else if (args [i] == "--disabled") {
90 if (i + 1 >= args.Length) {
91 Console.WriteLine ("Missing argument to --disabled command line option.");
92 return 1;
93 }
94 disabled_tests = args [i + 1];
95 i += 2;
96 } else if (args [i] == "--runtime") {
97 if (i + 1 >= args.Length) {
98 Console.WriteLine ("Missing argument to --runtime command line option.");
99 return 1;
100 }
101 runtime = args [i + 1];
102 i += 2;
103 } else if (args [i] == "--runtime-args") {
104 if (i + 1 >= args.Length) {
105 Console.WriteLine ("Missing argument to --runtime-args command line option.");
106 return 1;
107 }
108 runtime_args = (runtime_args ?? "") + " " + args [i + 1];
109 i += 2;
110 } else if (args [i] == "--config") {
111 if (i + 1 >= args.Length) {
112 Console.WriteLine ("Missing argument to --config command line option.");
113 return 1;
114 }
115 config = args [i + 1];
116 i += 2;
117 } else if (args [i] == "--opt-sets") {
118 if (i + 1 >= args.Length) {
119 Console.WriteLine ("Missing argument to --opt-sets command line option.");
120 return 1;
121 }
122 foreach (var s in args [i + 1].Split ())
123 opt_sets.Add (s);
124 i += 2;
125 } else if (args [i] == "--expected-exit-code") {
126 if (i + 1 >= args.Length) {
127 Console.WriteLine ("Missing argument to --expected-exit-code command line option.");
128 return 1;
129 }
130 expectedExitCode = Int32.Parse (args [i + 1]);
131 i += 2;
132 } else if (args [i] == "--testsuite-name") {
133 if (i + 1 >= args.Length) {
134 Console.WriteLine ("Missing argument to --testsuite-name command line option.");
135 return 1;
136 }
137 testsuiteName = args [i + 1];
138 i += 2;
139 } else if (args [i] == "--xunit") {
140 if (i + 1 >= args.Length) {
141 Console.WriteLine ("Missing argument to --xunit command line option.");
142 return 1;
143 }
144 xunit_output_path = args [i + 1].Substring(0, args [i + 1].Length);
145
146 i += 2;
147 } else if (args [i] == "--input-file") {
148 if (i + 1 >= args.Length) {
149 Console.WriteLine ("Missing argument to --input-file command line option.");
150 return 1;
151 }
152 inputFile = args [i + 1];
153 i += 2;
154 } else if (args [i] == "--mono-path") {
155 if (i + 1 >= args.Length) {
156 Console.WriteLine ("Missing argument to --mono-path command line option.");
157 return 1;
158 }
159 mono_path = args [i + 1].Substring(0, args [i + 1].Length);
160
161 i += 2;
162 } else if (args [i] == "--mono-gac-prefix") {
163 if (i + 1 >= args.Length) {
164 Console.WriteLine ("Missing argument to --mono-gac-prefix command line option.");
165 return 1;
166 }
167 mono_gac_prefix = args[i + 1];
168 i += 2;
169 } else if (args [i] == "--verbose") {
170 verbose = true;
171 i ++;
172 } else if (args [i] == "--repeat") {
173 if (i + 1 >= args.Length) {
174 Console.WriteLine ("Missing argument to --repeat command line option.");
175 return 1;
176 }
177 repeat = Int32.Parse (args [i + 1]);
178 if (repeat <= 1) {
179 Console.WriteLine ("Invalid argument to --repeat command line option, should be > 1");
180 return 1;
181 }
182 i += 2;
183 } else {
184 Console.WriteLine ("Unknown command line option: '" + args [i] + "'.");
185 return 1;
186 }
187 } else {
188 break;
189 }
190 }
191
192 if (String.IsNullOrEmpty (testsuiteName)) {
193 Console.WriteLine ("Missing the required --testsuite-name command line option.");
194 return 1;
195 }
196
197 var disabled = new Dictionary <string, string> ();
198
199 if (disabled_tests != null) {
200 foreach (string test in disabled_tests.Split ())
201 disabled [test] = test;
202 }
203
204 var tests = new List<string> ();
205
206 if (!String.IsNullOrEmpty (inputFile)) {
207 foreach (string l in File.ReadAllLines (inputFile)) {
208 for (int r = 0; r < repeat; ++r)
209 tests.Add (l);
210 }
211 } else {
212 // The remaining arguments are the tests
213 for (int j = i; j < args.Length; ++j)
214 if (!disabled.ContainsKey (args [j])) {
215 for (int r = 0; r < repeat; ++r)
216 tests.Add (args [j]);
217 }
218 }
219
220 if (!tests.Any ()) {
221 Console.WriteLine ("No tests selected, exiting.");
222 return 0;
223 }
224
225 /* If tests are repeated, we don't want the same test to run consecutively, so we need to randomise their order.
226 * But to ease reproduction of certain order-based bugs (if and only if test A and B execute at the same time),
227 * we want to use a constant seed so the tests always run in the same order. */
228 var random = new Random (0);
229 tests = tests.OrderBy (t => random.Next ()).ToList ();
230
231 var passed = new List<ProcessData> ();
232 var failed = new List<ProcessData> ();
233 var timedout = new List<ProcessData> ();
234
235 object monitor = new object ();
236
237 Console.WriteLine ("Running tests: ");
238
239 var test_info = new Queue<TestInfo> ();
240 if (opt_sets.Count == 0) {
241 foreach (string s in tests)
242 test_info.Enqueue (new TestInfo { test = s });
243 } else {
244 foreach (string opt in opt_sets) {
245 foreach (string s in tests)
246 test_info.Enqueue (new TestInfo { test = s, opt_set = opt });
247 }
248 }
249
250 /* compute the max length of test names, to have an optimal output width */
251 int output_width = -1;
252 foreach (TestInfo ti in test_info) {
253 if (ti.test.Length > output_width)
254 output_width = Math.Min (120, ti.test.Length);
255 }
256
257 List<Thread> threads = new List<Thread> (concurrency);
258
259 DateTime test_start_time = DateTime.UtcNow;
260
261 for (int j = 0; j < concurrency; ++j) {
262 Thread thread = new Thread (() => {
263 while (true) {
264 TestInfo ti;
265
266 lock (monitor) {
267 if (test_info.Count == 0)
268 break;
269 ti = test_info.Dequeue ();
270 }
271
272 var output = new StringWriter ();
273
274 string test = ti.test;
275 string opt_set = ti.opt_set;
276
277 if (verbose) {
278 output.Write (String.Format ("{{0,-{0}}} ", output_width), test);
279 } else {
280 Console.Write (".");
281 }
282
283 /* Spawn a new process */
284
285 string process_args = "";
286
287 if (opt_set != null)
288 process_args += " -O=" + opt_set;
289 if (runtime_args != null)
290 process_args += " " + runtime_args;
291
292 process_args += " " + test;
293
294 ProcessStartInfo info = new ProcessStartInfo (runtime, process_args);
295 info.UseShellExecute = false;
296 info.RedirectStandardOutput = true;
297 info.RedirectStandardError = true;
298 info.EnvironmentVariables[ENV_TIMEOUT] = timeout.ToString();
299 if (config != null)
300 info.EnvironmentVariables["MONO_CONFIG"] = config;
301 if (mono_path != null)
302 info.EnvironmentVariables[MONO_PATH] = mono_path;
303 if (mono_gac_prefix != null)
304 info.EnvironmentVariables[MONO_GAC_PREFIX] = mono_gac_prefix;
305 Process p = new Process ();
306 p.StartInfo = info;
307
308 ProcessData data = new ProcessData ();
309 data.test = test;
310
311 string log_prefix = "";
312 if (opt_set != null)
313 log_prefix = "." + opt_set.Replace ("-", "no").Replace (",", "_");
314
315 data.stdoutName = test + log_prefix + ".stdout";
316 data.stdout = new StringBuilder ();
317
318 data.stderrName = test + log_prefix + ".stderr";
319 data.stderr = new StringBuilder ();
320
321 p.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
322 lock (data.stdoutLock) {
323 if (e.Data != null)
324 data.stdout.AppendLine (e.Data);
325 }
326 };
327
328 p.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
329 lock (data.stderrLock) {
330 if (e.Data != null)
331 data.stderr.AppendLine (e.Data);
332 }
333 };
334
335 var start = DateTime.UtcNow;
336
337 p.Start ();
338
339 p.BeginOutputReadLine ();
340 p.BeginErrorReadLine ();
341
342 if (!p.WaitForExit (timeout * 1000)) {
343 var end = DateTime.UtcNow;
344 data.duration = end - start;
345
346 lock (monitor) {
347 timedout.Add (data);
348 }
349
350 // Force the process to print a thread dump
351 TryThreadDump (p.Id, data);
352
353 if (verbose) {
354 output.Write ($"timed out ({timeout}s)");
355 }
356
357 try {
358 p.Kill ();
359 } catch {
360 }
361 } else if (p.ExitCode != expectedExitCode) {
362 var end = DateTime.UtcNow;
363 data.duration = end - start;
364
365 lock (monitor) {
366 failed.Add (data);
367 }
368
369 if (verbose)
370 output.Write ("failed, time: {0}, exit code: {1}", data.duration.ToString (TEST_TIME_FORMAT), p.ExitCode);
371 } else {
372 var end = DateTime.UtcNow;
373 data.duration = end - start;
374
375 lock (monitor) {
376 passed.Add (data);
377 }
378
379 if (verbose)
380 output.Write ("passed, time: {0}", data.duration.ToString (TEST_TIME_FORMAT));
381 }
382
383 p.Close ();
384
385 lock (monitor) {
386 if (verbose)
387 Console.WriteLine (output.ToString ());
388 }
389 }
390 });
391
392 thread.Start ();
393
394 threads.Add (thread);
395 }
396
397 for (int j = 0; j < threads.Count; ++j)
398 threads [j].Join ();
399
400 TimeSpan test_time = DateTime.UtcNow - test_start_time;
401
402 if (xunit_output_path != null) {
403 WriteXUnitOutput (xunit_output_path, testsuiteName, test_time, passed, failed, timedout);
404 }
405 else {
406 var xmlPath = String.Format ("TestResult-{0}.xml", testsuiteName);
407 WriteNUnitOutput (xmlPath, testsuiteName, test_time, passed, failed, timedout);
408 string babysitterXmlList = Environment.GetEnvironmentVariable("MONO_BABYSITTER_NUNIT_XML_LIST_FILE");
409 if (!String.IsNullOrEmpty(babysitterXmlList)) {
410 try {
411 string fullXmlPath = Path.GetFullPath(xmlPath);
412 File.AppendAllText(babysitterXmlList, fullXmlPath + Environment.NewLine);
413 } catch (Exception e) {
414 Console.WriteLine("Attempted to record XML path to file {0} but failed.", babysitterXmlList);
415 }
416 }
417 }
418
419 if (verbose) {
420 Console.WriteLine ();
421 Console.WriteLine ("Time: {0}", test_time.ToString (TEST_TIME_FORMAT));
422 Console.WriteLine ();
423 Console.WriteLine ("{0,4} test(s) passed", passed.Count);
424 Console.WriteLine ("{0,4} test(s) failed", failed.Count);
425 Console.WriteLine ("{0,4} test(s) timed out", timedout.Count);
426 } else {
427 Console.WriteLine ();
428 Console.WriteLine (String.Format ("{0} test(s) passed, {1} test(s) did not pass.", passed.Count, failed.Count));
429 }
430
431 if (failed.Count > 0) {
432 Console.WriteLine ();
433 Console.WriteLine ("Failed test(s):");
434 foreach (ProcessData pd in failed) {
435 Console.WriteLine ();
436 Console.WriteLine (pd.test);
437 DumpFile (pd.stdoutName, pd.stdout.ToString ());
438 DumpFile (pd.stderrName, pd.stderr.ToString ());
439 }
440 }
441
442 if (timedout.Count > 0) {
443 Console.WriteLine ();
444 Console.WriteLine ("Timed out test(s):");
445 foreach (ProcessData pd in timedout) {
446 Console.WriteLine ();
447 Console.WriteLine (pd.test);
448 DumpFile (pd.stdoutName, pd.stdout.ToString ());
449 DumpFile (pd.stderrName, pd.stderr.ToString ());
450 }
451 }
452
453 return (timedout.Count == 0 && failed.Count == 0) ? 0 : 1;
454 }
455
456 static void DumpFile (string filename, string text) {
457 Console.WriteLine ("=============== {0} ===============", filename);
458 Console.WriteLine (text);
459 Console.WriteLine ("=============== EOF ===============");
460 }
461
462 static void WriteNUnitOutput (string path, string testsuiteName, TimeSpan test_time, List<ProcessData> passed, List<ProcessData> failed, List<ProcessData> timedout) {
463 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
464 xmlWriterSettings.NewLineOnAttributes = true;
465 xmlWriterSettings.Indent = true;
466
467 using (XmlWriter writer = XmlWriter.Create (path, xmlWriterSettings)) {
468 // <?xml version="1.0" encoding="utf-8" standalone="no"?>
469 writer.WriteStartDocument ();
470 // <!--This file represents the results of running a test suite-->
471 writer.WriteComment ("This file represents the results of running a test suite");
472 // <test-results name="/home/charlie/Dev/NUnit/nunit-2.5/work/src/bin/Debug/tests/mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2010-10-18" time="13:23:35">
473 writer.WriteStartElement ("test-results");
474 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
475 writer.WriteAttributeString ("total", (passed.Count + failed.Count + timedout.Count).ToString());
476 writer.WriteAttributeString ("failures", (failed.Count + timedout.Count).ToString());
477 writer.WriteAttributeString ("not-run", "0");
478 writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
479 writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));
480 // <environment nunit-version="2.4.8.0" clr-version="4.0.30319.17020" os-version="Unix 3.13.0.45" platform="Unix" cwd="/home/directhex/Projects/mono/mcs/class/corlib" machine-name="marceline" user="directhex" user-domain="marceline" />
481 writer.WriteStartElement ("environment");
482 writer.WriteAttributeString ("nunit-version", "2.4.8.0" );
483 writer.WriteAttributeString ("clr-version", Environment.Version.ToString() );
484 writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString() );
485 writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString() );
486 writer.WriteAttributeString ("cwd", Environment.CurrentDirectory );
487 writer.WriteAttributeString ("machine-name", Environment.MachineName );
488 writer.WriteAttributeString ("user", Environment.UserName );
489 writer.WriteAttributeString ("user-domain", Environment.UserDomainName );
490 writer.WriteEndElement ();
491 // <culture-info current-culture="en-GB" current-uiculture="en-GB" />
492 writer.WriteStartElement ("culture-info");
493 writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name );
494 writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name );
495 writer.WriteEndElement ();
496 // <test-suite name="corlib_test_net_4_5.dll" success="True" time="114.318" asserts="0">
497 writer.WriteStartElement ("test-suite");
498 writer.WriteAttributeString ("name", String.Format ("{0}-tests.dummy", testsuiteName));
499 writer.WriteAttributeString ("success", (failed.Count + timedout.Count == 0).ToString());
500 writer.WriteAttributeString ("time", test_time.TotalSeconds.ToString(CultureInfo.InvariantCulture));
501 writer.WriteAttributeString ("asserts", (failed.Count + timedout.Count).ToString());
502 // <results>
503 writer.WriteStartElement ("results");
504 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
505 writer.WriteStartElement ("test-suite");
506 writer.WriteAttributeString ("name","MonoTests");
507 writer.WriteAttributeString ("success", (failed.Count + timedout.Count == 0).ToString());
508 writer.WriteAttributeString ("time", test_time.TotalSeconds.ToString(CultureInfo.InvariantCulture));
509 writer.WriteAttributeString ("asserts", (failed.Count + timedout.Count).ToString());
510 // <results>
511 writer.WriteStartElement ("results");
512 // <test-suite name="MonoTests" success="True" time="114.318" asserts="0">
513 writer.WriteStartElement ("test-suite");
514 writer.WriteAttributeString ("name", testsuiteName);
515 writer.WriteAttributeString ("success", (failed.Count + timedout.Count == 0).ToString());
516 writer.WriteAttributeString ("time", test_time.TotalSeconds.ToString(CultureInfo.InvariantCulture));
517 writer.WriteAttributeString ("asserts", (failed.Count + timedout.Count).ToString());
518 // <results>
519 writer.WriteStartElement ("results");
520 // Dump all passing tests first
521 foreach (ProcessData pd in passed) {
522 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
523 writer.WriteStartElement ("test-case");
524 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
525 writer.WriteAttributeString ("executed", "True");
526 writer.WriteAttributeString ("success", "True");
527 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
528 writer.WriteAttributeString ("asserts", "0");
529 writer.WriteEndElement ();
530 }
531 // Now dump all failing tests
532 foreach (ProcessData pd in failed) {
533 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
534 writer.WriteStartElement ("test-case");
535 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}", testsuiteName, pd.test));
536 writer.WriteAttributeString ("executed", "True");
537 writer.WriteAttributeString ("success", "False");
538 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
539 writer.WriteAttributeString ("asserts", "1");
540 writer.WriteStartElement ("failure");
541 writer.WriteStartElement ("message");
542 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
543 writer.WriteEndElement ();
544 writer.WriteStartElement ("stack-trace");
545 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
546 writer.WriteEndElement ();
547 writer.WriteEndElement ();
548 writer.WriteEndElement ();
549 }
550 // Then dump all timing out tests
551 foreach (ProcessData pd in timedout) {
552 // <test-case name="MonoTests.Microsoft.Win32.RegistryKeyTest.bug79051" executed="True" success="True" time="0.063" asserts="0" />
553 writer.WriteStartElement ("test-case");
554 writer.WriteAttributeString ("name", String.Format ("MonoTests.{0}.{1}_timedout", testsuiteName, pd.test));
555 writer.WriteAttributeString ("executed", "True");
556 writer.WriteAttributeString ("success", "False");
557 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
558 writer.WriteAttributeString ("asserts", "1");
559 writer.WriteStartElement ("failure");
560 writer.WriteStartElement ("message");
561 writer.WriteCData (FilterInvalidXmlChars (pd.stdout.ToString ()));
562 writer.WriteEndElement ();
563 writer.WriteStartElement ("stack-trace");
564 writer.WriteCData (FilterInvalidXmlChars (pd.stderr.ToString ()));
565 writer.WriteEndElement ();
566 writer.WriteEndElement ();
567 writer.WriteEndElement ();
568 }
569 // </results>
570 writer.WriteEndElement ();
571 // </test-suite>
572 writer.WriteEndElement ();
573 // </results>
574 writer.WriteEndElement ();
575 // </test-suite>
576 writer.WriteEndElement ();
577 // </results>
578 writer.WriteEndElement ();
579 // </test-suite>
580 writer.WriteEndElement ();
581 // </test-results>
582 writer.WriteEndElement ();
583 writer.WriteEndDocument ();
584 }
585 }
586
587 static void WriteXUnitOutput (string path, string testsuiteName, TimeSpan test_time, List<ProcessData> passed, List<ProcessData> failed, List<ProcessData> timedout) {
588 XmlWriterSettings xmlWriterSettings = new XmlWriterSettings ();
589 xmlWriterSettings.Indent = true;
590
591 using (XmlWriter writer = XmlWriter.Create (path, xmlWriterSettings)) {
592
593 writer.WriteStartDocument ();
594
595 writer.WriteStartElement ("assemblies");
596
597 writer.WriteStartElement ("assembly");
598
599 writer.WriteAttributeString ("name", testsuiteName);
600 writer.WriteAttributeString ("environment", $"test-runner-version: {System.Reflection.Assembly.GetExecutingAssembly ().GetName ()}, clr-version: {Environment.Version}, os-version: {Environment.OSVersion}, platform: {Environment.OSVersion.Platform}, cwd: {Environment.CurrentDirectory}, machine-name: {Environment.MachineName}, user: {Environment.UserName}, user-domain: {Environment.UserDomainName}");
601 writer.WriteAttributeString ("test-framework", "test-runner");
602 writer.WriteAttributeString ("run-date", XmlConvert.ToString (DateTime.Now, "yyyy-MM-dd"));
603 writer.WriteAttributeString ("run-time", XmlConvert.ToString (DateTime.Now, "HH:mm:ss"));
604
605 writer.WriteAttributeString ("total", (passed.Count + failed.Count + timedout.Count).ToString ());
606 writer.WriteAttributeString ("errors", 0.ToString ());
607 writer.WriteAttributeString ("failed", (failed.Count + timedout.Count).ToString ());
608 writer.WriteAttributeString ("skipped", 0.ToString ());
609
610 writer.WriteAttributeString ("passed", passed.Count.ToString ());
611
612 writer.WriteStartElement ("collection");
613 writer.WriteAttributeString ("name", "tests");
614
615 foreach (var pd in passed) {
616 writer.WriteStartElement ("test");
617 writer.WriteAttributeString ("name", testsuiteName + ".tests." + pd.test);
618 writer.WriteAttributeString ("type", testsuiteName + ".tests");
619 writer.WriteAttributeString ("method", pd.test.ToString ());
620 writer.WriteAttributeString ("result", "Pass");
621 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
622 writer.WriteEndElement (); // test element
623 }
624
625 foreach (var pd in failed) {
626 writer.WriteStartElement ("test");
627 writer.WriteAttributeString ("name", testsuiteName + ".tests." + pd.test);
628 writer.WriteAttributeString ("type", testsuiteName + ".tests");
629 writer.WriteAttributeString ("method", pd.test.ToString ());
630 writer.WriteAttributeString ("result", "Fail");
631 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
632
633 writer.WriteStartElement ("failure");
634 writer.WriteAttributeString ("exception-type", "TestRunnerException");
635 writer.WriteStartElement ("message");
636 writer.WriteCData (FilterInvalidXmlChars ("STDOUT:\n" + pd.stdout.ToString () + "\n\nSTDERR:\n" + pd.stderr.ToString ()));
637 writer.WriteEndElement (); // message element
638 writer.WriteEndElement (); // failure element
639
640 writer.WriteEndElement(); // test element
641 }
642
643 foreach (var pd in timedout) {
644 writer.WriteStartElement ("test");
645 writer.WriteAttributeString ("name", testsuiteName + ".tests." + pd.test + "_timedout");
646 writer.WriteAttributeString ("type", testsuiteName + ".tests");
647 writer.WriteAttributeString ("method", pd.test.ToString ());
648 writer.WriteAttributeString ("result", "Fail");
649 writer.WriteAttributeString ("time", pd.duration.TotalSeconds.ToString(CultureInfo.InvariantCulture));
650
651 writer.WriteStartElement ("failure");
652 writer.WriteAttributeString ("exception-type", "TestRunnerException");
653 writer.WriteStartElement ("message");
654 writer.WriteCData (FilterInvalidXmlChars ("STDOUT:\n" + pd.stdout.ToString () + "\n\nSTDERR:\n" + pd.stderr.ToString ()));
655 writer.WriteEndElement (); // message element
656 writer.WriteEndElement (); // failure element
657
658 writer.WriteEndElement(); // test element
659 }
660
661 writer.WriteEndElement (); // collection
662 writer.WriteEndElement (); // assembly
663 writer.WriteEndElement (); // assemblies
664 writer.WriteEndDocument ();
665 }
666 }
667
668 static string FilterInvalidXmlChars (string text) {
669 // Spec at http://www.w3.org/TR/2008/REC-xml-20081126/#charsets says only the following chars are valid in XML:
670 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */
671 return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", "");
672 }
673
674 static void TryThreadDump (int pid, ProcessData data)
675 {
676 try {
677 TryGDB (pid, data);
678 return;
679 } catch {
680 }
681
682 #if !FULL_AOT_DESKTOP && !MOBILE
683 /* LLDB cannot produce managed stacktraces for all the threads */
684 try {
685 Syscall.kill (pid, Signum.SIGQUIT);
686 Thread.Sleep (1000);
687 } catch {
688 }
689 #endif
690
691 try {
692 TryLLDB (pid, data);
693 return;
694 } catch {
695 }
696 }
697
698 static void TryLLDB (int pid, ProcessData data)
699 {
700 string filename = Path.GetTempFileName ();
701
702 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
703 {
704 sw.WriteLine ("process attach --pid " + pid);
705 sw.WriteLine ("thread list");
706 sw.WriteLine ("thread backtrace all");
707 sw.WriteLine ("detach");
708 sw.WriteLine ("quit");
709 sw.Flush ();
710
711 ProcessStartInfo psi = new ProcessStartInfo {
712 FileName = "lldb",
713 Arguments = "--batch --source \"" + filename + "\" --no-lldbinit",
714 UseShellExecute = false,
715 RedirectStandardError = true,
716 RedirectStandardOutput = true,
717 };
718
719 using (Process process = new Process { StartInfo = psi })
720 {
721 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
722 lock (data.stdoutLock) {
723 if (e.Data != null)
724 data.stdout.AppendLine (e.Data);
725 }
726 };
727
728 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
729 lock (data.stderrLock) {
730 if (e.Data != null)
731 data.stderr.AppendLine (e.Data);
732 }
733 };
734
735 process.Start ();
736 process.BeginOutputReadLine ();
737 process.BeginErrorReadLine ();
738 if (!process.WaitForExit (60 * 1000))
739 process.Kill ();
740 }
741 }
742 }
743
744 static void TryGDB (int pid, ProcessData data)
745 {
746 string filename = Path.GetTempFileName ();
747
748 using (StreamWriter sw = new StreamWriter (new FileStream (filename, FileMode.Open, FileAccess.Write)))
749 {
750 sw.WriteLine ("attach " + pid);
751 sw.WriteLine ("info threads");
752 sw.WriteLine ("thread apply all p mono_print_thread_dump(0)");
753 sw.WriteLine ("thread apply all backtrace");
754 sw.Flush ();
755
756 ProcessStartInfo psi = new ProcessStartInfo {
757 FileName = "gdb",
758 Arguments = "-batch -x \"" + filename + "\" -nx",
759 UseShellExecute = false,
760 RedirectStandardError = true,
761 RedirectStandardOutput = true,
762 };
763
764 using (Process process = new Process { StartInfo = psi })
765 {
766 process.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e) {
767 lock (data.stdoutLock) {
768 if (e.Data != null)
769 data.stdout.AppendLine (e.Data);
770 }
771 };
772
773 process.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e) {
774 lock (data.stderrLock) {
775 if (e.Data != null)
776 data.stderr.AppendLine (e.Data);
777 }
778 };
779
780 process.Start ();
781 process.BeginOutputReadLine ();
782 process.BeginErrorReadLine ();
783 if (!process.WaitForExit (60 * 1000))
784 process.Kill ();
785 }
786 }
787 }
788 }
Mirror of the official Mono project git repository