?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
D'oh, forgot to add g_hash_table_steal() to glib.h
[mono-project/dkf.git] / mcs / tools / compiler-tester / compiler-tester.cs
blobcc66e8b1e5ca01971b53cc74b8e8939b1fbe60d7
1 //
2 // compiler-tester.cs
3 //
4 // Author:
5 // Marek Safar (marek.safar@gmail.com)
6 //
7
8 //
9 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
10 //
11 // Permission is hereby granted, free of charge, to any person obtaining
12 // a copy of this software and associated documentation files (the
13 // "Software"), to deal in the Software without restriction, including
14 // without limitation the rights to use, copy, modify, merge, publish,
15 // distribute, sublicense, and/or sell copies of the Software, and to
16 // permit persons to whom the Software is furnished to do so, subject to
17 // the following conditions:
18 //
19 // The above copyright notice and this permission notice shall be
20 // included in all copies or substantial portions of the Software.
21 //
22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 //
30
31 using System;
32 using System.IO;
33 using System.Diagnostics;
34 using System.Reflection;
35 using System.Text;
36 using System.Collections;
37 using System.Xml;
38 using System.Collections.Generic;
39
40 namespace TestRunner {
41
42 interface ITester
43 {
44 string Output { get; }
45 bool Invoke (string[] args);
46 bool IsWarning (int warningNumber);
47 }
48
49 class ReflectionTester: ITester {
50 MethodInfo ep;
51 object[] method_arg;
52 StringWriter output;
53 int[] all_warnings;
54
55 public ReflectionTester (Assembly a)
56 {
57 Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
58
59 if (t == null)
60 Console.Error.WriteLine ("null, huh?");
61
62 ep = t.GetMethod ("InvokeCompiler",
63 BindingFlags.Static | BindingFlags.Public);
64 if (ep == null)
65 throw new MissingMethodException ("static InvokeCompiler");
66 method_arg = new object [2];
67
68 PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
69 all_warnings = (int[])pi.GetValue (null, null);
70 Array.Sort (all_warnings);
71 }
72
73 public string Output {
74 get {
75 return output.GetStringBuilder ().ToString ();
76 }
77 }
78
79 public bool Invoke(string[] args)
80 {
81 output = new StringWriter ();
82 method_arg [0] = args;
83 method_arg [1] = output;
84 return (bool)ep.Invoke (null, method_arg);
85 }
86
87 public bool IsWarning (int warningNumber)
88 {
89 return Array.BinarySearch (all_warnings, warningNumber) >= 0;
90 }
91 }
92
93 #if !NET_2_1
94 class ProcessTester: ITester
95 {
96 ProcessStartInfo pi;
97 string output;
98
99 public ProcessTester (string p_path)
100 {
101 pi = new ProcessStartInfo ();
102 pi.FileName = p_path;
103 pi.CreateNoWindow = true;
104 pi.WindowStyle = ProcessWindowStyle.Hidden;
105 pi.RedirectStandardOutput = true;
106 pi.RedirectStandardError = true;
107 pi.UseShellExecute = false;
108 }
109
110 public string Output {
111 get {
112 return output;
113 }
114 }
115
116 public bool Invoke(string[] args)
117 {
118 StringBuilder sb = new StringBuilder ("/nologo ");
119 foreach (string s in args) {
120 sb.Append (s);
121 sb.Append (" ");
122 }
123 pi.Arguments = sb.ToString ();
124 Process p = Process.Start (pi);
125 output = p.StandardError.ReadToEnd ();
126 if (output.Length == 0)
127 output = p.StandardOutput.ReadToEnd ();
128 p.WaitForExit ();
129 return p.ExitCode == 0;
130 }
131
132 public bool IsWarning (int warningNumber)
133 {
134 throw new NotImplementedException ();
135 }
136 }
137 #endif
138
139 class TestCase : MarshalByRefObject
140 {
141 public readonly string FileName;
142 public readonly string[] CompilerOptions;
143 public readonly string[] Dependencies;
144
145 public TestCase (string filename, string[] options, string[] deps)
146 {
147 this.FileName = filename;
148 this.CompilerOptions = options;
149 this.Dependencies = deps;
150 }
151 }
152
153 class PositiveTestCase : TestCase
154 {
155 public class VerificationData : MarshalByRefObject
156 {
157 public class MethodData : MarshalByRefObject
158 {
159 public MethodData (MethodBase mi, int il_size)
160 {
161 this.Type = mi.DeclaringType.ToString ();
162 this.MethodName = mi.ToString ();
163 this.ILSize = il_size;
164 }
165
166 public MethodData (string type_name, string method_name, int il_size)
167 {
168 this.Type = type_name;
169 this.MethodName = method_name;
170 this.ILSize = il_size;
171 }
172
173 public string Type;
174 public string MethodName;
175 public int ILSize;
176 public bool Checked;
177 }
178
179 ArrayList methods;
180 public bool IsNewSet;
181
182 public VerificationData (string test_file)
183 {
184 #if NET_2_0
185 this.test_file = test_file;
186 #endif
187 }
188
189 #if NET_2_0
190 string test_file;
191
192 public static VerificationData FromFile (string name, XmlReader r)
193 {
194 VerificationData tc = new VerificationData (name);
195 ArrayList methods = new ArrayList ();
196 r.Read ();
197 while (r.ReadToNextSibling ("type")) {
198 string type_name = r ["name"];
199 r.Read ();
200 while (r.ReadToNextSibling ("method")) {
201 string m_name = r ["name"];
202
203 r.ReadToDescendant ("size");
204 int il_size = r.ReadElementContentAsInt ();
205 methods.Add (new MethodData (type_name, m_name, il_size));
206 r.Read ();
207 }
208 r.Read ();
209 }
210
211 tc.methods = methods;
212 return tc;
213 }
214
215 public void WriteCodeInfoTo (XmlWriter w)
216 {
217 w.WriteStartElement ("test");
218 w.WriteAttributeString ("name", test_file);
219
220 string type = null;
221 foreach (MethodData data in methods) {
222 if (!data.Checked)
223 continue;
224
225 if (type != data.Type) {
226 if (type != null)
227 w.WriteEndElement ();
228
229 type = data.Type;
230 w.WriteStartElement ("type");
231 w.WriteAttributeString ("name", type);
232 }
233
234 w.WriteStartElement ("method");
235 w.WriteAttributeString ("name", data.MethodName);
236 w.WriteStartElement ("size");
237 w.WriteValue (data.ILSize);
238 w.WriteEndElement ();
239 w.WriteEndElement ();
240 }
241
242 if (type != null)
243 w.WriteEndElement ();
244
245 w.WriteEndElement ();
246 }
247 #endif
248
249 public MethodData FindMethodData (string method_name, string declaring_type)
250 {
251 if (methods == null)
252 return null;
253
254 foreach (MethodData md in methods) {
255 if (md.MethodName == method_name && md.Type == declaring_type)
256 return md;
257 }
258
259 return null;
260 }
261
262 public void AddNewMethod (MethodBase mb, int il_size)
263 {
264 if (methods == null)
265 methods = new ArrayList ();
266
267 MethodData md = new MethodData (mb, il_size);
268 md.Checked = true;
269 methods.Add (md);
270 }
271 }
272
273 VerificationData verif_data;
274
275 public PositiveTestCase (string filename, string [] options, string [] deps)
276 : base (filename, options, deps)
277 {
278 }
279
280 public void CreateNewTest ()
281 {
282 verif_data = new VerificationData (FileName);
283 verif_data.IsNewSet = true;
284 }
285
286 public VerificationData VerificationProvider {
287 set {
288 verif_data = value;
289 }
290 get {
291 return verif_data;
292 }
293 }
294 }
295
296 class Checker: MarshalByRefObject, IDisposable
297 {
298 protected ITester tester;
299 protected int success;
300 protected int total;
301 protected int ignored;
302 protected int syntax_errors;
303 string issue_file;
304 StreamWriter log_file;
305 protected string[] extra_compiler_options;
306 // protected string[] compiler_options;
307 // protected string[] dependencies;
308
309 protected ArrayList tests = new ArrayList ();
310 protected Hashtable test_hash = new Hashtable ();
311 protected ArrayList regression = new ArrayList ();
312 protected ArrayList know_issues = new ArrayList ();
313 protected ArrayList ignore_list = new ArrayList ();
314 protected ArrayList no_error_list = new ArrayList ();
315
316 protected bool verbose;
317 protected bool safe_execution;
318
319 int total_known_issues;
320
321 protected Checker (ITester tester)
322 {
323 this.tester = tester;
324 }
325
326 public string IssueFile {
327 set {
328 this.issue_file = value;
329 ReadWrongErrors (issue_file);
330 }
331 }
332
333 public string LogFile {
334 set {
335 this.log_file = new StreamWriter (value, false);
336 }
337 }
338
339 public bool Verbose {
340 set {
341 verbose = value;
342 }
343 }
344
345 public bool SafeExecution {
346 set {
347 safe_execution = value;
348 }
349 }
350
351 public string[] ExtraCompilerOptions {
352 set {
353 extra_compiler_options = value;
354 }
355 }
356
357 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
358 out string[] dependencies)
359 {
360 int row = 0;
361 compiler_options = null;
362 dependencies = null;
363 try {
364 using (StreamReader sr = new StreamReader (file)) {
365 String line;
366 while (row++ < 3 && (line = sr.ReadLine()) != null) {
367 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
368 ref dependencies))
369 return false;
370 }
371 }
372 } catch {
373 return false;
374 }
375 return true;
376 }
377
378 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
379 ref string[] compiler_options,
380 ref string[] dependencies)
381 {
382 const string options = "// Compiler options:";
383 const string depends = "// Dependencies:";
384
385 if (row == 1) {
386 compiler_options = null;
387 dependencies = null;
388 }
389
390 int index = line.IndexOf (options);
391 if (index != -1) {
392 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
393 for (int i = 0; i < compiler_options.Length; i++)
394 compiler_options[i] = compiler_options[i].TrimStart ();
395 }
396 index = line.IndexOf (depends);
397 if (index != -1) {
398 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
399 for (int i = 0; i < dependencies.Length; i++)
400 dependencies[i] = dependencies[i].TrimStart ();
401 }
402
403 return true;
404 }
405
406 public bool Do (string filename)
407 {
408 if (test_hash.Contains (filename))
409 return true;
410
411 if (verbose)
412 Log (filename + "...\t");
413
414 if (ignore_list.Contains (filename)) {
415 ++ignored;
416 LogFileLine (filename, "NOT TESTED");
417 return false;
418 }
419
420 string[] compiler_options, dependencies;
421 if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
422 LogFileLine (filename, "ERROR");
423 return false;
424 }
425
426 if (extra_compiler_options != null) {
427 if (compiler_options == null)
428 compiler_options = extra_compiler_options;
429 else {
430 string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
431 extra_compiler_options.CopyTo (new_options, 0);
432 compiler_options.CopyTo (new_options, extra_compiler_options.Length);
433 compiler_options = new_options;
434 }
435 }
436
437 TestCase test = CreateTestCase (filename, compiler_options, dependencies);
438 test_hash.Add (filename, test);
439
440 ++total;
441 if (dependencies != null) {
442 foreach (string dependency in dependencies) {
443 if (!Do (dependency)) {
444 LogFileLine (filename, "DEPENDENCY FAILED");
445 return false;
446 }
447 }
448 }
449
450 tests.Add (test);
451
452 return Check (test);
453 }
454
455 protected virtual bool Check (TestCase test)
456 {
457 string[] test_args;
458
459 if (test.CompilerOptions != null) {
460 test_args = new string [2 + test.CompilerOptions.Length];
461 test.CompilerOptions.CopyTo (test_args, 0);
462 } else {
463 test_args = new string [2];
464 }
465 test_args [test_args.Length - 2] = test.FileName;
466 test_args [test_args.Length - 1] = "-debug";
467
468 return tester.Invoke (test_args);
469 }
470
471 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
472 {
473 return new TestCase (filename, options, deps);
474 }
475
476 void ReadWrongErrors (string file)
477 {
478 const string ignored = "IGNORE";
479 const string no_error = "NO ERROR";
480
481 using (StreamReader sr = new StreamReader (file)) {
482 string line;
483 while ((line = sr.ReadLine()) != null) {
484 if (line.StartsWith ("#"))
485 continue;
486
487 ArrayList active_cont = know_issues;
488
489 if (line.IndexOf (ignored) > 0)
490 active_cont = ignore_list;
491 else if (line.IndexOf (no_error) > 0)
492 active_cont = no_error_list;
493
494 string file_name = line.Split (' ')[0];
495 if (file_name.Length == 0)
496 continue;
497
498 active_cont.Add (file_name);
499 }
500 }
501 total_known_issues = know_issues.Count;
502 }
503
504 protected virtual void PrintSummary ()
505 {
506 LogLine ("Done" + Environment.NewLine);
507 float rate = 0;
508 if (total > 0)
509 rate = (float) (success) / (float)total;
510 LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
511
512 if (syntax_errors > 0)
513 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
514
515 if (ignored > 0)
516 LogLine ("{0} test(s) ignored", ignored);
517
518 if (total_known_issues - know_issues.Count > 0)
519 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
520
521 know_issues.AddRange (no_error_list);
522 if (know_issues.Count > 0) {
523 LogLine ("");
524 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
525 foreach (string s in know_issues)
526 LogLine (s);
527 }
528 if (regression.Count > 0) {
529 LogLine ("");
530 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
531 foreach (string s in regression)
532 LogLine (s);
533 }
534 }
535
536 public int ResultCode
537 {
538 get {
539 return regression.Count == 0 ? 0 : 1;
540 }
541 }
542
543 protected void Log (string msg, params object [] rest)
544 {
545 Console.Write (msg, rest);
546 if (log_file != null)
547 log_file.Write (msg, rest);
548 }
549
550 protected void LogLine (string msg)
551 {
552 Console.WriteLine (msg);
553 if (log_file != null)
554 log_file.WriteLine (msg);
555 }
556
557 protected void LogLine (string msg, params object [] rest)
558 {
559 Console.WriteLine (msg, rest);
560 if (log_file != null)
561 log_file.WriteLine (msg, rest);
562 }
563
564 public void LogFileLine (string file, string msg, params object [] args)
565 {
566 string s = verbose ?
567 string.Format (msg, args) :
568 file + "...\t" + string.Format (msg, args);
569
570 Console.WriteLine (s);
571 if (log_file != null)
572 log_file.WriteLine (s);
573 }
574
575 #region IDisposable Members
576
577 public void Dispose()
578 {
579 if (log_file != null)
580 log_file.Close ();
581 }
582
583 #endregion
584
585 public virtual void Initialize ()
586 {
587 }
588
589 public virtual void CleanUp ()
590 {
591 PrintSummary ();
592 }
593 }
594
595 class PositiveChecker: Checker
596 {
597 readonly string files_folder;
598 readonly static object[] default_args = new object[1] { new string[] {} };
599 string doc_output;
600 string verif_file;
601 bool update_verif_file;
602 Hashtable verif_data;
603
604 #if !NET_2_1
605 ProcessStartInfo pi;
606 #endif
607 readonly string mono;
608
609 public enum TestResult {
610 CompileError,
611 ExecError,
612 LoadError,
613 XmlError,
614 Success,
615 ILError
616 }
617
618 public PositiveChecker (ITester tester, string verif_file):
619 base (tester)
620 {
621 files_folder = Directory.GetCurrentDirectory ();
622 this.verif_file = verif_file;
623
624 #if !NET_2_1
625 pi = new ProcessStartInfo ();
626 pi.CreateNoWindow = true;
627 pi.WindowStyle = ProcessWindowStyle.Hidden;
628 pi.RedirectStandardOutput = true;
629 pi.RedirectStandardError = true;
630 pi.UseShellExecute = false;
631
632 mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
633 if (mono != null) {
634 pi.FileName = mono;
635 }
636 #endif
637 }
638
639 public bool UpdateVerificationDataFile {
640 set {
641 update_verif_file = value;
642 }
643 get {
644 return update_verif_file;
645 }
646 }
647
648 protected override bool GetExtraOptions(string file, out string[] compiler_options,
649 out string[] dependencies) {
650 if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
651 return false;
652
653 doc_output = null;
654 if (compiler_options == null)
655 return true;
656
657 foreach (string one_opt in compiler_options) {
658 if (one_opt.StartsWith ("-doc:")) {
659 doc_output = one_opt.Split (':', '/')[1];
660 }
661 }
662 return true;
663 }
664
665 class DomainTester : MarshalByRefObject
666 {
667 public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
668 {
669 Assembly assembly = Assembly.LoadFile (file);
670
671 bool success = true;
672 Type[] types = assembly.GetTypes ();
673 foreach (Type t in types) {
674
675 // Skip interfaces
676 if (!t.IsClass && !t.IsValueType)
677 continue;
678
679 if (test.VerificationProvider == null) {
680 if (!checker.UpdateVerificationDataFile)
681 checker.LogFileLine (test.FileName, "Missing IL verification data");
682 test.CreateNewTest ();
683 }
684
685 foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
686 MethodBase mi = m as MethodBase;
687 if (mi == null)
688 continue;
689
690 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
691 continue;
692
693 success &= CompareIL (mi, test, checker);
694 }
695 }
696
697 return success;
698 }
699
700 bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
701 {
702 string m_name = mi.ToString ();
703 string decl_type = mi.DeclaringType.ToString ();
704 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
705
706 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
707 if (md == null) {
708 data_provider.AddNewMethod (mi, GetILSize (mi));
709 if (!data_provider.IsNewSet) {
710 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
711 return false;
712 }
713
714 return true;
715 }
716
717 if (md.Checked) {
718 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
719 return false;
720 }
721
722 md.Checked = true;
723
724 int il_size = GetILSize (mi);
725 if (md.ILSize == il_size)
726 return true;
727
728 if (md.ILSize > il_size) {
729 checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
730 md.ILSize = il_size;
731 return true;
732 }
733
734 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
735 string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
736
737 md.ILSize = il_size;
738
739 return false;
740 }
741
742 static int GetILSize (MethodBase mi)
743 {
744 #if NET_2_0
745 MethodBody body = mi.GetMethodBody ();
746 if (body != null)
747 return body.GetILAsByteArray ().Length;
748 #endif
749 return 0;
750 }
751
752 bool ExecuteFile (MethodInfo entry_point, string filename)
753 {
754 TextWriter stdout = Console.Out;
755 TextWriter stderr = Console.Error;
756 Console.SetOut (TextWriter.Null);
757 Console.SetError (TextWriter.Null);
758 ParameterInfo[] pi = entry_point.GetParameters ();
759 object[] args = pi.Length == 0 ? null : default_args;
760
761 object result = null;
762 try {
763 try {
764 result = entry_point.Invoke (null, args);
765 } finally {
766 Console.SetOut (stdout);
767 Console.SetError (stderr);
768 }
769 } catch (Exception e) {
770 throw new ApplicationException (e.ToString ());
771 }
772
773 if (result is int && (int) result != 0)
774 throw new ApplicationException ("Wrong return code: " + result.ToString ());
775
776 return true;
777 }
778
779 public bool Test (string file)
780 {
781 Assembly assembly = Assembly.LoadFile (file);
782 return ExecuteFile (assembly.EntryPoint, file);
783 }
784 }
785
786 protected override bool Check(TestCase test)
787 {
788 string filename = test.FileName;
789 try {
790 if (!base.Check (test)) {
791 HandleFailure (filename, TestResult.CompileError, tester.Output);
792 return false;
793 }
794 }
795 catch (Exception e) {
796 if (e.InnerException != null)
797 e = e.InnerException;
798
799 HandleFailure (filename, TestResult.CompileError, e.ToString ());
800 return false;
801 }
802
803 // Test setup
804 if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
805 if (verbose)
806 LogFileLine (filename, "OK");
807 --total;
808 return true;
809 }
810
811 string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
812
813 // Enable .dll only tests (no execution required)
814 if (!File.Exists(file)) {
815 HandleFailure (filename, TestResult.Success, null);
816 return true;
817 }
818
819 AppDomain domain = null;
820 #if !NET_2_1
821 if (safe_execution) {
822 // Create a new AppDomain, with the current directory as the base.
823 AppDomainSetup setupInfo = new AppDomainSetup ();
824 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
825 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
826 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
827 }
828 #endif
829 try {
830 DomainTester tester;
831 try {
832 #if !NET_2_1
833 if (domain != null)
834 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
835 else
836 #endif
837 tester = new DomainTester ();
838
839 if (!tester.Test (file))
840 return false;
841
842 } catch (ApplicationException e) {
843 HandleFailure (filename, TestResult.ExecError, e.Message);
844 return false;
845 } catch (Exception e) {
846 HandleFailure (filename, TestResult.LoadError, e.ToString ());
847 return false;
848 }
849
850 if (doc_output != null) {
851 string ref_file = filename.Replace (".cs", "-ref.xml");
852 try {
853 #if !NET_2_1
854 XmlComparer.Compare (ref_file, doc_output);
855 #endif
856 } catch (Exception e) {
857 HandleFailure (filename, TestResult.XmlError, e.Message);
858 return false;
859 }
860 } else {
861 if (verif_file != null) {
862 PositiveTestCase pt = (PositiveTestCase) test;
863 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
864
865 if (!tester.CheckILSize (pt, this, file))
866 return false;
867 }
868 }
869 } finally {
870 if (domain != null)
871 AppDomain.Unload (domain);
872 }
873
874 HandleFailure (filename, TestResult.Success, null);
875 return true;
876 }
877
878 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
879 {
880 return new PositiveTestCase (filename, options, deps);
881 }
882
883 public void HandleFailure (string file, TestResult status, string extra)
884 {
885 switch (status) {
886 case TestResult.Success:
887 success++;
888 if (know_issues.Contains (file)) {
889 LogFileLine (file, "FIXED ISSUE");
890 return;
891 }
892 if (verbose)
893 LogFileLine (file, "OK");
894 return;
895
896 case TestResult.CompileError:
897 if (know_issues.Contains (file)) {
898 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
899 know_issues.Remove (file);
900 return;
901 }
902 LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
903 break;
904
905 case TestResult.ExecError:
906 if (know_issues.Contains (file)) {
907 LogFileLine (file, "KNOWN ISSUE (Execution error)");
908 know_issues.Remove (file);
909 return;
910 }
911 LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
912 break;
913
914 case TestResult.XmlError:
915 if (know_issues.Contains (file)) {
916 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
917 know_issues.Remove (file);
918 return;
919 }
920 LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
921 break;
922
923 case TestResult.LoadError:
924 LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
925 break;
926
927 case TestResult.ILError:
928 if (!update_verif_file) {
929 LogFileLine (file, "IL REGRESSION: " + extra);
930 }
931 extra = null;
932 break;
933 }
934
935 if (extra != null)
936 LogLine ("{0}", extra);
937
938 if (!regression.Contains (file))
939 regression.Add (file);
940 }
941
942 public override void Initialize ()
943 {
944 if (verif_file != null) {
945 #if NET_2_0
946 LoadVerificationData (verif_file);
947 #else
948 throw new NotSupportedException ();
949 #endif
950 }
951
952 base.Initialize ();
953 }
954
955 public override void CleanUp ()
956 {
957 base.CleanUp ();
958
959 if (update_verif_file) {
960 #if NET_2_0
961 UpdateVerificationData (verif_file);
962 #else
963 throw new NotSupportedException ();
964 #endif
965 }
966 }
967
968 #if NET_2_0
969 void LoadVerificationData (string file)
970 {
971 LogLine ("Loading verification data from `{0}' ...", file);
972
973 using (XmlReader r = XmlReader.Create (file)) {
974 r.ReadStartElement ("tests");
975 verif_data = new Hashtable ();
976
977 while (r.Read ()) {
978 if (r.Name != "test")
979 continue;
980
981 string name = r.GetAttribute ("name");
982 PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
983 verif_data.Add (name, tc);
984 }
985 }
986 }
987
988 void UpdateVerificationData (string file)
989 {
990 LogLine ("Updating verification data `{0}' ...", file);
991
992 XmlWriterSettings s = new XmlWriterSettings ();
993 s.Indent = true;
994 using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
995 w.WriteStartDocument ();
996 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
997 w.WriteStartElement ("tests");
998 foreach (PositiveTestCase tc in tests) {
999 if (tc.VerificationProvider != null)
1000 tc.VerificationProvider.WriteCodeInfoTo (w);
1001 }
1002 w.WriteEndElement ();
1003 }
1004 }
1005 #endif
1006 }
1007
1008 class NegativeChecker: Checker
1009 {
1010 string expected_message;
1011 string error_message;
1012 bool check_msg;
1013 bool check_error_line;
1014 bool is_warning;
1015 IDictionary wrong_warning;
1016
1017 protected enum CompilerError {
1018 Expected,
1019 Wrong,
1020 Missing,
1021 WrongMessage,
1022 MissingLocation,
1023 Duplicate
1024 }
1025
1026 public NegativeChecker (ITester tester, bool check_msg):
1027 base (tester)
1028 {
1029 this.check_msg = check_msg;
1030 wrong_warning = new Hashtable ();
1031 }
1032
1033 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1034 ref string[] compiler_options,
1035 ref string[] dependencies)
1036 {
1037 if (row == 1) {
1038 expected_message = null;
1039
1040 int index = line.IndexOf (':');
1041 if (index == -1 || index > 15) {
1042 LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1043 ++syntax_errors;
1044 base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1045 ref dependencies);
1046 return false;
1047 }
1048
1049 expected_message = line.Substring (index + 1).Trim ();
1050 }
1051
1052 if (row == 2) {
1053 string filtered = line.Replace(" ", "");
1054
1055 // Some error tests require to have different error text for different runtimes.
1056 if (filtered.StartsWith ("//GMCS")) {
1057 row = 1;
1058 #if !NET_2_0
1059 return true;
1060 #else
1061 return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1062 #endif
1063 }
1064
1065 check_error_line = !filtered.StartsWith ("//Line:0");
1066
1067 if (!filtered.StartsWith ("//Line:")) {
1068 LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1069 ++syntax_errors;
1070 return false;
1071 }
1072 }
1073
1074 if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1075 return false;
1076
1077 is_warning = false;
1078 if (compiler_options != null) {
1079 foreach (string s in compiler_options) {
1080 if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1081 is_warning = true;
1082 }
1083 }
1084 return true;
1085 }
1086
1087
1088 protected override bool Check (TestCase test)
1089 {
1090 string filename = test.FileName;
1091
1092 int start_char = 0;
1093 while (Char.IsLetter (filename, start_char))
1094 ++start_char;
1095
1096 int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1097 string expected = filename.Substring (start_char, end_char - start_char);
1098
1099 try {
1100 if (base.Check (test)) {
1101 HandleFailure (filename, CompilerError.Missing);
1102 return false;
1103 }
1104 }
1105 catch (Exception e) {
1106 HandleFailure (filename, CompilerError.Missing);
1107 if (e.InnerException != null)
1108 e = e.InnerException;
1109
1110 Log (e.ToString ());
1111 return false;
1112 }
1113
1114 int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1115 if (tester.IsWarning (err_id)) {
1116 if (!is_warning)
1117 wrong_warning [err_id] = true;
1118 } else {
1119 if (is_warning)
1120 wrong_warning [err_id] = false;
1121 }
1122
1123 CompilerError result_code = GetCompilerError (expected, tester.Output);
1124 if (HandleFailure (filename, result_code)) {
1125 success++;
1126 return true;
1127 }
1128
1129 if (result_code == CompilerError.Wrong)
1130 LogLine (tester.Output);
1131
1132 return false;
1133 }
1134
1135 CompilerError GetCompilerError (string expected, string buffer)
1136 {
1137 const string error_prefix = "CS";
1138 const string ignored_error = "error CS5001";
1139 string tested_text = "error " + error_prefix + expected;
1140 StringReader sr = new StringReader (buffer);
1141 string line = sr.ReadLine ();
1142 ArrayList ld = new ArrayList ();
1143 CompilerError result = CompilerError.Missing;
1144 while (line != null) {
1145 if (ld.Contains (line) && result == CompilerError.Expected) {
1146 if (line.IndexOf ("Location of the symbol related to previous") == -1)
1147 return CompilerError.Duplicate;
1148 }
1149 ld.Add (line);
1150
1151 if (result != CompilerError.Expected) {
1152 if (line.IndexOf (tested_text) != -1) {
1153 if (check_msg) {
1154 int first = line.IndexOf (':');
1155 int second = line.IndexOf (':', first + 1);
1156 if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
1157 if (check_error_line) {
1158 second = line.IndexOf (':', second + 1);
1159 }
1160 } else if (second == -1 || !check_error_line) {
1161 second = first;
1162 }
1163
1164 string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1165 if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1166 error_message = msg;
1167 return CompilerError.WrongMessage;
1168 }
1169
1170 if (check_error_line && line.IndexOf (".cs(") == -1)
1171 return CompilerError.MissingLocation;
1172 }
1173 result = CompilerError.Expected;
1174 } else if (line.IndexOf (error_prefix) != -1 &&
1175 line.IndexOf (ignored_error) == -1)
1176 result = CompilerError.Wrong;
1177 }
1178
1179 line = sr.ReadLine ();
1180 }
1181
1182 return result;
1183 }
1184
1185 bool HandleFailure (string file, CompilerError status)
1186 {
1187 switch (status) {
1188 case CompilerError.Expected:
1189 if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1190 LogFileLine (file, "FIXED ISSUE");
1191 return true;
1192 }
1193
1194 if (verbose)
1195 LogFileLine (file, "OK");
1196 return true;
1197
1198 case CompilerError.Wrong:
1199 if (know_issues.Contains (file)) {
1200 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1201 know_issues.Remove (file);
1202 return false;
1203 }
1204 if (no_error_list.Contains (file)) {
1205 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1206 no_error_list.Remove (file);
1207 }
1208 else {
1209 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1210 }
1211 break;
1212
1213 case CompilerError.WrongMessage:
1214 if (know_issues.Contains (file)) {
1215 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1216 know_issues.Remove (file);
1217 return false;
1218 }
1219 if (no_error_list.Contains (file)) {
1220 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1221 no_error_list.Remove (file);
1222 }
1223 else {
1224 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1225 LogLine ("Exp: {0}", expected_message);
1226 LogLine ("Was: {0}", error_message);
1227 }
1228 break;
1229
1230 case CompilerError.Missing:
1231 if (no_error_list.Contains (file)) {
1232 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1233 no_error_list.Remove (file);
1234 return false;
1235 }
1236
1237 if (know_issues.Contains (file)) {
1238 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1239 know_issues.Remove (file);
1240 }
1241 else {
1242 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1243 }
1244
1245 break;
1246
1247 case CompilerError.MissingLocation:
1248 if (know_issues.Contains (file)) {
1249 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1250 know_issues.Remove (file);
1251 return false;
1252 }
1253 if (no_error_list.Contains (file)) {
1254 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1255 no_error_list.Remove (file);
1256 }
1257 else {
1258 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1259 }
1260 break;
1261
1262 case CompilerError.Duplicate:
1263 // Will become an error soon
1264 LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1265 return true;
1266 }
1267
1268 regression.Add (file);
1269 return false;
1270 }
1271
1272 protected override void PrintSummary()
1273 {
1274 base.PrintSummary ();
1275
1276 if (wrong_warning.Count > 0) {
1277 LogLine ("");
1278 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1279 LogLine ("");
1280 foreach (DictionaryEntry de in wrong_warning)
1281 LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1282 }
1283 }
1284
1285 }
1286
1287 class Tester {
1288
1289 static int Main(string[] args)
1290 {
1291 string temp;
1292
1293 if (GetOption ("help", args, false, out temp)) {
1294 Usage ();
1295 return 0;
1296 }
1297
1298 string compiler;
1299 if (!GetOption ("compiler", args, true, out compiler)) {
1300 Usage ();
1301 return 1;
1302 }
1303
1304 ITester tester;
1305 try {
1306 Console.WriteLine ("Loading " + compiler + " ...");
1307 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1308 }
1309 catch (Exception) {
1310 #if NET_2_1
1311 throw;
1312 #else
1313 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1314 if (!File.Exists (compiler)) {
1315 Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1316 return 1;
1317 }
1318 tester = new ProcessTester (compiler);
1319 #endif
1320 }
1321
1322 string mode;
1323 if (!GetOption ("mode", args, true, out mode)) {
1324 Usage ();
1325 return 1;
1326 }
1327
1328 Checker checker;
1329 bool positive;
1330 switch (mode) {
1331 case "neg":
1332 checker = new NegativeChecker (tester, true);
1333 positive = false;
1334 break;
1335 case "pos":
1336 string iltest;
1337 GetOption ("il", args, false, out iltest);
1338 checker = new PositiveChecker (tester, iltest);
1339 positive = true;
1340
1341 if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1342 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1343 }
1344
1345 break;
1346 default:
1347 Console.Error.WriteLine ("Invalid -mode argument");
1348 return 1;
1349 }
1350
1351
1352 if (GetOption ("issues", args, true, out temp))
1353 checker.IssueFile = temp;
1354 if (GetOption ("log", args, true, out temp))
1355 checker.LogFile = temp;
1356 if (GetOption ("verbose", args, false, out temp))
1357 checker.Verbose = true;
1358 if (GetOption ("safe-execution", args, false, out temp))
1359 checker.SafeExecution = true;
1360 if (GetOption ("compiler-options", args, true, out temp)) {
1361 string[] extra = temp.Split (' ');
1362 checker.ExtraCompilerOptions = extra;
1363 }
1364
1365 string test_pattern;
1366 if (!GetOption ("files", args, true, out test_pattern)) {
1367 Usage ();
1368 return 1;
1369 }
1370
1371 var files = new List<string> ();
1372 switch (test_pattern) {
1373 case "v1":
1374 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1375 break;
1376 case "v2":
1377 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1378 goto case "v1";
1379 case "v4":
1380 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1381 goto case "v2";
1382 default:
1383 files.AddRange (Directory.GetFiles (".", test_pattern));
1384 break;
1385 }
1386
1387 if (files.Count == 0) {
1388 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1389 return 2;
1390 }
1391
1392 checker.Initialize ();
1393 /*
1394 files.Sort ((a, b) => {
1395 if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1396 if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
1397 return -1;
1398 } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
1399 if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
1400 return 1;
1401 }
1402
1403 return a.CompareTo (b);
1404 });
1405 */
1406 foreach (string s in files) {
1407 string filename = Path.GetFileName (s);
1408 if (Char.IsUpper (filename, 0)) { // Windows hack
1409 continue;
1410 }
1411
1412 if (filename.EndsWith ("-p2.cs"))
1413 continue;
1414
1415 checker.Do (filename);
1416 }
1417
1418 checker.CleanUp ();
1419
1420 checker.Dispose ();
1421
1422 return checker.ResultCode;
1423 }
1424
1425 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1426 {
1427 opt = "-" + opt;
1428 foreach (string a in args) {
1429 if (a.StartsWith (opt)) {
1430 int sep = a.IndexOf (':');
1431 if (sep > 0) {
1432 value = a.Substring (sep + 1);
1433 } else {
1434 value = null;
1435 if (req_arg) {
1436 Console.Error.WriteLine ("Missing argument in option " + opt);
1437 return false;
1438 }
1439 }
1440
1441 return true;
1442 }
1443 }
1444
1445 value = null;
1446 return false;
1447 }
1448
1449 static void Usage ()
1450 {
1451 Console.WriteLine (
1452 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1453 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1454 " \n" +
1455 " -compiler:FILE The file which will be used to compiler tests\n" +
1456 " -compiler-options:OPTIONS Add global compiler options\n" +
1457 " -il:IL-FILE XML file with expected IL details for each test\n" +
1458 " -issues:FILE The list of expected failures\n" +
1459 " -log:FILE Writes any output also to the file\n" +
1460 " -help Lists all options\n" +
1461 " -mode:[pos|neg] Specifies compiler test mode\n" +
1462 " -safe-execution Runs compiled executables in separate app-domain\n" +
1463 " -update-il Updates IL-FILE to match compiler output\n" +
1464 " -verbose Prints more details during testing\n"
1465 );
1466 }
1467 }
1468 }