?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
2010-04-08 Jb Evain <jbevain@novell.com>
[mcs.git] / tools / compiler-tester / compiler-tester.cs
blob89a8bec67daf731971901c84ddd5f9c61ad0ccf2
1 //
2 // compiler-tester.cs
3 //
4 // Author:
5 // Marek Safar (marek.safar@gmail.com)
6 //
7
8 //
9 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
10 //
11 // Permission is hereby granted, free of charge, to any person obtaining
12 // a copy of this software and associated documentation files (the
13 // "Software"), to deal in the Software without restriction, including
14 // without limitation the rights to use, copy, modify, merge, publish,
15 // distribute, sublicense, and/or sell copies of the Software, and to
16 // permit persons to whom the Software is furnished to do so, subject to
17 // the following conditions:
18 //
19 // The above copyright notice and this permission notice shall be
20 // included in all copies or substantial portions of the Software.
21 //
22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 //
30
31 using System;
32 using System.IO;
33 using System.Diagnostics;
34 using System.Reflection;
35 using System.Text;
36 using System.Collections;
37 using System.Xml;
38
39 namespace TestRunner {
40
41 interface ITester
42 {
43 string Output { get; }
44 bool Invoke (string[] args);
45 bool IsWarning (int warningNumber);
46 }
47
48 class ReflectionTester: ITester {
49 MethodInfo ep;
50 object[] method_arg;
51 StringWriter output;
52 int[] all_warnings;
53
54 public ReflectionTester (Assembly a)
55 {
56 Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
57
58 if (t == null)
59 Console.Error.WriteLine ("null, huh?");
60
61 ep = t.GetMethod ("InvokeCompiler",
62 BindingFlags.Static | BindingFlags.Public);
63 if (ep == null)
64 throw new MissingMethodException ("static InvokeCompiler");
65 method_arg = new object [2];
66
67 PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
68 all_warnings = (int[])pi.GetValue (null, null);
69 Array.Sort (all_warnings);
70 }
71
72 public string Output {
73 get {
74 return output.GetStringBuilder ().ToString ();
75 }
76 }
77
78 public bool Invoke(string[] args)
79 {
80 output = new StringWriter ();
81 method_arg [0] = args;
82 method_arg [1] = output;
83 return (bool)ep.Invoke (null, method_arg);
84 }
85
86 public bool IsWarning (int warningNumber)
87 {
88 return Array.BinarySearch (all_warnings, warningNumber) >= 0;
89 }
90 }
91
92 #if !NET_2_1
93 class ProcessTester: ITester
94 {
95 ProcessStartInfo pi;
96 string output;
97
98 public ProcessTester (string p_path)
99 {
100 pi = new ProcessStartInfo ();
101 pi.FileName = p_path;
102 pi.CreateNoWindow = true;
103 pi.WindowStyle = ProcessWindowStyle.Hidden;
104 pi.RedirectStandardOutput = true;
105 pi.RedirectStandardError = true;
106 pi.UseShellExecute = false;
107 }
108
109 public string Output {
110 get {
111 return output;
112 }
113 }
114
115 public bool Invoke(string[] args)
116 {
117 StringBuilder sb = new StringBuilder ("/nologo ");
118 foreach (string s in args) {
119 sb.Append (s);
120 sb.Append (" ");
121 }
122 pi.Arguments = sb.ToString ();
123 Process p = Process.Start (pi);
124 output = p.StandardError.ReadToEnd ();
125 if (output.Length == 0)
126 output = p.StandardOutput.ReadToEnd ();
127 p.WaitForExit ();
128 return p.ExitCode == 0;
129 }
130
131 public bool IsWarning (int warningNumber)
132 {
133 throw new NotImplementedException ();
134 }
135 }
136 #endif
137
138 class TestCase : MarshalByRefObject
139 {
140 public readonly string FileName;
141 public readonly string[] CompilerOptions;
142 public readonly string[] Dependencies;
143
144 public TestCase (string filename, string[] options, string[] deps)
145 {
146 this.FileName = filename;
147 this.CompilerOptions = options;
148 this.Dependencies = deps;
149 }
150 }
151
152 class PositiveTestCase : TestCase
153 {
154 public class VerificationData : MarshalByRefObject
155 {
156 public class MethodData : MarshalByRefObject
157 {
158 public MethodData (MethodBase mi, int il_size)
159 {
160 this.Type = mi.DeclaringType.ToString ();
161 this.MethodName = mi.ToString ();
162 this.ILSize = il_size;
163 }
164
165 public MethodData (string type_name, string method_name, int il_size)
166 {
167 this.Type = type_name;
168 this.MethodName = method_name;
169 this.ILSize = il_size;
170 }
171
172 public string Type;
173 public string MethodName;
174 public int ILSize;
175 public bool Checked;
176 }
177
178 ArrayList methods;
179 public bool IsNewSet;
180
181 public VerificationData (string test_file)
182 {
183 #if NET_2_0
184 this.test_file = test_file;
185 #endif
186 }
187
188 #if NET_2_0
189 string test_file;
190
191 public static VerificationData FromFile (string name, XmlReader r)
192 {
193 VerificationData tc = new VerificationData (name);
194 ArrayList methods = new ArrayList ();
195 r.Read ();
196 while (r.ReadToNextSibling ("type")) {
197 string type_name = r ["name"];
198 r.Read ();
199 while (r.ReadToNextSibling ("method")) {
200 string m_name = r ["name"];
201
202 r.ReadToDescendant ("size");
203 int il_size = r.ReadElementContentAsInt ();
204 methods.Add (new MethodData (type_name, m_name, il_size));
205 r.Read ();
206 }
207 r.Read ();
208 }
209
210 tc.methods = methods;
211 return tc;
212 }
213
214 public void WriteCodeInfoTo (XmlWriter w)
215 {
216 w.WriteStartElement ("test");
217 w.WriteAttributeString ("name", test_file);
218
219 string type = null;
220 foreach (MethodData data in methods) {
221 if (!data.Checked)
222 continue;
223
224 if (type != data.Type) {
225 if (type != null)
226 w.WriteEndElement ();
227
228 type = data.Type;
229 w.WriteStartElement ("type");
230 w.WriteAttributeString ("name", type);
231 }
232
233 w.WriteStartElement ("method");
234 w.WriteAttributeString ("name", data.MethodName);
235 w.WriteStartElement ("size");
236 w.WriteValue (data.ILSize);
237 w.WriteEndElement ();
238 w.WriteEndElement ();
239 }
240
241 if (type != null)
242 w.WriteEndElement ();
243
244 w.WriteEndElement ();
245 }
246 #endif
247
248 public MethodData FindMethodData (string method_name, string declaring_type)
249 {
250 if (methods == null)
251 return null;
252
253 foreach (MethodData md in methods) {
254 if (md.MethodName == method_name && md.Type == declaring_type)
255 return md;
256 }
257
258 return null;
259 }
260
261 public void AddNewMethod (MethodBase mb, int il_size)
262 {
263 if (methods == null)
264 methods = new ArrayList ();
265
266 MethodData md = new MethodData (mb, il_size);
267 md.Checked = true;
268 methods.Add (md);
269 }
270 }
271
272 VerificationData verif_data;
273
274 public PositiveTestCase (string filename, string [] options, string [] deps)
275 : base (filename, options, deps)
276 {
277 }
278
279 public void CreateNewTest ()
280 {
281 verif_data = new VerificationData (FileName);
282 verif_data.IsNewSet = true;
283 }
284
285 public VerificationData VerificationProvider {
286 set {
287 verif_data = value;
288 }
289 get {
290 return verif_data;
291 }
292 }
293 }
294
295 class Checker: MarshalByRefObject, IDisposable
296 {
297 protected ITester tester;
298 protected int success;
299 protected int total;
300 protected int ignored;
301 protected int syntax_errors;
302 string issue_file;
303 StreamWriter log_file;
304 protected string[] extra_compiler_options;
305 // protected string[] compiler_options;
306 // protected string[] dependencies;
307
308 protected ArrayList tests = new ArrayList ();
309 protected Hashtable test_hash = new Hashtable ();
310 protected ArrayList regression = new ArrayList ();
311 protected ArrayList know_issues = new ArrayList ();
312 protected ArrayList ignore_list = new ArrayList ();
313 protected ArrayList no_error_list = new ArrayList ();
314
315 protected bool verbose;
316 protected bool safe_execution;
317
318 int total_known_issues;
319
320 protected Checker (ITester tester)
321 {
322 this.tester = tester;
323 }
324
325 public string IssueFile {
326 set {
327 this.issue_file = value;
328 ReadWrongErrors (issue_file);
329 }
330 }
331
332 public string LogFile {
333 set {
334 this.log_file = new StreamWriter (value, false);
335 }
336 }
337
338 public bool Verbose {
339 set {
340 verbose = value;
341 }
342 }
343
344 public bool SafeExecution {
345 set {
346 safe_execution = value;
347 }
348 }
349
350 public string[] ExtraCompilerOptions {
351 set {
352 extra_compiler_options = value;
353 }
354 }
355
356 protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
357 out string[] dependencies)
358 {
359 int row = 0;
360 compiler_options = null;
361 dependencies = null;
362 try {
363 using (StreamReader sr = new StreamReader (file)) {
364 String line;
365 while (row++ < 3 && (line = sr.ReadLine()) != null) {
366 if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
367 ref dependencies))
368 return false;
369 }
370 }
371 } catch {
372 return false;
373 }
374 return true;
375 }
376
377 protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
378 ref string[] compiler_options,
379 ref string[] dependencies)
380 {
381 const string options = "// Compiler options:";
382 const string depends = "// Dependencies:";
383
384 if (row == 1) {
385 compiler_options = null;
386 dependencies = null;
387 }
388
389 int index = line.IndexOf (options);
390 if (index != -1) {
391 compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
392 for (int i = 0; i < compiler_options.Length; i++)
393 compiler_options[i] = compiler_options[i].TrimStart ();
394 }
395 index = line.IndexOf (depends);
396 if (index != -1) {
397 dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
398 for (int i = 0; i < dependencies.Length; i++)
399 dependencies[i] = dependencies[i].TrimStart ();
400 }
401
402 return true;
403 }
404
405 public bool Do (string filename)
406 {
407 if (test_hash.Contains (filename))
408 return true;
409
410 if (verbose)
411 Log (filename + "...\t");
412
413 if (ignore_list.Contains (filename)) {
414 ++ignored;
415 LogFileLine (filename, "NOT TESTED");
416 return false;
417 }
418
419 string[] compiler_options, dependencies;
420 if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
421 LogFileLine (filename, "ERROR");
422 return false;
423 }
424
425 if (extra_compiler_options != null) {
426 if (compiler_options == null)
427 compiler_options = extra_compiler_options;
428 else {
429 string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
430 extra_compiler_options.CopyTo (new_options, 0);
431 compiler_options.CopyTo (new_options, extra_compiler_options.Length);
432 compiler_options = new_options;
433 }
434 }
435
436 TestCase test = CreateTestCase (filename, compiler_options, dependencies);
437 test_hash.Add (filename, test);
438
439 ++total;
440 if (dependencies != null) {
441 foreach (string dependency in dependencies) {
442 if (!Do (dependency)) {
443 LogFileLine (filename, "DEPENDENCY FAILED");
444 return false;
445 }
446 }
447 }
448
449 tests.Add (test);
450
451 return Check (test);
452 }
453
454 protected virtual bool Check (TestCase test)
455 {
456 string[] test_args;
457
458 if (test.CompilerOptions != null) {
459 test_args = new string [2 + test.CompilerOptions.Length];
460 test.CompilerOptions.CopyTo (test_args, 0);
461 } else {
462 test_args = new string [2];
463 }
464 test_args [test_args.Length - 2] = test.FileName;
465 test_args [test_args.Length - 1] = "-debug";
466
467 return tester.Invoke (test_args);
468 }
469
470 protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
471 {
472 return new TestCase (filename, options, deps);
473 }
474
475 void ReadWrongErrors (string file)
476 {
477 const string ignored = "IGNORE";
478 const string no_error = "NO ERROR";
479
480 using (StreamReader sr = new StreamReader (file)) {
481 string line;
482 while ((line = sr.ReadLine()) != null) {
483 if (line.StartsWith ("#"))
484 continue;
485
486 ArrayList active_cont = know_issues;
487
488 if (line.IndexOf (ignored) > 0)
489 active_cont = ignore_list;
490 else if (line.IndexOf (no_error) > 0)
491 active_cont = no_error_list;
492
493 string file_name = line.Split (' ')[0];
494 if (file_name.Length == 0)
495 continue;
496
497 active_cont.Add (file_name);
498 }
499 }
500 total_known_issues = know_issues.Count;
501 }
502
503 protected virtual void PrintSummary ()
504 {
505 LogLine ("Done" + Environment.NewLine);
506 float rate = 0;
507 if (total > 0)
508 rate = (float) (success) / (float)total;
509 LogLine ("{0} test cases passed ({1:0.##%})", success, rate);
510
511 if (syntax_errors > 0)
512 LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
513
514 if (ignored > 0)
515 LogLine ("{0} test(s) ignored", ignored);
516
517 if (total_known_issues - know_issues.Count > 0)
518 LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
519
520 know_issues.AddRange (no_error_list);
521 if (know_issues.Count > 0) {
522 LogLine ("");
523 LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
524 foreach (string s in know_issues)
525 LogLine (s);
526 }
527 if (regression.Count > 0) {
528 LogLine ("");
529 LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
530 foreach (string s in regression)
531 LogLine (s);
532 }
533 }
534
535 public int ResultCode
536 {
537 get {
538 return regression.Count == 0 ? 0 : 1;
539 }
540 }
541
542 protected void Log (string msg, params object [] rest)
543 {
544 Console.Write (msg, rest);
545 if (log_file != null)
546 log_file.Write (msg, rest);
547 }
548
549 protected void LogLine (string msg)
550 {
551 Console.WriteLine (msg);
552 if (log_file != null)
553 log_file.WriteLine (msg);
554 }
555
556 protected void LogLine (string msg, params object [] rest)
557 {
558 Console.WriteLine (msg, rest);
559 if (log_file != null)
560 log_file.WriteLine (msg, rest);
561 }
562
563 public void LogFileLine (string file, string msg, params object [] args)
564 {
565 string s = verbose ?
566 string.Format (msg, args) :
567 file + "...\t" + string.Format (msg, args);
568
569 Console.WriteLine (s);
570 if (log_file != null)
571 log_file.WriteLine (s);
572 }
573
574 #region IDisposable Members
575
576 public void Dispose()
577 {
578 if (log_file != null)
579 log_file.Close ();
580 }
581
582 #endregion
583
584 public virtual void Initialize ()
585 {
586 }
587
588 public virtual void CleanUp ()
589 {
590 PrintSummary ();
591 }
592 }
593
594 class PositiveChecker: Checker
595 {
596 readonly string files_folder;
597 readonly static object[] default_args = new object[1] { new string[] {} };
598 string doc_output;
599 string verif_file;
600 bool update_verif_file;
601 Hashtable verif_data;
602
603 #if !NET_2_1
604 ProcessStartInfo pi;
605 #endif
606 readonly string mono;
607
608 public enum TestResult {
609 CompileError,
610 ExecError,
611 LoadError,
612 XmlError,
613 Success,
614 ILError
615 }
616
617 public PositiveChecker (ITester tester, string verif_file):
618 base (tester)
619 {
620 files_folder = Directory.GetCurrentDirectory ();
621 this.verif_file = verif_file;
622
623 #if !NET_2_1
624 pi = new ProcessStartInfo ();
625 pi.CreateNoWindow = true;
626 pi.WindowStyle = ProcessWindowStyle.Hidden;
627 pi.RedirectStandardOutput = true;
628 pi.RedirectStandardError = true;
629 pi.UseShellExecute = false;
630
631 mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
632 if (mono != null) {
633 pi.FileName = mono;
634 }
635 #endif
636 }
637
638 public bool UpdateVerificationDataFile {
639 set {
640 update_verif_file = value;
641 }
642 get {
643 return update_verif_file;
644 }
645 }
646
647 protected override bool GetExtraOptions(string file, out string[] compiler_options,
648 out string[] dependencies) {
649 if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
650 return false;
651
652 doc_output = null;
653 if (compiler_options == null)
654 return true;
655
656 foreach (string one_opt in compiler_options) {
657 if (one_opt.StartsWith ("-doc:")) {
658 doc_output = one_opt.Split (':', '/')[1];
659 }
660 }
661 return true;
662 }
663
664 class DomainTester : MarshalByRefObject
665 {
666 public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
667 {
668 Assembly assembly = Assembly.LoadFile (file);
669
670 bool success = true;
671 Type[] types = assembly.GetTypes ();
672 foreach (Type t in types) {
673
674 // Skip interfaces
675 if (!t.IsClass && !t.IsValueType)
676 continue;
677
678 if (test.VerificationProvider == null) {
679 if (!checker.UpdateVerificationDataFile)
680 checker.LogFileLine (test.FileName, "Missing IL verification data");
681 test.CreateNewTest ();
682 }
683
684 foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
685 MethodBase mi = m as MethodBase;
686 if (mi == null)
687 continue;
688
689 if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
690 continue;
691
692 success &= CompareIL (mi, test, checker);
693 }
694 }
695
696 return success;
697 }
698
699 bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
700 {
701 string m_name = mi.ToString ();
702 string decl_type = mi.DeclaringType.ToString ();
703 PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
704
705 PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
706 if (md == null) {
707 data_provider.AddNewMethod (mi, GetILSize (mi));
708 if (!data_provider.IsNewSet) {
709 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
710 return false;
711 }
712
713 return true;
714 }
715
716 if (md.Checked) {
717 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
718 return false;
719 }
720
721 md.Checked = true;
722
723 int il_size = GetILSize (mi);
724 if (md.ILSize == il_size)
725 return true;
726
727 if (md.ILSize > il_size) {
728 checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
729 md.ILSize = il_size;
730 return true;
731 }
732
733 checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
734 string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
735
736 md.ILSize = il_size;
737
738 return false;
739 }
740
741 static int GetILSize (MethodBase mi)
742 {
743 #if NET_2_0
744 MethodBody body = mi.GetMethodBody ();
745 if (body != null)
746 return body.GetILAsByteArray ().Length;
747 #endif
748 return 0;
749 }
750
751 bool ExecuteFile (MethodInfo entry_point, string filename)
752 {
753 TextWriter stdout = Console.Out;
754 TextWriter stderr = Console.Error;
755 Console.SetOut (TextWriter.Null);
756 Console.SetError (TextWriter.Null);
757 ParameterInfo[] pi = entry_point.GetParameters ();
758 object[] args = pi.Length == 0 ? null : default_args;
759
760 object result = null;
761 try {
762 try {
763 result = entry_point.Invoke (null, args);
764 } finally {
765 Console.SetOut (stdout);
766 Console.SetError (stderr);
767 }
768 } catch (Exception e) {
769 throw new ApplicationException (e.ToString ());
770 }
771
772 if (result is int && (int) result != 0)
773 throw new ApplicationException ("Wrong return code: " + result.ToString ());
774
775 return true;
776 }
777
778 public bool Test (string file)
779 {
780 Assembly assembly = Assembly.LoadFile (file);
781 return ExecuteFile (assembly.EntryPoint, file);
782 }
783 }
784
785 protected override bool Check(TestCase test)
786 {
787 string filename = test.FileName;
788 try {
789 if (!base.Check (test)) {
790 HandleFailure (filename, TestResult.CompileError, tester.Output);
791 return false;
792 }
793 }
794 catch (Exception e) {
795 if (e.InnerException != null)
796 e = e.InnerException;
797
798 HandleFailure (filename, TestResult.CompileError, e.ToString ());
799 return false;
800 }
801
802 // Test setup
803 if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
804 if (verbose)
805 LogFileLine (filename, "OK");
806 --total;
807 return true;
808 }
809
810 string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
811
812 // Enable .dll only tests (no execution required)
813 if (!File.Exists(file)) {
814 HandleFailure (filename, TestResult.Success, null);
815 return true;
816 }
817
818 AppDomain domain = null;
819 #if !NET_2_1
820 if (safe_execution) {
821 // Create a new AppDomain, with the current directory as the base.
822 AppDomainSetup setupInfo = new AppDomainSetup ();
823 setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
824 setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
825 domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (file), null, setupInfo);
826 }
827 #endif
828 try {
829 DomainTester tester;
830 try {
831 #if !NET_2_1
832 if (domain != null)
833 tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
834 else
835 #endif
836 tester = new DomainTester ();
837
838 if (!tester.Test (file))
839 return false;
840
841 } catch (ApplicationException e) {
842 HandleFailure (filename, TestResult.ExecError, e.Message);
843 return false;
844 } catch (Exception e) {
845 HandleFailure (filename, TestResult.LoadError, e.ToString ());
846 return false;
847 }
848
849 if (doc_output != null) {
850 string ref_file = filename.Replace (".cs", "-ref.xml");
851 try {
852 #if !NET_2_1
853 XmlComparer.Compare (ref_file, doc_output);
854 #endif
855 } catch (Exception e) {
856 HandleFailure (filename, TestResult.XmlError, e.Message);
857 return false;
858 }
859 } else {
860 if (verif_file != null) {
861 PositiveTestCase pt = (PositiveTestCase) test;
862 pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
863
864 if (!tester.CheckILSize (pt, this, file))
865 return false;
866 }
867 }
868 } finally {
869 if (domain != null)
870 AppDomain.Unload (domain);
871 }
872
873 HandleFailure (filename, TestResult.Success, null);
874 return true;
875 }
876
877 protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
878 {
879 return new PositiveTestCase (filename, options, deps);
880 }
881
882 public void HandleFailure (string file, TestResult status, string extra)
883 {
884 switch (status) {
885 case TestResult.Success:
886 success++;
887 if (know_issues.Contains (file)) {
888 LogFileLine (file, "FIXED ISSUE");
889 return;
890 }
891 if (verbose)
892 LogFileLine (file, "OK");
893 return;
894
895 case TestResult.CompileError:
896 if (know_issues.Contains (file)) {
897 LogFileLine (file, "KNOWN ISSUE (Compilation error)");
898 know_issues.Remove (file);
899 return;
900 }
901 LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
902 break;
903
904 case TestResult.ExecError:
905 if (know_issues.Contains (file)) {
906 LogFileLine (file, "KNOWN ISSUE (Execution error)");
907 know_issues.Remove (file);
908 return;
909 }
910 LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
911 break;
912
913 case TestResult.XmlError:
914 if (know_issues.Contains (file)) {
915 LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
916 know_issues.Remove (file);
917 return;
918 }
919 LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
920 break;
921
922 case TestResult.LoadError:
923 LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)");
924 break;
925
926 case TestResult.ILError:
927 if (!update_verif_file) {
928 LogFileLine (file, "IL REGRESSION: " + extra);
929 }
930 extra = null;
931 break;
932 }
933
934 if (extra != null)
935 LogLine ("{0}", extra);
936
937 if (!regression.Contains (file))
938 regression.Add (file);
939 }
940
941 public override void Initialize ()
942 {
943 if (verif_file != null) {
944 #if NET_2_0
945 LoadVerificationData (verif_file);
946 #else
947 throw new NotSupportedException ();
948 #endif
949 }
950
951 base.Initialize ();
952 }
953
954 public override void CleanUp ()
955 {
956 base.CleanUp ();
957
958 if (update_verif_file) {
959 #if NET_2_0
960 UpdateVerificationData (verif_file);
961 #else
962 throw new NotSupportedException ();
963 #endif
964 }
965 }
966
967 #if NET_2_0
968 void LoadVerificationData (string file)
969 {
970 LogLine ("Loading verification data from `{0}' ...", file);
971
972 using (XmlReader r = XmlReader.Create (file)) {
973 r.ReadStartElement ("tests");
974 verif_data = new Hashtable ();
975
976 while (r.Read ()) {
977 if (r.Name != "test")
978 continue;
979
980 string name = r.GetAttribute ("name");
981 PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
982 verif_data.Add (name, tc);
983 }
984 }
985 }
986
987 void UpdateVerificationData (string file)
988 {
989 LogLine ("Updating verification data `{0}' ...", file);
990
991 XmlWriterSettings s = new XmlWriterSettings ();
992 s.Indent = true;
993 using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
994 w.WriteStartDocument ();
995 w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
996 w.WriteStartElement ("tests");
997 foreach (PositiveTestCase tc in tests) {
998 if (tc.VerificationProvider != null)
999 tc.VerificationProvider.WriteCodeInfoTo (w);
1000 }
1001 w.WriteEndElement ();
1002 }
1003 }
1004 #endif
1005 }
1006
1007 class NegativeChecker: Checker
1008 {
1009 string expected_message;
1010 string error_message;
1011 bool check_msg;
1012 bool check_error_line;
1013 bool is_warning;
1014 IDictionary wrong_warning;
1015
1016 protected enum CompilerError {
1017 Expected,
1018 Wrong,
1019 Missing,
1020 WrongMessage,
1021 MissingLocation,
1022 Duplicate
1023 }
1024
1025 public NegativeChecker (ITester tester, bool check_msg):
1026 base (tester)
1027 {
1028 this.check_msg = check_msg;
1029 wrong_warning = new Hashtable ();
1030 }
1031
1032 protected override bool AnalyzeTestFile (string file, ref int row, string line,
1033 ref string[] compiler_options,
1034 ref string[] dependencies)
1035 {
1036 if (row == 1) {
1037 expected_message = null;
1038
1039 int index = line.IndexOf (':');
1040 if (index == -1 || index > 15) {
1041 LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
1042 ++syntax_errors;
1043 base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
1044 ref dependencies);
1045 return false;
1046 }
1047
1048 expected_message = line.Substring (index + 1).Trim ();
1049 }
1050
1051 if (row == 2) {
1052 string filtered = line.Replace(" ", "");
1053
1054 // Some error tests require to have different error text for different runtimes.
1055 if (filtered.StartsWith ("//GMCS")) {
1056 row = 1;
1057 #if !NET_2_0
1058 return true;
1059 #else
1060 return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
1061 #endif
1062 }
1063
1064 check_error_line = !filtered.StartsWith ("//Line:0");
1065
1066 if (!filtered.StartsWith ("//Line:")) {
1067 LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
1068 ++syntax_errors;
1069 return false;
1070 }
1071 }
1072
1073 if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
1074 return false;
1075
1076 is_warning = false;
1077 if (compiler_options != null) {
1078 foreach (string s in compiler_options) {
1079 if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
1080 is_warning = true;
1081 }
1082 }
1083 return true;
1084 }
1085
1086
1087 protected override bool Check (TestCase test)
1088 {
1089 string filename = test.FileName;
1090
1091 int start_char = 0;
1092 while (Char.IsLetter (filename, start_char))
1093 ++start_char;
1094
1095 int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
1096 string expected = filename.Substring (start_char, end_char - start_char);
1097
1098 try {
1099 if (base.Check (test)) {
1100 HandleFailure (filename, CompilerError.Missing);
1101 return false;
1102 }
1103 }
1104 catch (Exception e) {
1105 HandleFailure (filename, CompilerError.Missing);
1106 if (e.InnerException != null)
1107 e = e.InnerException;
1108
1109 Log (e.ToString ());
1110 return false;
1111 }
1112
1113 int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
1114 if (tester.IsWarning (err_id)) {
1115 if (!is_warning)
1116 wrong_warning [err_id] = true;
1117 } else {
1118 if (is_warning)
1119 wrong_warning [err_id] = false;
1120 }
1121
1122 CompilerError result_code = GetCompilerError (expected, tester.Output);
1123 if (HandleFailure (filename, result_code)) {
1124 success++;
1125 return true;
1126 }
1127
1128 if (result_code == CompilerError.Wrong)
1129 LogLine (tester.Output);
1130
1131 return false;
1132 }
1133
1134 CompilerError GetCompilerError (string expected, string buffer)
1135 {
1136 const string error_prefix = "CS";
1137 const string ignored_error = "error CS5001";
1138 string tested_text = "error " + error_prefix + expected;
1139 StringReader sr = new StringReader (buffer);
1140 string line = sr.ReadLine ();
1141 ArrayList ld = new ArrayList ();
1142 CompilerError result = CompilerError.Missing;
1143 while (line != null) {
1144 if (ld.Contains (line)) {
1145 if (line.IndexOf ("Location of the symbol related to previous") == -1)
1146 return CompilerError.Duplicate;
1147 }
1148 ld.Add (line);
1149
1150 if (result != CompilerError.Expected) {
1151 if (line.IndexOf (tested_text) != -1) {
1152 if (check_msg) {
1153 int first = line.IndexOf (':');
1154 int second = line.IndexOf (':', first + 1);
1155 if (second == -1 || !check_error_line)
1156 second = first;
1157
1158 string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
1159 if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
1160 error_message = msg;
1161 return CompilerError.WrongMessage;
1162 }
1163
1164 if (check_error_line && line.IndexOf (".cs(") == -1)
1165 return CompilerError.MissingLocation;
1166 }
1167 result = CompilerError.Expected;
1168 } else if (line.IndexOf (error_prefix) != -1 &&
1169 line.IndexOf (ignored_error) == -1)
1170 result = CompilerError.Wrong;
1171 }
1172
1173 line = sr.ReadLine ();
1174 }
1175
1176 return result;
1177 }
1178
1179 bool HandleFailure (string file, CompilerError status)
1180 {
1181 switch (status) {
1182 case CompilerError.Expected:
1183 if (know_issues.Contains (file) || no_error_list.Contains (file)) {
1184 LogFileLine (file, "FIXED ISSUE");
1185 return true;
1186 }
1187
1188 if (verbose)
1189 LogFileLine (file, "OK");
1190 return true;
1191
1192 case CompilerError.Wrong:
1193 if (know_issues.Contains (file)) {
1194 LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
1195 know_issues.Remove (file);
1196 return false;
1197 }
1198 if (no_error_list.Contains (file)) {
1199 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
1200 no_error_list.Remove (file);
1201 }
1202 else {
1203 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
1204 }
1205 break;
1206
1207 case CompilerError.WrongMessage:
1208 if (know_issues.Contains (file)) {
1209 LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
1210 know_issues.Remove (file);
1211 return false;
1212 }
1213 if (no_error_list.Contains (file)) {
1214 LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
1215 no_error_list.Remove (file);
1216 }
1217 else {
1218 LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
1219 LogLine ("Exp: {0}", expected_message);
1220 LogLine ("Was: {0}", error_message);
1221 }
1222 break;
1223
1224 case CompilerError.Missing:
1225 if (no_error_list.Contains (file)) {
1226 LogFileLine (file, "KNOWN ISSUE (No error reported)");
1227 no_error_list.Remove (file);
1228 return false;
1229 }
1230
1231 if (know_issues.Contains (file)) {
1232 LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
1233 know_issues.Remove (file);
1234 }
1235 else {
1236 LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
1237 }
1238
1239 break;
1240
1241 case CompilerError.MissingLocation:
1242 if (know_issues.Contains (file)) {
1243 LogFileLine (file, "KNOWN ISSUE (Missing error location)");
1244 know_issues.Remove (file);
1245 return false;
1246 }
1247 if (no_error_list.Contains (file)) {
1248 LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
1249 no_error_list.Remove (file);
1250 }
1251 else {
1252 LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
1253 }
1254 break;
1255
1256 case CompilerError.Duplicate:
1257 // Will become an error soon
1258 LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
1259 return true;
1260 }
1261
1262 regression.Add (file);
1263 return false;
1264 }
1265
1266 protected override void PrintSummary()
1267 {
1268 base.PrintSummary ();
1269
1270 if (wrong_warning.Count > 0) {
1271 LogLine ("");
1272 LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
1273 LogLine ("");
1274 foreach (DictionaryEntry de in wrong_warning)
1275 LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
1276 }
1277 }
1278
1279 }
1280
1281 class Tester {
1282
1283 static int Main(string[] args)
1284 {
1285 string temp;
1286
1287 if (GetOption ("help", args, false, out temp)) {
1288 Usage ();
1289 return 0;
1290 }
1291
1292 string compiler;
1293 if (!GetOption ("compiler", args, true, out compiler)) {
1294 Usage ();
1295 return 1;
1296 }
1297
1298 ITester tester;
1299 try {
1300 Console.WriteLine ("Loading " + compiler + " ...");
1301 tester = new ReflectionTester (Assembly.LoadFile (compiler));
1302 }
1303 catch (Exception) {
1304 #if NET_2_1
1305 throw;
1306 #else
1307 Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
1308 if (!File.Exists (compiler)) {
1309 Console.Error.WriteLine ("ERROR: Tested compiler was not found");
1310 return 1;
1311 }
1312 tester = new ProcessTester (compiler);
1313 #endif
1314 }
1315
1316 string mode;
1317 if (!GetOption ("mode", args, true, out mode)) {
1318 Usage ();
1319 return 1;
1320 }
1321
1322 Checker checker;
1323 bool positive;
1324 switch (mode) {
1325 case "neg":
1326 checker = new NegativeChecker (tester, true);
1327 positive = false;
1328 break;
1329 case "pos":
1330 string iltest;
1331 GetOption ("il", args, false, out iltest);
1332 checker = new PositiveChecker (tester, iltest);
1333 positive = true;
1334
1335 if (iltest != null && GetOption ("update-il", args, false, out temp)) {
1336 ((PositiveChecker) checker).UpdateVerificationDataFile = true;
1337 }
1338
1339 break;
1340 default:
1341 Console.Error.WriteLine ("Invalid -mode argument");
1342 return 1;
1343 }
1344
1345
1346 if (GetOption ("issues", args, true, out temp))
1347 checker.IssueFile = temp;
1348 if (GetOption ("log", args, true, out temp))
1349 checker.LogFile = temp;
1350 if (GetOption ("verbose", args, false, out temp))
1351 checker.Verbose = true;
1352 if (GetOption ("safe-execution", args, false, out temp))
1353 checker.SafeExecution = true;
1354 if (GetOption ("compiler-options", args, true, out temp)) {
1355 string[] extra = temp.Split (' ');
1356 checker.ExtraCompilerOptions = extra;
1357 }
1358
1359 string test_pattern;
1360 if (!GetOption ("files", args, true, out test_pattern)) {
1361 Usage ();
1362 return 1;
1363 }
1364
1365 ArrayList files = new ArrayList ();
1366 switch (test_pattern) {
1367 case "v1":
1368 files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
1369 break;
1370 case "v2":
1371 files.AddRange (Directory.GetFiles (".", positive ? "gtest*.cs" : "gcs*.cs"));
1372 goto case "v1";
1373 case "v4":
1374 files.AddRange (Directory.GetFiles (".", positive ? "dtest*.cs" : "dcs*.cs"));
1375 goto case "v2";
1376 default:
1377 files.AddRange (Directory.GetFiles (".", test_pattern));
1378 break;
1379 }
1380
1381 if (files.Count == 0) {
1382 Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
1383 return 2;
1384 }
1385
1386 checker.Initialize ();
1387
1388 foreach (string s in files) {
1389 string filename = Path.GetFileName (s);
1390 if (Char.IsUpper (filename, 0)) { // Windows hack
1391 continue;
1392 }
1393
1394 if (filename.EndsWith ("-p2.cs"))
1395 continue;
1396
1397 checker.Do (filename);
1398 }
1399
1400 checker.CleanUp ();
1401
1402 checker.Dispose ();
1403
1404 return checker.ResultCode;
1405 }
1406
1407 static bool GetOption (string opt, string[] args, bool req_arg, out string value)
1408 {
1409 opt = "-" + opt;
1410 foreach (string a in args) {
1411 if (a.StartsWith (opt)) {
1412 int sep = a.IndexOf (':');
1413 if (sep > 0) {
1414 value = a.Substring (sep + 1);
1415 } else {
1416 value = null;
1417 if (req_arg) {
1418 Console.Error.WriteLine ("Missing argument in option " + opt);
1419 return false;
1420 }
1421 }
1422
1423 return true;
1424 }
1425 }
1426
1427 value = null;
1428 return false;
1429 }
1430
1431 static void Usage ()
1432 {
1433 Console.WriteLine (
1434 "Mono compiler tester, (C) 2009 Novell, Inc.\n" +
1435 "compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
1436 " \n" +
1437 " -compiler:FILE The file which will be used to compiler tests\n" +
1438 " -compiler-options:OPTIONS Add global compiler options\n" +
1439 " -il:IL-FILE XML file with expected IL details for each test\n" +
1440 " -issues:FILE The list of expected failures\n" +
1441 " -log:FILE Writes any output also to the file\n" +
1442 " -help Lists all options\n" +
1443 " -mode:[pos|neg] Specifies compiler test mode\n" +
1444 " -safe-execution Runs compiled executables in separate app-domain\n" +
1445 " -update-il Updates IL-FILE to match compiler output\n" +
1446 " -verbose Prints more details during testing\n"
1447 );
1448 }
1449 }
1450 }
git-svn mirror of mcs