mono/mini/TestDriver.cs

   1 using System;
   2 using System.Reflection;
   3 using System.Collections.Generic;
   4
   5 [AttributeUsageAttribute(AttributeTargets.All, Inherited = true, AllowMultiple = true)]
   6 public class CategoryAttribute : Attribute
   7 {
   8         public CategoryAttribute (string category) {
   9                 Category = category;
  10         }
  11
  12         public string Category {
  13                 get; set;
  14         }
  15 }
  16 public class TestDriverReporter
  17 {
  18         public int FailedTests { get; private set; }
  19         public int SkippedTests { get; private set; }
  20         public int ExecutedTests { get; private set; }
  21
  22         public void ReportResults (int executed, int skipped, int failed) {
  23                 ExecutedTests = executed;
  24                 SkippedTests = skipped;
  25                 FailedTests = failed;
  26         }
  27 };
  28
  29 public class TestDriver {
  30
  31         static public int RunTests(Type type, string[] args, TestDriverReporter reporter) {
  32                 int failed = 0, ran = 0;
  33                 int result, expected;
  34                 int i, j, iterations;
  35                 string name;
  36                 MethodInfo[] methods;
  37                 bool do_timings = false;
  38                 bool verbose = false;
  39                 bool quiet = false;
  40                 int tms = 0;
  41                 DateTime start, end = DateTime.Now;
  42
  43                 iterations = 1;
  44
  45                 var exclude = new Dictionary<string, string> ();
  46                 List<string> run_only = new List<string> ();
  47                 List<string> exclude_test = new List<string> ();
  48                 if (args != null && args.Length > 0) {
  49                         for (j = 0; j < args.Length;) {
  50                                 if (args [j] == "--time") {
  51                                         do_timings = !quiet;
  52                                         j ++;
  53                                 } else if (args [j] == "--iter") {
  54                                         iterations = Int32.Parse (args [j + 1]);
  55                                         j += 2;
  56                                 } else if ((args [j] == "-v") || (args [j] == "--verbose")) {
  57                                         verbose = !quiet;
  58                                         j += 1;
  59                                 } else if ((args [j] == "-q") || (args [j] == "--quiet")) {
  60                                         quiet = true;
  61                                         verbose = false;
  62                                         do_timings = false;
  63                                         j += 1;
  64                                 } else if (args [j] == "--exclude") {
  65                                         exclude [args [j + 1]] = args [j + 1];
  66                                         j += 2;
  67                                 } else if (args [j] == "--exclude-test") {
  68                                         exclude_test.Add (args [j + 1]);
  69                                         j += 2;
  70                                 } else if (args [j] == "--run-only") {
  71                                         run_only.Add (args [j + 1]);
  72                                         j += 2;
  73                                 } else {
  74                                         Console.WriteLine ("Unknown argument: " + args [j]);
  75                                         return 1;
  76                                 }
  77                         }
  78                 }
  79                 int nskipped = 0;
  80                 methods = type.GetMethods (BindingFlags.Public|BindingFlags.NonPublic|BindingFlags.Static);
  81                 for (int iter = 0; iter < iterations; ++iter) {
  82                         for (i = 0; i < methods.Length; ++i) {
  83                                 name = methods [i].Name;
  84                                 if (!name.StartsWith ("test_", StringComparison.Ordinal))
  85                                         continue;
  86                                 if (run_only.Count > 0) {
  87                                         bool found = false;
  88                                         for (j = 0; j < run_only.Count; j++) {
  89                                                 if (name.EndsWith (run_only [j])) {
  90                                                         found = true;
  91                                                         break;
  92                                                 }
  93                                         }
  94                                         if (!found)
  95                                                 continue;
  96                                 }
  97                                 if (exclude.Count > 0 || exclude_test.Count > 0) {
  98                                         var attrs = methods [i].GetCustomAttributes (typeof (CategoryAttribute), false);
  99                                         bool skip = false;
 100                                         for (j = 0; j < exclude_test.Count; j++) {
 101                                                 if (name.EndsWith (exclude_test [j])) {
 102                                                         skip = true;
 103                                                         break;
 104                                                 }
 105                                         }
 106                                         foreach (CategoryAttribute attr in attrs) {
 107                                                 if (exclude.ContainsKey (attr.Category))
 108                                                         skip = true;
 109                                         }
 110                                         if (skip) {
 111                                                 if (verbose)
 112                                                         Console.WriteLine ("Skipping '{0}'.", name);
 113                                                 nskipped ++;
 114                                                 continue;
 115                                         }
 116                                 }
 117                                 for (j = 5; j < name.Length; ++j)
 118                                         if (!Char.IsDigit (name [j]))
 119                                                 break;
 120                                 if (verbose)
 121                                         Console.WriteLine ("Running '{0}' ...", name);
 122                                 expected = Int32.Parse (name.Substring (5, j - 5));
 123                                 start = DateTime.Now;
 124                                 result = (int)methods [i].Invoke (null, null);
 125                                 if (do_timings) {
 126                                         end = DateTime.Now;
 127                                         long tdiff = end.Ticks - start.Ticks;
 128                                         int mdiff = (int)tdiff/10000;
 129                                         tms += mdiff;
 130                                         Console.WriteLine ("{0} took {1} ms", name, mdiff);
 131                                 }
 132                                 ran++;
 133                                 if (result != expected) {
 134                                         failed++;
 135                                         Console.WriteLine ("{0} failed: got {1}, expected {2}", name, result, expected);
 136                                 }
 137                         }
 138
 139                         if (!quiet) {
 140                                 if (do_timings) {
 141                                         Console.WriteLine ("Total ms: {0}", tms);
 142                                 }
 143                                 if (nskipped > 0)
 144                                         Console.WriteLine ("Regression tests: {0} ran, {1} skipped, {2} failed in {3}", ran, nskipped, failed, type);
 145                                 else
 146                                         Console.WriteLine ("Regression tests: {0} ran, {1} failed in {2}", ran, failed, type);
 147                         }
 148                 }
 149
 150                 if (reporter != null) {
 151                         reporter.ReportResults (ran, nskipped, failed);
 152                 }
 153
 154                 //Console.WriteLine ("Regression tests: {0} ran, {1} failed in [{2}]{3}", ran, failed, type.Assembly.GetName().Name, type);
 155                 return failed;
 156         }
 157
 158         static public int RunTests (Type type, string[] args) {
 159                 return RunTests (type, args, null);
 160         }
 161
 162         static public int RunTests (Type type) {
 163                 return RunTests (type, null, null);
 164         }
 165 }
 166
 167 /// Provide tests with the ability to find out how much time they have to run before being timed out.
 168 public class TestTimeout {
 169         const string ENV_TIMEOUT = "TEST_DRIVER_TIMEOUT_SEC";
 170         private readonly TimeSpan availableTime;
 171         private TimeSpan slack;
 172         private DateTime startTime;
 173
 174         /// <summary>
 175         ///   How much time the test runner provided for us or TimeSpan.Zero if there is no bound.
 176         /// </summary>
 177         public TimeSpan AvailableTime { get { return availableTime; } }
 178
 179         public DateTime StartTime { get { return startTime; } }
 180
 181         /// <summary> Extra time to add when deciding if there
 182         ///   is still time to run.  Bigger slack means less
 183         ///   time left.
 184         /// </summary>
 185         public TimeSpan Slack {
 186                 get { return slack; }
 187                 set { slack = value; }
 188         }
 189
 190         public TestTimeout () {
 191                 availableTime = initializeAvailableTime ();
 192                 slack = defaultSlack ();
 193         }
 194
 195         /// <summary>
 196         ///    Consider the test started.
 197         /// </summary>
 198         public void Start ()
 199         {
 200                 startTime = DateTime.UtcNow;
 201         }
 202
 203         public bool HaveTimeLeft ()
 204         {
 205                 if (availableTime == TimeSpan.Zero)
 206                         return true;
 207                 var t = DateTime.UtcNow;
 208                 var finishTime = startTime + availableTime - slack;
 209                 return (t < finishTime);
 210         }
 211
 212         private TimeSpan defaultSlack ()
 213         {
 214                 return TimeSpan.FromSeconds (5);
 215         }
 216
 217         private TimeSpan initializeAvailableTime ()
 218         {
 219                 var e = System.Environment.GetEnvironmentVariable(ENV_TIMEOUT);
 220                 double d;
 221                 if (Double.TryParse(e, out d)) {
 222                         return TimeSpan.FromSeconds(d);
 223                 } else {
 224                         return TimeSpan.Zero;
 225                 }
 226         }
 227
 228 }