Source/cmCTest.h

   1 /*=========================================================================
   2
   3   Program:   CMake - Cross-Platform Makefile Generator
   4   Module:    $RCSfile: cmCTest.h,v $
   5   Language:  C++
   6   Date:      $Date: 2009-03-11 17:31:33 $
   7   Version:   $Revision: 1.116 $
   8
   9   Copyright (c) 2002 Kitware, Inc., Insight Consortium.  All rights reserved.
  10   See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
  11
  12      This software is distributed WITHOUT ANY WARRANTY; without even
  13      the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  14      PURPOSE.  See the above copyright notices for more information.
  15
  16 =========================================================================*/
  17
  18 #ifndef cmCTest_h
  19 #define cmCTest_h
  20
  21 #include "cmStandardIncludes.h"
  22 #include "cmListFileCache.h"
  23 #include <time.h>
  24
  25 class cmake;
  26 class cmMakefile;
  27 class cmCTestGenericHandler;
  28 class cmGeneratedFileStream;
  29 class cmCTestCommand;
  30 class cmCTestScriptHandler;
  31
  32 #define cmCTestLog(ctSelf, logType, msg) \
  33   do { \
  34   cmOStringStream cmCTestLog_msg; \
  35   cmCTestLog_msg << msg; \
  36   (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__,\
  37                 cmCTestLog_msg.str().c_str());\
  38   } while ( 0 )
  39
  40 #ifdef cerr
  41 #  undef cerr
  42 #endif
  43 #define cerr no_cerr_use_cmCTestLog
  44
  45 #ifdef cout
  46 #  undef cout
  47 #endif
  48 #define cout no_cout_use_cmCTestLog
  49
  50 class cmCTest
  51 {
  52 public:
  53   /** Enumerate parts of the testing and submission process.  */
  54   enum Part
  55   {
  56     PartStart,
  57     PartUpdate,
  58     PartConfigure,
  59     PartBuild,
  60     PartTest,
  61     PartCoverage,
  62     PartMemCheck,
  63     PartSubmit,
  64     PartNotes,
  65     PartExtraFiles,
  66     PartCount // Update names in constructor when adding a part
  67   };
  68
  69   /** Representation of one part.  */
  70   struct PartInfo
  71   {
  72     PartInfo(): Enabled(false) {}
  73
  74     void SetName(const char* name) { this->Name = name; }
  75     const char* GetName() const { return this->Name.c_str(); }
  76
  77     void Enable() { this->Enabled = true; }
  78     operator bool() const { return this->Enabled; }
  79
  80     std::vector<std::string> SubmitFiles;
  81   private:
  82     bool Enabled;
  83     std::string Name;
  84   };
  85
  86   /** Get a testing part id from its string name.  Returns PartCount
  87       if the string does not name a valid part.  */
  88   Part GetPartFromName(const char* name);
  89
  90   typedef std::vector<cmStdString> VectorOfStrings;
  91   typedef std::set<cmStdString> SetOfStrings;
  92
  93   ///! Process Command line arguments
  94   int Run(std::vector<std::string> &, std::string* output = 0);
  95
  96   /**
  97    * Initialize and finalize testing
  98    */
  99   int Initialize(const char* binary_dir, bool new_tag = false,
 100     bool verbose_tag = true);
 101   bool InitializeFromCommand(cmCTestCommand* command, bool first = false);
 102   void Finalize();
 103
 104   /**
 105    * Process the tests. This is the main routine. The execution of the
 106    * tests should look like this:
 107    *
 108    * ctest foo;
 109    * foo.Initialize();
 110    * // Set some things on foo
 111    * foo.ProcessTests();
 112    * foo.Finalize();
 113    */
 114   int ProcessTests();
 115
 116   /*
 117    * A utility function that returns the nightly time
 118    */
 119   struct tm* GetNightlyTime(std::string str,
 120     bool tomorrowtag);
 121
 122   /*
 123    * Is the tomorrow tag set?
 124    */
 125   bool GetTomorrowTag() { return this->TomorrowTag; };
 126
 127   /**
 128    * Try to run tests of the project
 129    */
 130   int TestDirectory(bool memcheck);
 131
 132   ///! what is the configuraiton type, e.g. Debug, Release etc.
 133   std::string const& GetConfigType();
 134   double GetTimeOut() { return this->TimeOut; }
 135   void SetTimeOut(double t) { this->TimeOut = t; }
 136   // how many test to run at the same time
 137   int GetParallelLevel() { return this->ParallelLevel; }
 138   void SetParallelLevel(int t) { this->ParallelLevel = t; }
 139
 140   bool GetParallelSubprocess() { return this->ParallelSubprocess; }
 141   void SetParallelSubprocess() { this->ParallelSubprocess = true; }
 142
 143   void SetParallelSubprocessId(int id) { this->ParallelSubprocessId = id;}
 144   int GetParallelSubprocessId() { return this->ParallelSubprocessId;}
 145   const char* GetParallelCacheFile()
 146     { return this->ParallelCacheFile.c_str();}
 147   void SetParallelCacheFile(const char* c) { this->ParallelCacheFile = c; }
 148
 149   /**
 150    * Check if CTest file exists
 151    */
 152   bool CTestFileExists(const std::string& filename);
 153   bool AddIfExists(Part part, const char* file);
 154
 155   /**
 156    * Set the cmake test
 157    */
 158   bool SetTest(const char*, bool report = true);
 159
 160   /**
 161    * Set the cmake test mode (experimental, nightly, continuous).
 162    */
 163   void SetTestModel(int mode);
 164   int GetTestModel() { return this->TestModel; };
 165
 166   std::string GetTestModelString();
 167   static int GetTestModelFromString(const char* str);
 168   static std::string CleanString(const std::string& str);
 169   std::string GetCTestConfiguration(const char *name);
 170   void SetCTestConfiguration(const char *name, const char* value);
 171   void EmptyCTestConfiguration();
 172
 173   /**
 174    * constructor and destructor
 175    */
 176   cmCTest();
 177   ~cmCTest();
 178
 179   //! Set the notes files to be created.
 180   void SetNotesFiles(const char* notes);
 181
 182   void PopulateCustomVector(cmMakefile* mf, const char* definition,
 183     VectorOfStrings& vec);
 184   void PopulateCustomInteger(cmMakefile* mf, const char* def,
 185     int& val);
 186
 187   ///! Get the current time as string
 188   std::string CurrentTime();
 189
 190   /**
 191    * Return the time remaianing that the script is allowed to run in
 192    * seconds if the user has set the variable CTEST_TIME_LIMIT. If that has
 193    * not been set it returns 1e7 seconds
 194    */
 195   double GetRemainingTimeAllowed();
 196
 197   ///! Open file in the output directory and set the stream
 198   bool OpenOutputFile(const std::string& path,
 199                       const std::string& name,
 200                       cmGeneratedFileStream& stream,
 201                       bool compress = false);
 202
 203   ///! Should we only show what we would do?
 204   bool GetShowOnly();
 205
 206
 207   ///! The max output width
 208   int GetMaxTestNameWidth() const;
 209   void SetMaxTestNameWidth(int w) { this->MaxTestNameWidth = w;}
 210
 211   /**
 212    * Run a single executable command and put the stdout and stderr
 213    * in output.
 214    *
 215    * If verbose is false, no user-viewable output from the program
 216    * being run will be generated.
 217    *
 218    * If timeout is specified, the command will be terminated after
 219    * timeout expires. Timeout is specified in seconds.
 220    *
 221    * Argument retVal should be a pointer to the location where the
 222    * exit code will be stored. If the retVal is not specified and
 223    * the program exits with a code other than 0, then the this
 224    * function will return false.
 225    *
 226    * If the command has spaces in the path the caller MUST call
 227    * cmSystemTools::ConvertToRunCommandPath on the command before passing
 228    * it into this function or it will not work.  The command must be correctly
 229    * escaped for this to with spaces.
 230    */
 231   bool RunCommand(const char* command,
 232     std::string* stdOut, std::string* stdErr,
 233     int* retVal = 0, const char* dir = 0, double timeout = 0.0);
 234
 235   //! Start CTest XML output file
 236   void StartXML(std::ostream& ostr, bool append);
 237
 238   //! End CTest XML output file
 239   void EndXML(std::ostream& ostr);
 240
 241   //! Run command specialized for make and configure. Returns process status
 242   // and retVal is return value or exception.
 243   int RunMakeCommand(const char* command, std::string* output,
 244     int* retVal, const char* dir, int timeout,
 245     std::ofstream& ofs);
 246
 247   /*
 248    * return the current tag
 249    */
 250   std::string GetCurrentTag();
 251
 252   //! Get the path to the build tree
 253   std::string GetBinaryDir();
 254
 255   //! Get the short path to the file. This means if the file is in binary or
 256   //source directory, it will become /.../relative/path/to/file
 257   std::string GetShortPathToFile(const char* fname);
 258
 259   //! Get the path to CTest
 260   const char* GetCTestExecutable() { return this->CTestSelf.c_str(); }
 261   const char* GetCMakeExecutable() { return this->CMakeSelf.c_str(); }
 262
 263   enum {
 264     EXPERIMENTAL,
 265     NIGHTLY,
 266     CONTINUOUS
 267   };
 268
 269   // provide some more detailed info on the return code for ctest
 270   enum {
 271     UPDATE_ERRORS    = 0x01,
 272     CONFIGURE_ERRORS = 0x02,
 273     BUILD_ERRORS     = 0x04,
 274     TEST_ERRORS      = 0x08,
 275     MEMORY_ERRORS    = 0x10,
 276     COVERAGE_ERRORS  = 0x20,
 277     SUBMIT_ERRORS    = 0x40
 278   };
 279
 280   ///! Are we producing XML
 281   bool GetProduceXML();
 282   void SetProduceXML(bool v);
 283
 284   //! Run command specialized for tests. Returns process status and retVal is
 285   // return value or exception. If environment is non-null, it is used to set
 286   // environment variables prior to running the test. After running the test,
 287   // environment variables are restored to their previous values.
 288   int RunTest(std::vector<const char*> args, std::string* output, int *retVal,
 289     std::ostream* logfile, double testTimeOut,
 290     std::vector<std::string>* environment);
 291
 292   /**
 293    * Execute handler and return its result. If the handler fails, it returns
 294    * negative value.
 295    */
 296   int ExecuteHandler(const char* handler);
 297
 298   /*
 299    * Get the handler object
 300    */
 301   cmCTestGenericHandler* GetHandler(const char* handler);
 302   cmCTestGenericHandler* GetInitializedHandler(const char* handler);
 303
 304   /*
 305    * Set the CTest variable from CMake variable
 306    */
 307   bool SetCTestConfigurationFromCMakeVariable(cmMakefile* mf,
 308     const char* dconfig, const char* cmake_var);
 309
 310   //! Make string safe to be send as an URL
 311   static std::string MakeURLSafe(const std::string&);
 312
 313   /** Decode a URL to the original string.  */
 314   static std::string DecodeURL(const std::string&);
 315
 316   //! Should ctect configuration be updated. When using new style ctest
 317   // script, this should be true.
 318   void SetSuppressUpdatingCTestConfiguration(bool val)
 319     {
 320     this->SuppressUpdatingCTestConfiguration = val;
 321     }
 322
 323   //! Add overwrite to ctest configuration.
 324   // The format is key=value
 325   void AddCTestConfigurationOverwrite(const char* encstr);
 326
 327   //! Create XML file that contains all the notes specified
 328   int GenerateNotesFile(const std::vector<cmStdString> &files);
 329
 330   //! Submit extra files to the server
 331   bool SubmitExtraFiles(const char* files);
 332   bool SubmitExtraFiles(const std::vector<cmStdString> &files);
 333
 334   //! Set the output log file name
 335   void SetOutputLogFileName(const char* name);
 336
 337   //! Set the visual studio or Xcode config type
 338   void SetConfigType(const char* ct);
 339
 340   //! Various log types
 341   enum {
 342     DEBUG = 0,
 343     OUTPUT,
 344     HANDLER_OUTPUT,
 345     HANDLER_VERBOSE_OUTPUT,
 346     WARNING,
 347     ERROR_MESSAGE,
 348     OTHER
 349   };
 350
 351   //! Add log to the output
 352   void Log(int logType, const char* file, int line, const char* msg);
 353
 354   //! Get the version of dart server
 355   int GetDartVersion() { return this->DartVersion; }
 356
 357   //! Add file to be submitted
 358   void AddSubmitFile(Part part, const char* name);
 359   std::vector<std::string> const& GetSubmitFiles(Part part)
 360     { return this->Parts[part].SubmitFiles; }
 361   void ClearSubmitFiles(Part part) { this->Parts[part].SubmitFiles.clear(); }
 362
 363   //! Read the custom configuration files and apply them to the current ctest
 364   int ReadCustomConfigurationFileTree(const char* dir, cmMakefile* mf);
 365
 366   std::vector<cmStdString> &GetInitialCommandLineArguments()
 367   { return this->InitialCommandLineArguments; };
 368
 369   //! Set the track to submit to
 370   void SetSpecificTrack(const char* track);
 371   const char* GetSpecificTrack();
 372
 373   bool GetVerbose() { return this->Verbose;}
 374   bool GetExtraVerbose() { return this->ExtraVerbose;}
 375
 376   /** Direct process output to given streams.  */
 377   void SetStreams(std::ostream* out, std::ostream* err)
 378     { this->StreamOut = out; this->StreamErr = err; }
 379   void AddSiteProperties(std::ostream& );
 380 private:
 381   std::string ConfigType;
 382   bool Verbose;
 383   bool ExtraVerbose;
 384   bool ProduceXML;
 385
 386   bool ForceNewCTestProcess;
 387
 388   bool RunConfigurationScript;
 389
 390   int GenerateNotesFile(const char* files);
 391
 392   // these are helper classes
 393   typedef std::map<cmStdString,cmCTestGenericHandler*> t_TestingHandlers;
 394   t_TestingHandlers TestingHandlers;
 395
 396   bool ShowOnly;
 397
 398   //! Map of configuration properties
 399   typedef std::map<cmStdString, cmStdString> CTestConfigurationMap;
 400
 401   std::string             CTestConfigFile;
 402   // TODO: The ctest configuration should be a hierarchy of
 403   // configuration option sources: command-line, script, ini file.
 404   // Then the ini file can get re-loaded whenever it changes without
 405   // affecting any higher-precedence settings.
 406   CTestConfigurationMap CTestConfiguration;
 407   CTestConfigurationMap CTestConfigurationOverwrites;
 408   PartInfo                Parts[PartCount];
 409   typedef std::map<cmStdString, Part> PartMapType;
 410   PartMapType             PartMap;
 411
 412   std::string             CurrentTag;
 413   bool                    TomorrowTag;
 414
 415   int                     TestModel;
 416   std::string             SpecificTrack;
 417
 418   double                  TimeOut;
 419
 420   int                     MaxTestNameWidth;
 421
 422   std::string             ParallelCacheFile;
 423   int                     ParallelLevel;
 424   int                     ParallelSubprocessId;
 425   bool                    ParallelSubprocess;
 426   int                     CompatibilityMode;
 427
 428   // information for the --build-and-test options
 429   std::string              CMakeSelf;
 430   std::string              CTestSelf;
 431   std::string              BinaryDir;
 432
 433   std::string              NotesFiles;
 434
 435
 436   bool                     InteractiveDebugMode;
 437
 438   bool                     ShortDateFormat;
 439
 440   bool                     CompressXMLFiles;
 441
 442   void InitStreams();
 443   std::ostream* StreamOut;
 444   std::ostream* StreamErr;
 445
 446   void BlockTestErrorDiagnostics();
 447
 448
 449   //! parse the option after -D and convert it into the appropriate steps
 450   bool AddTestsForDashboardType(std::string &targ);
 451
 452   //! parse and process most common command line arguments
 453   void HandleCommandLineArguments(size_t &i,
 454                                   std::vector<std::string> &args);
 455
 456   //! hande the -S -SP and -SR arguments
 457   void HandleScriptArguments(size_t &i,
 458                              std::vector<std::string> &args,
 459                              bool &SRArgumentSpecified);
 460
 461   //! Reread the configuration file
 462   bool UpdateCTestConfiguration();
 463
 464   //! Create not from files.
 465   int GenerateCTestNotesOutput(std::ostream& os,
 466     const VectorOfStrings& files);
 467
 468   ///! Find the running cmake
 469   void FindRunningCMake();
 470
 471   //! Check if the argument is the one specified
 472   bool CheckArgument(const std::string& arg, const char* varg1,
 473     const char* varg2 = 0);
 474
 475   //! Output errors from a test
 476   void OutputTestErrors(std::vector<char> const &process_output);
 477
 478   bool                      SuppressUpdatingCTestConfiguration;
 479
 480   bool Debug;
 481   bool ShowLineNumbers;
 482   bool Quiet;
 483
 484   int  DartVersion;
 485
 486   std::vector<cmStdString> InitialCommandLineArguments;
 487
 488   int SubmitIndex;
 489
 490   cmGeneratedFileStream* OutputLogFile;
 491   int OutputLogFileLastTag;
 492
 493   bool OutputTestOutputOnTestFailure;
 494 };
 495
 496 class cmCTestLogWrite
 497 {
 498 public:
 499   cmCTestLogWrite(const char* data, size_t length)
 500     : Data(data), Length(length) {}
 501
 502   const char* Data;
 503   size_t Length;
 504 };
 505
 506 inline std::ostream& operator<< (std::ostream& os, const cmCTestLogWrite& c)
 507 {
 508   if (!c.Length)
 509     {
 510     return os;
 511     }
 512   os.write(c.Data, c.Length);
 513   os.flush();
 514   return os;
 515 }
 516
 517 #endif