2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2011-2018, The GROMACS development team.
5 * Copyright (c) 2019,2020, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
36 /*! \libinternal \file
38 * Functionality for writing tests that can produce their own reference data.
40 * See \ref page_refdata for more details.
42 * \author Teemu Murtola <teemu.murtola@gmail.com>
44 * \ingroup module_testutils
46 #ifndef GMX_TESTUTILS_REFDATA_H
47 #define GMX_TESTUTILS_REFDATA_H
53 #include "gromacs/utility/basedefinitions.h"
54 #include "gromacs/utility/classhelpers.h"
59 class IOptionsContainer
;
60 class KeyValueTreeObject
;
61 class KeyValueTreeValue
;
67 class FloatingPointTolerance
;
69 /*! \libinternal \brief
70 * Mode of operation for reference data handling.
72 * There should be no need to use this type outside the test utility module.
74 * \ingroup module_testutils
76 enum class ReferenceDataMode
: int
79 * Compare to existing reference data.
81 * If reference data does not exist, or if the test results differ from
82 * those in the reference data, the test fails.
86 * Create missing reference data.
88 * If reference data does not exist for a test, that test behaves as if
89 * ReferenceDataMode::UpdateAll had been specified. Tests for which reference data
90 * exists, behave like with ReferenceDataMode::Compare.
94 * Update reference data that does not pass comparison.
96 * Tests utilizing reference data should always pass in this mode unless
97 * there is an I/O error.
101 * Update reference data, overwriting old data.
103 * Tests utilizing reference data should always pass in this mode unless
104 * there is an I/O error.
107 //! Marks the end of the enum
111 /*! \libinternal \brief
112 * Initializes reference data handling.
114 * Adds command-line options to \p options to set the reference data mode.
115 * By default, ReferenceDataMode::Compare is used, but ``--ref-data create`` or
116 * ``--ref-data update`` can be used to change it.
118 * This function is automatically called by initTestUtils().
120 * \ingroup module_testutils
122 void initReferenceData(IOptionsContainer
* options
);
124 class TestReferenceChecker
;
128 class TestReferenceDataImpl
;
131 /*! \libinternal \brief
132 * Handles creation of and comparison to test reference data.
134 * See \ref page_refdata for an overview of the functionality.
136 * This class provides functionality to use the same code to generate reference
137 * data and then on later runs compare the results of the code against that
138 * reference. The mode in which the class operates (writing reference data or
139 * comparing against existing data) is set using a command-line option that
140 * is automatically managed when using the testutils module to implement tests.
141 * Tests only need to create an instance of TestReferenceData, obtain a
142 * TestReferenceChecker using the rootChecker() method and use the various
143 * check*() methods in TestReferenceChecker to indicate values to check. If
144 * the test is running in reference data creation mode, it will produce an XML
145 * file with the values recorder. In comparison mode, it will read that same
146 * XML file and produce a Google Test non-fatal assertion for every discrepancy
147 * it detects with the reference data (including missing reference data file or
148 * individual item). Exceptions derived from TestException are thrown for I/O
149 * errors and syntax errors in the reference data.
151 * Simple example (using Google Test):
153 int functionToTest(int param);
159 TEST(MyTest, SimpleTest)
161 TestReferenceData data;
163 TestReferenceChecker checker(data.rootChecker());
164 checker.checkInteger(functionToTest(3), "ValueWith3");
165 checker.checkInteger(functionToTest(5), "ValueWith5");
166 TestReferenceChecker compound(
167 checker.checkCompound("CustomCompound", "Item"));
168 compound.checkInteger(function2ToTest(3), "ValueWith3");
169 compound.checkInteger(function2ToTest(5), "ValueWith5");
170 checker.checkInteger(functionToTest(4), "ValueWith4");
176 * If rootChecker() is never called, no comparison is done (i.e., missing
177 * reference data file is not reported as an error, nor is empty reference data
178 * file created in write mode).
180 * For floating-point comparisons, the reference data should be generated in
181 * double precision (currently, no warning is provided even if this is not the
182 * case, but the double precision tests will then very likely fail).
185 * \ingroup module_testutils
187 class TestReferenceData
191 * Initializes the reference data in the global mode.
195 * Initializes the reference data in a specific mode.
197 * This function is only useful for self-testing the reference data
198 * framework. As such, it also puts the framework in a state where it
199 * logs additional internal information for failures to help diagnosing
200 * problems in the framework, and stores the reference data in a
201 * temporary directory instead of the source tree.
202 * The default constructor should be used in tests utilizing this class.
204 explicit TestReferenceData(ReferenceDataMode mode
);
206 * Frees reference data structures.
208 * The reference data is written out if necessary automatically when
211 ~TestReferenceData();
214 * Returns a root-level checker object for comparisons.
216 * Each call returns an independent instance.
218 TestReferenceChecker
rootChecker();
221 std::shared_ptr
<internal::TestReferenceDataImpl
> impl_
;
223 GMX_DISALLOW_COPY_AND_ASSIGN(TestReferenceData
);
226 /*! \libinternal \brief
227 * Handles comparison to test reference data.
229 * Every check*() method takes an id string as the last parameter. This id is
230 * used to uniquely identify the value in the reference data, and it makes the
231 * output XML more human-friendly and more robust to errors. The id can be
232 * NULL; in this case, multiple elements with no id are created, and they will
233 * be matched in the same order as in which they are created. The
234 * checkCompound() method can be used to create a set of reference values
235 * grouped together. In this case, all check*() calls using the returned child
236 * TestReferenceChecker object will create the reference data within this
237 * group, and the ids only need to be unique within the compound. Compounds
240 * For usage example, see TestReferenceData.
242 * Copies of this class behave have independent internal state.
245 * \ingroup module_testutils
247 class TestReferenceChecker
251 * Creates a checker that cannot be used for checking.
253 * Attempting to call the check methods generates an assert.
254 * It is possible to check whether the checker is initialized by
256 * This constructor exists to allow declaring checker variables that
257 * will receive their value later without resorting to dynamic
260 TestReferenceChecker();
261 //! Creates a deep copy of the other checker.
262 explicit TestReferenceChecker(const TestReferenceChecker
& other
);
263 //! Moves the checker.
264 TestReferenceChecker(TestReferenceChecker
&& other
) noexcept
;
265 ~TestReferenceChecker();
267 //! Prevents implicit copying during assignment.
268 TestReferenceChecker
& operator=(const TestReferenceChecker
&) = delete;
269 //! Assigns a test reference checker.
270 TestReferenceChecker
& operator=(TestReferenceChecker
&& other
) noexcept
;
272 //! Returns whether the checker is initialized.
273 bool isValid() const;
274 //! Allows testing whether the checker is initialized directly with if.
275 explicit operator bool() const { return isValid(); }
278 * Sets the tolerance for floating-point comparisons.
280 * All following floating-point comparisons using this checker will use
281 * the new tolerance. Child checkers created with checkCompound()
282 * will inherit the tolerance from their parent checker at the time
283 * checkCompound() is called.
287 void setDefaultTolerance(const FloatingPointTolerance
& tolerance
);
290 * Checks that all reference values have been compared against.
292 * All values under the compound represented by this checker are
293 * checked, and a non-fatal Google Test assertion is produced if some
294 * values have not been used.
296 * If not called explicitly, the same check will be done for all
297 * reference data values when the test ends.
299 * This method also marks the values used, so that subsequent checks
300 * (including the check at the end of the test) will not produce
301 * another assertion about the same values.
303 void checkUnusedEntries();
306 * Checks whether a data item is present.
308 * \param[in] bPresent Whether to check for presence or absence.
309 * \param[in] id Unique identifier of the item to check.
310 * \returns true if bPresent was true and the data item was found.
312 * If \p bPresent is true, checks that a data item with \p id is
313 * present, otherwise checks that the data item is absent.
314 * If the check fails, a non-fatal Google Test assertion is generated.
316 * If reference data is being written, the check always succeeds and the
317 * return value is \p bPresent.
319 * The main use of this method is to assign meaning for missing
320 * reference data. Example use:
322 if (checker.checkPresent(bHaveVelocities, "Velocities"))
324 // <check the velocities>
328 bool checkPresent(bool bPresent
, const char* id
);
331 * Initializes comparison of a group of related data items.
333 * \param[in] type Informational type for the compound.
334 * \param[in] id Unique identifier for the compound among its
336 * \returns Checker to use for comparison within the compound.
338 * All checks performed with the returned checker only
339 * need to have unique ids within the compound, not globally.
341 * Compound structures can be nested.
343 TestReferenceChecker
checkCompound(const char* type
, const char* id
);
344 //! \copydoc checkCompound(const char *, const char *)
345 TestReferenceChecker
checkCompound(const char* type
, const std::string
& id
);
347 //! Check a single boolean value.
348 void checkBoolean(bool value
, const char* id
);
349 //! Check a single string value.
350 void checkString(const char* value
, const char* id
);
351 //! Check a single string value.
352 void checkString(const std::string
& value
, const char* id
);
354 * Check a multi-line string value.
356 * This method works as checkString(), but should be used for long
357 * strings that may contain, e.g., newlines. Typically used to check
358 * formatted output, and attempts to make the output XML such that it
359 * is easier to edit by hand to set the desired output formatting.
361 void checkTextBlock(const std::string
& value
, const char* id
);
362 //! Check a single unsigned char value.
363 void checkUChar(unsigned char value
, const char* id
);
364 //! Check a single integer value.
365 void checkInteger(int value
, const char* id
);
366 //! Check a single int32 value.
367 void checkInt32(int32_t value
, const char* id
);
368 //! Check a single uint32 value.
369 void checkUInt32(uint32_t value
, const char* id
);
370 //! Check a single int64 value.
371 void checkInt64(int64_t value
, const char* id
);
372 //! Check a single uint64 value.
373 void checkUInt64(uint64_t value
, const char* id
);
374 //! Check a single single-precision floating point value.
375 void checkFloat(float value
, const char* id
);
376 //! Check a single double-precision floating point value.
377 void checkDouble(double value
, const char* id
);
378 //! Check a single floating point value.
379 void checkReal(float value
, const char* id
);
380 //! Check a single floating point value.
381 void checkReal(double value
, const char* id
);
382 //! Check a vector of three integer values.
383 void checkVector(const int value
[3], const char* id
);
384 //! Check a vector of three single-precision floating point values.
385 void checkVector(const float value
[3], const char* id
);
386 //! Check a vector of three double-precision floating point values.
387 void checkVector(const double value
[3], const char* id
);
388 //! Check a single floating-point value from a string.
389 void checkRealFromString(const std::string
& value
, const char* id
);
390 //! Checks a any value that contains a supported simple type.
391 void checkAny(const Any
& value
, const char* id
);
392 //! Checks a key-value tree rooted at a object.
393 void checkKeyValueTreeObject(const KeyValueTreeObject
& tree
, const char* id
);
394 //! Checks a generic key-value tree value.
395 void checkKeyValueTreeValue(const KeyValueTreeValue
& value
, const char* id
);
397 /*! \name Methods to read values from reference data
399 * These methods assume that a value with the given `id` has already
400 * been created in the test with `check*()` methods, and that it has
403 * Currently, these methods do not work correctly if the reference data
404 * file does not exist, so a test using them may fail with exceptions
405 * before the reference data has been generated.
408 //! Reads an unsigned char value.
409 unsigned char readUChar(const char* id
);
410 //! Reads an integer value.
411 int readInteger(const char* id
);
412 //! Reads a 32-bit integer value.
413 int32_t readInt32(const char* id
);
414 //! Reads a 64-bit integer value.
415 int64_t readInt64(const char* id
);
416 //! Reads a float value.
417 float readFloat(const char* id
);
418 //! Reads a double value.
419 double readDouble(const char* id
);
420 //! Reads a string value.
421 std::string
readString(const char* id
);
424 /*! \name Overloaded versions of simple checker methods
426 * These methods provide overloads under a single name for all the
427 * methods checkBoolean(), checkString(), checkReal() and checkVector().
428 * They are provided mainly to allow template implementations (such as
429 * checkSequence()). Typically callers should use the individually
430 * named versions for greater clarity.
433 //! Check a single boolean value.
434 void checkValue(bool value
, const char* id
) { checkBoolean(value
, id
); }
435 //! Check a single string value.
436 void checkValue(const char* value
, const char* id
) { checkString(value
, id
); }
437 //! Check a single string value.
438 void checkValue(const std::string
& value
, const char* id
) { checkString(value
, id
); }
439 //! Check a single signed integer value
440 void checkValue(int value
, const char* id
) { checkInteger(value
, id
); }
441 //! Check a single signed integer value of width 64 bits.
442 void checkValue(int64_t value
, const char* id
) { checkInt64(value
, id
); }
443 //! Check a single unsigned integer value of width 64 bits.
444 void checkValue(uint64_t value
, const char* id
) { checkUInt64(value
, id
); }
445 //! Check a single single-precision floating point value.
446 void checkValue(float value
, const char* id
) { checkFloat(value
, id
); }
447 //! Check a single double-precision floating point value.
448 void checkValue(double value
, const char* id
) { checkDouble(value
, id
); }
449 //! Check a vector of three integer values.
450 void checkValue(const int value
[3], const char* id
) { checkVector(value
, id
); }
451 //! Check a vector of three single-precision floating point values.
452 void checkValue(const float value
[3], const char* id
) { checkVector(value
, id
); }
453 //! Check a vector of three double-precision floating point values.
454 void checkValue(const double value
[3], const char* id
) { checkVector(value
, id
); }
455 //! Check a generic key-value tree value.
456 void checkValue(const KeyValueTreeValue
& value
, const char* id
)
458 checkKeyValueTreeValue(value
, id
);
463 * Generic method to check a sequence of simple values.
465 * \tparam Iterator Input iterator that allows multiple (two) passes.
466 * Value type must be one of those accepted by checkValue(), or
467 * implicitly convertible to one.
468 * \param[in] begin Iterator to the start of the range to check.
469 * \param[in] end Iterator to the end of the range to check.
470 * \param[in] id Unique identifier for the sequence among its
473 template<class Iterator
>
474 void checkSequence(Iterator begin
, Iterator end
, const char* id
)
476 typename
std::iterator_traits
<Iterator
>::difference_type length
= std::distance(begin
, end
);
477 TestReferenceChecker
compound(checkSequenceCompound(id
, length
));
478 for (Iterator i
= begin
; i
!= end
; ++i
)
480 compound
.checkValue(*i
, nullptr);
484 * Generic method to check a sequence of custom values.
486 * \tparam Iterator Input iterator that allows multiple (two) passes.
487 * \tparam ItemChecker Functor to check an individual value. Signature
488 * void(TestReferenceChecker *, const T &), where T is the value
489 * type of \p Iterator.
490 * \param[in] begin Iterator to the start of the range to check.
491 * \param[in] end Iterator to the end of the range to check.
492 * \param[in] id Unique identifier for the sequence among its
494 * \param[in] checkItem Functor to check an individual item.
496 * This method creates a compound checker \c compound within which all
497 * values of the sequence are checked. Calls \c checkItem(&compound, *i)
498 * with that compound for each iterator \c i in the range [begin, end).
499 * \p checkItem should use the various check methods in the passed
500 * checker to check each value.
502 * This method can be used to check a sequence made of compound types.
503 * Typically \p checkItem will create a compound within the passed
504 * checker to check different aspects of the value that was passed
505 * to it. Either NULL or a unique identifier string must be used for
506 * the id value of that compound. */
507 template<class Iterator
, class ItemChecker
>
508 void checkSequence(Iterator begin
, Iterator end
, const char* id
, ItemChecker checkItem
)
510 typename
std::iterator_traits
<Iterator
>::difference_type length
= std::distance(begin
, end
);
511 TestReferenceChecker
compound(checkSequenceCompound(id
, length
));
512 for (Iterator i
= begin
; i
!= end
; ++i
)
514 checkItem(&compound
, *i
);
518 * Check an array of values.
520 * \tparam T Type of values to check. Should be one of those accepted
521 * by checkValue(), or implicitly convertible to one.
523 * \param[in] length Number of values to check.
524 * \param[in] values Pointer to the first value to check.
525 * \param[in] id Unique identifier for the sequence among its
528 * This is a convenience method that delegates all work to
532 void checkSequenceArray(size_t length
, const T
* values
, const char* id
)
534 checkSequence(values
, values
+ length
, id
);
537 * Convenience method for checking that a sequence is empty.
539 * \param[in] id Unique identifier for the sequence among its
542 * This method provides a convenient solution for a case where there is
543 * implicitly a sequence to be checked, but there is no pointer
544 * available to the values since the sequence is empty.
545 * Since this method does not require the type of the values, it can be
546 * used in such cases easily.
548 void checkEmptySequence(const char* id
);
550 * Initializes a compound for a sequence of items.
552 * \param[in] id Unique identifier for the sequence among its
554 * \param[in] length Number of items that will be in the sequence.
555 * \returns Checker to use for comparison within the sequence.
557 * This method can be used to check custom sequences where
558 * checkSequence() is not appropriate.
560 TestReferenceChecker
checkSequenceCompound(const char* id
, size_t length
);
566 * Constructs a checker with a specific internal state.
568 * Is private to only allow users of this class to create instances
569 * using TestReferenceData::rootChecker() or checkCompound()
572 explicit TestReferenceChecker(Impl
* impl
);
574 PrivateImplPointer
<Impl
> impl_
;
577 * Needed to expose the constructor only to TestReferenceData.
579 friend class TestReferenceData
;