22 #include "system-path.h"
40 double delta, difference;
46 double max = (std::fabs (x1) > std::fabs (x2)) ? x1 : x2;
47 (void)std::frexp (max, &exponent);
53 delta = std::ldexp (epsilon, exponent);
56 if (difference > delta || difference < -delta)
66 std::string _limit, std::string _message,
67 std::string _file, int32_t _line);
79 std::vector<TestCaseFailure> failure;
87 void StartTestCase (std::string name);
88 void EndTestCase (
void);
89 void ReportTestFailure (std::string cond, std::string actual,
90 std::string limit, std::string message,
91 std::string file, int32_t line);
92 bool MustAssertOnFailure (
void)
const;
93 bool MustContinueOnFailure (
void)
const;
94 bool MustUpdateData (
void)
const;
95 std::string GetTopLevelSourceDir (
void)
const;
96 std::string GetTempDir (
void)
const;
98 int Run (
int argc,
char *argv[]);
106 bool IsTopLevelSourceDir (std::string path)
const;
107 std::string ReplaceXmlSpecialCharacters (std::string xml)
const;
108 void PrintReport (
TestCase *test, std::ostream *os,
bool xml,
int level);
109 void PrintTestNameList (std::list<TestCase *>::const_iterator begin,
110 std::list<TestCase *>::const_iterator end,
111 bool printTestType)
const;
112 void PrintTestTypeList (
void)
const;
113 void PrintHelp (
const char *programName)
const;
114 std::list<TestCase *> FilterTests (std::string testName,
119 typedef std::vector<TestSuite *> TestSuiteVector;
121 TestSuiteVector m_suites;
122 std::string m_tempDir;
124 bool m_assertOnFailure;
125 bool m_continueOnFailure;
131 TestCaseFailure::TestCaseFailure (std::string _cond, std::string _actual,
132 std::string _limit, std::string _message,
133 std::string _file, int32_t _line)
134 : cond (_cond), actual (_actual), limit (_limit),
135 message (_message), file (_file), line (_line)
137 NS_LOG_FUNCTION (
this << _cond << _actual << _limit << _message << _file << _line);
139 TestCase::Result::Result ()
140 : childrenFailed (false)
158 TestCase::~TestCase ()
164 for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
181 testCase->m_duration = duration;
184 m_children.push_back (testCase);
185 testCase->m_parent =
this;
187 std::string::size_type slash, antislash;
188 slash = testCase->m_name.find (
"/");
189 antislash = testCase->m_name.find (
"\\");
190 if (slash != std::string::npos || antislash != std::string::npos)
192 std::string fullname = testCase->m_name;
193 TestCase *current = testCase->m_parent;
196 fullname = current->m_name +
"/" + fullname;
197 current = current->m_parent;
199 if (slash != std::string::npos)
201 NS_FATAL_ERROR (
"Invalid test name: cannot contain slashes: \"" << fullname <<
"\"");
203 if (antislash != std::string::npos)
205 NS_FATAL_ERROR (
"Invalid test name: cannot contain antislashes: \"" << fullname <<
"\"");
211 TestCase::IsFailed (
void)
const
214 return m_result->childrenFailed || !m_result->failure.empty ();
218 TestCase::Run (TestRunnerImpl *runner)
221 m_result =
new Result ();
224 m_result->clock.
Start ();
225 for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
236 m_result->clock.
End ();
247 TestCase::ReportTestFailure (std::string cond, std::string actual,
248 std::string limit, std::string message,
249 std::string file, int32_t line)
251 NS_LOG_FUNCTION (
this << cond << actual << limit << message << file << line);
253 message, file, line));
258 current->m_result->childrenFailed =
true;
259 current = current->m_parent;
264 TestCase::MustAssertOnFailure (
void)
const
267 return m_runner->MustAssertOnFailure ();
270 TestCase::MustContinueOnFailure (
void)
const
273 return m_runner->MustContinueOnFailure ();
277 TestCase::CreateDataDirFilename (std::string filename)
281 while (current->m_dataDir ==
"" && current != 0)
283 current = current->m_parent;
287 NS_FATAL_ERROR (
"No one called SetDataDir prior to calling this function");
290 std::string a =
SystemPath::Append (m_runner->GetTopLevelSourceDir (), current->m_dataDir);
295 TestCase::CreateTempDirFilename (std::string filename)
298 if (m_runner->MustUpdateData ())
300 return CreateDataDirFilename (filename);
304 std::list<std::string> names;
308 names.push_front (current->m_name);
309 current = current->m_parent;
332 return m_result->failure.empty ();
339 m_dataDir = directory;
359 TestRunnerImpl::Instance ()->AddTestSuite (
this);
375 TestRunnerImpl::TestRunnerImpl ()
377 m_assertOnFailure (false),
378 m_continueOnFailure (true),
384 TestRunnerImpl::~TestRunnerImpl ()
392 TestRunnerImpl::Instance (
void)
395 static TestRunnerImpl runner;
400 TestRunnerImpl::AddTestSuite (TestSuite *testSuite)
403 m_suites.push_back (testSuite);
408 TestRunnerImpl::MustAssertOnFailure (
void)
const
411 return m_assertOnFailure;
414 TestRunnerImpl::MustContinueOnFailure (
void)
const
417 return m_continueOnFailure;
421 TestRunnerImpl::MustUpdateData (
void)
const
427 TestRunnerImpl::GetTempDir (
void)
const
433 TestRunnerImpl::IsTopLevelSourceDir (std::string path)
const
436 bool haveVersion =
false;
437 bool haveLicense =
false;
445 for (std::list<std::string>::const_iterator i = files.begin (); i != files.end (); ++i)
451 else if (*i ==
"LICENSE")
457 return haveVersion && haveLicense;
461 TestRunnerImpl::GetTopLevelSourceDir (
void)
const
466 while (!elements.empty ())
469 if (IsTopLevelSourceDir (path))
473 elements.pop_back ();
475 NS_FATAL_ERROR (
"Could not find source directory from self=" <<
self);
484 TestRunnerImpl::ReplaceXmlSpecialCharacters (std::string xml)
const
487 std::string specials =
"<>&\"'";
488 std::string replacements[] = {
"<",
">",
"&",
"'",
"""};
490 std::size_t index, length = xml.length ();
492 for (
size_t i = 0; i < length; ++i)
494 char character = xml[i];
496 if ((index = specials.find (character)) == std::string::npos)
498 result.push_back (character);
502 result += replacements[index];
513 Indent::Indent (
int _level)
518 std::ostream &
operator << (std::ostream &os,
const Indent &val)
520 for (
int i = 0; i < val.level; i++)
528 TestRunnerImpl::PrintReport (TestCase *test, std::ostream *os,
bool xml,
int level)
531 if (test->m_result == 0)
537 const double MS_PER_SEC = 1000.;
538 double real = test->m_result->clock.GetElapsedReal () / MS_PER_SEC;
539 double user = test->m_result->clock.GetElapsedUser () / MS_PER_SEC;
540 double system = test->m_result->clock.GetElapsedSystem () / MS_PER_SEC;
545 std::string statusString = test->IsFailed ()?
"FAIL":
"PASS";
548 *os << Indent (level) <<
"<Test>" << std::endl;
549 *os << Indent (level+1) <<
"<Name>" << ReplaceXmlSpecialCharacters (test->m_name)
550 <<
"</Name>" << std::endl;
551 *os << Indent (level+1) <<
"<Result>" << statusString <<
"</Result>" << std::endl;
552 *os << Indent (level+1) <<
"<Time real=\"" << real <<
"\" user=\"" << user
553 <<
"\" system=\"" << system <<
"\"/>" << std::endl;
554 for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
556 TestCaseFailure failure = test->m_result->failure[i];
557 *os << Indent (level+2) <<
"<FailureDetails>" << std::endl
558 << Indent (level+3) <<
"<Condition>"
559 << ReplaceXmlSpecialCharacters (failure.cond) <<
"</Condition>" << std::endl
560 << Indent (level+3) <<
"<Actual>"
561 << ReplaceXmlSpecialCharacters (failure.actual) <<
"</Actual>" << std::endl
562 << Indent (level+3) <<
"<Limit>"
563 << ReplaceXmlSpecialCharacters (failure.limit) <<
"</Limit>" << std::endl
564 << Indent (level+3) <<
"<Message>"
565 << ReplaceXmlSpecialCharacters (failure.message) <<
"</Message>" << std::endl
566 << Indent (level+3) <<
"<File>"
567 << ReplaceXmlSpecialCharacters (failure.file) <<
"</File>" << std::endl
568 << Indent (level+3) <<
"<Line>" << failure.line <<
"</Line>" << std::endl
569 << Indent (level+2) <<
"</FailureDetails>" << std::endl;
571 for (uint32_t i = 0; i < test->m_children.size (); i++)
573 TestCase *child = test->m_children[i];
574 PrintReport (child, os, xml, level + 1);
576 *os << Indent (level) <<
"</Test>" << std::endl;
580 *os << Indent (level) << statusString <<
" " << test->GetName ()
581 <<
" " << real <<
" s" << std::endl;
584 for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
586 TestCaseFailure failure = test->m_result->failure[i];
587 *os << Indent (level) <<
" got=\"" << failure.cond <<
"\" expected=\""
588 << failure.actual <<
"\" in=\"" << failure.file <<
":" << failure.line
589 <<
"\" " << failure.message << std::endl;
591 for (uint32_t i = 0; i < test->m_children.size (); i++)
593 TestCase *child = test->m_children[i];
594 PrintReport (child, os, xml, level + 1);
601 TestRunnerImpl::PrintHelp (
const char *program_name)
const
604 std::cout <<
"Usage: " << program_name <<
" [OPTIONS]" << std::endl
606 <<
"Options: " << std::endl
607 <<
" --help : print these options" << std::endl
608 <<
" --print-test-name-list : print the list of names of tests available" << std::endl
609 <<
" --list : an alias for --print-test-name-list" << std::endl
610 <<
" --print-test-types : print the type of tests along with their names" << std::endl
611 <<
" --print-test-type-list : print the list of types of tests available" << std::endl
612 <<
" --print-temp-dir : print name of temporary directory before running " << std::endl
613 <<
" the tests" << std::endl
614 <<
" --test-type=TYPE : process only tests of type TYPE" << std::endl
615 <<
" --test-name=NAME : process only test whose name matches NAME" << std::endl
616 <<
" --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
617 <<
" for --test-name=NAME" << std::endl
618 <<
" --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
619 <<
" when running under a debugger" << std::endl
620 <<
" --stop-on-failure : when a test fails, stop immediately" << std::endl
621 <<
" --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
622 <<
" EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
623 <<
" includes QUICK and TAKES_FOREVER includes " << std::endl
624 <<
" QUICK and EXTENSIVE (only QUICK tests are " << std::endl
625 <<
" run by default)" << std::endl
626 <<
" --verbose : print details of test execution" << std::endl
627 <<
" --xml : format test run output as xml" << std::endl
628 <<
" --tempdir=DIR : set temp dir for tests to store output files" << std::endl
629 <<
" --datadir=DIR : set data dir for tests to read reference files" << std::endl
630 <<
" --out=FILE : send test result to FILE instead of standard "
631 <<
"output" << std::endl
632 <<
" --append=FILE : append test result to FILE instead of standard "
633 <<
"output" << std::endl
638 TestRunnerImpl::PrintTestNameList (std::list<TestCase *>::const_iterator begin,
639 std::list<TestCase *>::const_iterator end,
640 bool printTestType)
const
643 std::map<TestSuite::Type, std::string> label;
645 label[TestSuite::ALL] =
"all ";
652 for (std::list<TestCase *>::const_iterator i = begin; i != end; ++i)
654 TestSuite * test=
dynamic_cast<TestSuite *
>(*i);
657 std::cout << label[test->GetTestType ()];
659 std::cout << test->GetName () << std::endl;
664 TestRunnerImpl::PrintTestTypeList (
void)
const
667 std::cout <<
" bvt: Build Verification Tests (to see if build completed successfully)" << std::endl;
668 std::cout <<
" core: Run all TestSuite-based tests (exclude examples)" << std::endl;
669 std::cout <<
" example: Examples (to see if example programs run successfully)" << std::endl;
670 std::cout <<
" performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
671 std::cout <<
" system: System Tests (spans modules to check integration of modules)" << std::endl;
672 std::cout <<
" unit: Unit Tests (within modules to check basic functionality)" << std::endl;
676 std::list<TestCase *>
677 TestRunnerImpl::FilterTests (std::string testName,
682 std::list<TestCase *> tests;
683 for (uint32_t i = 0; i < m_suites.size (); ++i)
685 TestSuite *test = m_suites[i];
686 if (testType != TestSuite::ALL && test->GetTestType () != testType)
691 if (testName !=
"" && test->GetName () != testName)
698 std::vector<TestCase *>::iterator j;
699 for (j = test->m_children.begin (); j != test->m_children.end ();)
701 TestCase *testCase = *j;
705 if (testCase->m_duration > maximumTestDuration)
711 test->m_children.erase (j);
722 tests.push_back (test);
729 TestRunnerImpl::Run (
int argc,
char *argv[])
732 std::string testName =
"";
733 std::string testTypeString =
"";
734 std::string out =
"";
735 std::string fullness =
"";
738 bool printTempDir =
false;
739 bool printTestTypeList =
false;
740 bool printTestNameList =
false;
741 bool printTestTypeAndName =
false;
743 char *progname = argv[0];
751 if (strcmp(arg,
"--assert-on-failure") == 0)
753 m_assertOnFailure =
true;
755 else if (strcmp (arg,
"--stop-on-failure") == 0)
757 m_continueOnFailure =
false;
759 else if (strcmp (arg,
"--verbose") == 0)
763 else if (strcmp (arg,
"--print-temp-dir") == 0)
767 else if (strcmp (arg,
"--update-data") == 0)
771 else if (strcmp (arg,
"--help") == 0)
773 PrintHelp (progname);
776 else if (strcmp (arg,
"--print-test-name-list") == 0 ||
777 strcmp(arg,
"--list") == 0)
779 printTestNameList =
true;
781 else if (strcmp (arg,
"--print-test-types") == 0)
783 printTestTypeAndName =
true;
785 else if (strcmp (arg,
"--print-test-type-list") == 0)
787 printTestTypeList =
true;
789 else if (strcmp(arg,
"--append") == 0)
793 else if (strcmp(arg,
"--xml") == 0)
797 else if (strncmp(arg,
"--test-type=", strlen(
"--test-type=")) == 0)
799 testTypeString = arg + strlen(
"--test-type=");
801 else if (strncmp(arg,
"--test-name=", strlen(
"--test-name=")) == 0)
803 testName = arg + strlen(
"--test-name=");
805 else if (strncmp(arg,
"--suite=", strlen(
"--suite=")) == 0)
807 testName = arg + strlen(
"--suite=");
809 else if (strncmp(arg,
"--tempdir=", strlen(
"--tempdir=")) == 0)
811 m_tempDir = arg + strlen(
"--tempdir=");
813 else if (strncmp(arg,
"--out=", strlen(
"--out=")) == 0)
815 out = arg + strlen(
"--out=");
817 else if (strncmp(arg,
"--fullness=", strlen(
"--fullness=")) == 0)
819 fullness = arg + strlen(
"--fullness=");
822 if (fullness ==
"EXTENSIVE")
826 else if (fullness ==
"TAKES_FOREVER")
832 maximumTestDuration = TestCase::QUICK;
838 PrintHelp (progname);
844 if (testTypeString ==
"")
846 testType = TestSuite::ALL;
848 else if (testTypeString ==
"bvt")
852 else if (testTypeString ==
"core")
854 testType = TestSuite::ALL;
856 else if (testTypeString ==
"example")
860 else if (testTypeString ==
"unit")
864 else if (testTypeString ==
"system")
868 else if (testTypeString ==
"performance")
874 std::cout <<
"Invalid test type specified: " << testTypeString << std::endl;
875 PrintTestTypeList ();
879 std::list<TestCase *> tests = FilterTests (testName, testType, maximumTestDuration);
887 std::cout << m_tempDir << std::endl;
889 if (printTestNameList)
891 PrintTestNameList (tests.begin (), tests.end (), printTestTypeAndName);
894 if (printTestTypeList)
896 PrintTestTypeList ();
905 ofs =
new std::ofstream();
906 std::ios_base::openmode mode = std::ios_base::out;
909 mode |= std::ios_base::app;
913 mode |= std::ios_base::trunc;
915 ofs->open (out.c_str (), mode);
925 for (std::list<TestCase *>::const_iterator i = tests.begin (); i != tests.end (); ++i)
930 PrintReport (test, os, xml, 0);
931 if (test->IsFailed ())
934 if (!m_continueOnFailure)
950 TestRunner::Run (
int argc,
char *argv[])
953 return TestRunnerImpl::Instance ()->Run (argc, argv);
virtual void DoSetup(void)
Implementation to do any local setup required for this test case.
virtual void DoTeardown(void)
Implementation to do any local setup required for this test case.
#define NS_LOG_FUNCTION(parameters)
#define NS_ASSERT(condition)
#define NS_LOG_COMPONENT_DEFINE(name)
void MakeDirectories(std::string path)
#define NS_LOG_FUNCTION_NOARGS()
TestSuite(std::string name, Type type=UNIT)
Constuct a new test suite.
#define NS_FATAL_ERROR(msg)
fatal error handling
virtual void DoRun(void)
Implementation to actually run this test case.
measure elapsed time in milliseconds
std::string Join(std::list< std::string >::const_iterator begin, std::list< std::string >::const_iterator end)
bool TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
Compare two double precision floating point numbers and declare them equal if they are within some ep...
std::list< std::string > ReadFiles(std::string path)
TestCase(std::string name)
std::string FindSelfDirectory(void)
std::ostream & operator<<(std::ostream &os, const Angles &a)
bool GetErrorStatus(void) const NS_DEPRECATED
TestDuration
How long the test takes to execute.
virtual void DoRun(void)=0
Implementation to actually run this test case.
bool IsStatusFailure(void) const
std::string Append(std::string left, std::string right)
std::string MakeTemporaryDirectoryName(void)
void AddTestCase(TestCase *testCase) NS_DEPRECATED
Add an individual test case to this test suite.
bool IsStatusSuccess(void) const
void SetDataDir(std::string directory)
std::string GetName(void) const
TestSuite::Type GetTestType(void)
get the kind of test this test suite implements
std::list< std::string > Split(std::string path)
int64_t End(void)
Stop measuring the time since Start() was called.