From c7e1277ecaf40c6d8ee945418a306f5b15189b97 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 8 Aug 2023 16:33:38 -0700 Subject: Unit test augmentations and harness. Added some features to the mkunit program, including cleanup routine support. Added reporting modes for the UnitSuite class, and it can now generate machine readable reports. Added a new program, rununits that runs all unit tests and generates a synopsis of what you really care about at the end, issues! --- src/stable/unitsuite.cpp | 337 +++++++++++++++++++++++++++++++++++------------ src/stable/unitsuite.h | 73 ++++++++++ 2 files changed, 328 insertions(+), 82 deletions(-) (limited to 'src/stable') diff --git a/src/stable/unitsuite.cpp b/src/stable/unitsuite.cpp index a38c77a..b2544f2 100644 --- a/src/stable/unitsuite.cpp +++ b/src/stable/unitsuite.cpp @@ -9,6 +9,7 @@ #include "bu/file.h" #include "bu/sio.h" #include "bu/optparser.h" +#include "bu/json.h" #include #include @@ -18,28 +19,33 @@ using namespace Bu; Bu::UnitSuite::UnitSuite() : iOptions( 0 ), - iNameWidth( 0 ) + pReport( NULL ) { } Bu::UnitSuite::UnitSuite( int iOptions ) : - iOptions( iOptions ), - iNameWidth( 0 ) + iOptions( iOptions ) { } Bu::UnitSuite::~UnitSuite() { + delete pReport; } int Bu::UnitSuite::run( int argc, char *argv[] ) { bool bCleanup = true; + bool bInterop = false; OptParser p; p.addOption( Bu::slot( this, &Bu::UnitSuite::onListCases ), 'l', "list", "List available test cases." ); + p.addOption( Bu::slot( this, &Bu::UnitSuite::onPrintName ), "print-name", + "Print the internal name of this test suite."); p.addOption( bCleanup, "no-cleanup", "Don't erase temp files."); p.setOverride( "no-cleanup", false ); + p.addOption( bInterop, "interop", "Output machine parsable json."); + p.setOverride( "interop", true ); p.setNonOption( Bu::slot( this, &Bu::UnitSuite::onAddTest ) ); p.addHelpOption(); p.parse( argc, argv ); @@ -56,108 +62,58 @@ int Bu::UnitSuite::run( int argc, char *argv[] ) lTests = lSub; } - int iEPass = 0; - int iEFail = 0; - int iUPass = 0; - int iUFail = 0; + if( bInterop ) + pReport = new ReportJson(); + else + pReport = new ReportConsole(); + pReport->suiteStarting( *this, lTests ); + for( TestList::iterator i = lTests.begin(); i != lTests.end(); i++ ) { - sio << Fmt( iNameWidth+3, Fmt::Left ).fill('.') << i->sName - << sio.flush; try { iStepCount = -1; iProgress = 0; + if( pReport ) + pReport->testStarting( *i ); (this->*(i->fTest))(); - switch( i->eExpect ) - { - case expectPass: - sio << "pass." << sio.nl; - iEPass++; - break; - - case expectFail: - sio << "unexpected pass." << sio.nl; - iUPass++; - break; - } + if( pReport ) + pReport->testEnded( *i ); } catch( Failed &e ) { - switch( i->eExpect ) - { - case expectPass: - sio << "unexpected "; - iUFail++; - break; - - case expectFail: - sio << "expected "; - iEFail++; - break; - } - if( e.bFile ) - { - sio << "fail in unitTest(" << e.str << "). (" << e.sFile - << ":" << e.nLine << ")." << sio.nl; - } - else - { - sio << "fail in unitTest(" << e.str << ")." << sio.nl; - } + if( pReport ) + pReport->testEnded( *i, e ); if( (iOptions & optStopOnError) ) + { return 0; + } } catch( std::exception &e ) { - switch( i->eExpect ) - { - case expectPass: - sio << "unexpected "; - iUFail++; - break; - - case expectFail: - sio << "expected "; - iEFail++; - break; - } - sio << "fail with unknown exception. what: " << e.what() << sio.nl; + if( pReport ) + pReport->testException( *i, e ); if( (iOptions & optStopOnError) ) + { return 0; + } } catch( ... ) { - switch( i->eExpect ) - { - case expectPass: - sio << "unexpected "; - iUFail++; - break; - - case expectFail: - sio << "expected "; - iEFail++; - break; - } sio << "fail with external exception." << sio.nl; + return -1; if( (iOptions & optStopOnError) ) + { return 0; + } } } - sio << sio.nl - << "Report:" << sio.nl - << "\tTotal tests run: " << lTests.getSize() << sio.nl - << "\tExpected passes: " << iEPass << sio.nl - << "\tExpected failures: " << iEFail << sio.nl - << "\tUnexpected passes: " << iUPass << sio.nl - << "\tUnexpected failures: " << iUFail << sio.nl << sio.nl; - if( iUPass == 0 && iUFail == 0 ) - sio << "\tNothing unexpected." << sio.nl << sio.nl; + if( pReport ) + pReport->suiteEnded(); if( bCleanup ) { @@ -165,6 +121,8 @@ int Bu::UnitSuite::run( int argc, char *argv[] ) { unlink( (*i).getStr() ); } + + cleanup(); } return 0; @@ -191,8 +149,6 @@ void Bu::UnitSuite::add( Test fTest, const Bu::String &sName, Expect e ) } ti.fTest = fTest; lTests.append( ti ); - if( iNameWidth < ti.sName.getSize() ) - iNameWidth = ti.sName.getSize(); } void Bu::UnitSuite::setName( const String &sName ) @@ -200,12 +156,21 @@ void Bu::UnitSuite::setName( const String &sName ) sSuiteName = sName; } +Bu::String Bu::UnitSuite::getName() const +{ + return sSuiteName; +} + +void Bu::UnitSuite::cleanup() +{ +} + void Bu::UnitSuite::dispProgress() { if( tLastUpdate == time( NULL ) ) return; - sio << Fmt(3) << (iProgress*100/iStepCount) << "%" << "\b\b\b\b" - << sio.flush; + if( pReport ) + pReport->updateProgress( iProgress, iStepCount ); tLastUpdate = time( NULL ); } @@ -251,6 +216,13 @@ int Bu::UnitSuite::onListCases( StrArray ) return 0; } +int Bu::UnitSuite::onPrintName( StrArray ) +{ + Bu::println("%1").arg( sSuiteName ); + exit( 0 ); + return 0; +} + int Bu::UnitSuite::onAddTest( StrArray aParam ) { hSelTests.insert( aParam[0], true ); @@ -262,12 +234,213 @@ Bu::Formatter &Bu::operator<<( Bu::Formatter &f, const Bu::UnitSuite::Expect &e switch( e ) { case Bu::UnitSuite::expectPass: - return f << "expect pass"; + return f << "pass"; case Bu::UnitSuite::expectFail: - return f << "expect fail"; + return f << "fail"; } return f << "**error**"; } +///////// +// Bu::UnitSuite::Report +//// + +Bu::UnitSuite::Report::Report() +{ +} + +Bu::UnitSuite::Report::~Report() +{ +} + +///////// +// Bu::UnitSuite::ReportConsole +//// + +Bu::UnitSuite::ReportConsole::ReportConsole() : + iTestCount( 0 ), + iNameWidth( 0 ), + iEPass( 0 ), + iEFail( 0 ), + iUPass( 0 ), + iUFail( 0 ) +{ +} + +Bu::UnitSuite::ReportConsole::~ReportConsole() +{ +} + +void Bu::UnitSuite::ReportConsole::suiteStarting( const UnitSuite & /*rSuite*/, + const TestList &lTests ) +{ + iTestCount = lTests.getSize(); + for( TestList::const_iterator i = lTests.begin(); i; i++ ) + { + if( iNameWidth < i->sName.getSize() ) + iNameWidth = i->sName.getSize(); + } +} + +void Bu::UnitSuite::ReportConsole::testStarting( const TestInfo &rTest ) +{ + sio << Fmt( iNameWidth+3, Fmt::Left ).fill('.') << rTest.sName + << sio.flush; +} + +void Bu::UnitSuite::ReportConsole::updateProgress( int iProgress, + int iStepCount ) +{ + sio << Fmt(3) << (iProgress*100/iStepCount) << "%" << "\b\b\b\b" + << sio.flush; +} + +void Bu::UnitSuite::ReportConsole::testEnded( const TestInfo &rTest ) +{ + switch( rTest.eExpect ) + { + case expectPass: + sio << "pass." << sio.nl; + iEPass++; + break; + + case expectFail: + sio << "unexpected pass." << sio.nl; + iUPass++; + break; + } +} + +void Bu::UnitSuite::ReportConsole::testEnded( const TestInfo &rTest, + const Bu::UnitSuite::Failed &rFail ) +{ + switch( rTest.eExpect ) + { + case expectPass: + sio << "unexpected "; + iUFail++; + break; + + case expectFail: + sio << "expected "; + iEFail++; + break; + } + if( rFail.bFile ) + { + sio << "fail in unitTest(" << rFail.str << "). (" << rFail.sFile + << ":" << rFail.nLine << ")." << sio.nl; + } + else + { + sio << "fail in unitTest(" << rFail.str << ")." << sio.nl; + } +} + +void Bu::UnitSuite::ReportConsole::testException( const TestInfo &rTest, + std::exception &e ) +{ + switch( rTest.eExpect ) + { + case expectPass: + sio << "unexpected "; + iUFail++; + break; + + case expectFail: + sio << "expected "; + iEFail++; + break; + } + sio << "fail with unknown exception. what: " << e.what() << sio.nl; +} + +void Bu::UnitSuite::ReportConsole::suiteEnded() +{ + sio << sio.nl + << "Report:" << sio.nl + << "\tTotal tests run: " << iTestCount << sio.nl; + + if( iEPass > 0 ) + sio << "\tExpected passes: " << iEPass << sio.nl; + if( iEFail > 0 ) + sio << "\tExpected failures: " << iEFail << sio.nl; + if( iUPass > 0 ) + sio << "\tUnexpected passes: " << iUPass << sio.nl; + if( iUFail > 0 ) + sio << "\tUnexpected failures: " << iUFail << sio.nl; + sio << sio.nl; + if( iUPass == 0 && iUFail == 0 ) + sio << "\tNothing unexpected." << sio.nl << sio.nl; +} + + +///////// +// Bu::UnitSuite::Report +//// + +Bu::UnitSuite::ReportJson::ReportJson() : + pReport( new Bu::Json( Bu::Json::Object ) ) +{ +} + +Bu::UnitSuite::ReportJson::~ReportJson() +{ + delete pReport; +} + +void Bu::UnitSuite::ReportJson::suiteStarting( const UnitSuite &rSuite, + const TestList & /*lTests*/ ) +{ + pReport->insert("type", "report"); + pReport->insert("suite", rSuite.getName() ); + pReport->insertArray("tests"); +} + +void Bu::UnitSuite::ReportJson::testStarting( const TestInfo & ) +{ +} + +void Bu::UnitSuite::ReportJson::updateProgress( int, int ) +{ +} + +void Bu::UnitSuite::ReportJson::testEnded( const TestInfo &rTest ) +{ + Bu::Json &rOb = (*pReport)["tests"].appendObject(); + rOb.insert("name", rTest.sName ); + rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) ); + rOb.insert("result", "pass"); +} + +void Bu::UnitSuite::ReportJson::testEnded( const TestInfo &rTest, + const Bu::UnitSuite::Failed &rFail ) +{ + Bu::Json &rOb = (*pReport)["tests"].appendObject(); + rOb.insert("name", rTest.sName ); + rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) ); + rOb.insert("result", "fail"); + Bu::Json &rFailOb = rOb.insertObject("fail"); + rFailOb.insert("action", rFail.str ); + rFailOb.insert("file", rFail.sFile ); + rFailOb.insert("line", (double)rFail.nLine ); +} + +void Bu::UnitSuite::ReportJson::testException( const TestInfo &rTest, + std::exception &e ) +{ + Bu::Json &rOb = (*pReport)["tests"].appendObject(); + rOb.insert("name", rTest.sName ); + rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) ); + rOb.insert("result", "fail"); + Bu::Json &rFail = rOb.insertObject("fail"); + rFail.insert("what", e.what() ); +} + +void Bu::UnitSuite::ReportJson::suiteEnded() +{ + Bu::println("%1").arg( pReport->toString() ); +} + diff --git a/src/stable/unitsuite.h b/src/stable/unitsuite.h index ea5e389..b1e2953 100644 --- a/src/stable/unitsuite.h +++ b/src/stable/unitsuite.h @@ -17,6 +17,7 @@ namespace Bu { + class Json; /** * Provides a unit testing framework. This is pretty easy to use, probably * the best way to get started is to use ch to generate a template, or use @@ -96,6 +97,8 @@ namespace Bu protected: void add( Test fTest, const Bu::String &sName, Expect e=expectPass ); void setName( const String &sName ); + String getName() const; + virtual void cleanup(); void dispProgress(); void setStepCount( int iSteps ); @@ -104,6 +107,7 @@ namespace Bu private: int onListCases( Bu::Array aParam ); + int onPrintName( Bu::Array aParam ); int onAddTest( Bu::Array aParam ); private: @@ -128,6 +132,75 @@ namespace Bu time_t tLastUpdate; Bu::Hash hSelTests; + + public: + class Report + { + public: + Report(); + virtual ~Report(); + + virtual void suiteStarting( const UnitSuite &rSuite, + const TestList &lTests )=0; + virtual void testStarting( const TestInfo &rTest )=0; + virtual void updateProgress( int iProgress, int iStepCount )=0; + virtual void testEnded( const TestInfo &rTest )=0; + virtual void testEnded( const TestInfo &rTest, + const Failed &rFail )=0; + virtual void testException( const TestInfo &rTest, + std::exception &e )=0; + virtual void suiteEnded()=0; + }; + + class ReportConsole : public Report + { + public: + ReportConsole(); + virtual ~ReportConsole(); + + virtual void suiteStarting( const UnitSuite &rSuite, + const TestList &lTests ); + virtual void testStarting( const TestInfo &rTest ); + virtual void updateProgress( int iProgress, int iStepCount ); + virtual void testEnded( const TestInfo &rTest ); + virtual void testEnded( const TestInfo &rTest, + const Failed &rFail ); + virtual void testException( const TestInfo &rTest, + std::exception &e ); + virtual void suiteEnded(); + + private: + int iTestCount; + int iNameWidth; + int iEPass; + int iEFail; + int iUPass; + int iUFail; + }; + + class ReportJson : public Report + { + public: + ReportJson(); + virtual ~ReportJson(); + + virtual void suiteStarting( const UnitSuite &rSuite, + const TestList &lTests ); + virtual void testStarting( const TestInfo &rTest ); + virtual void updateProgress( int iProgress, int iStepCount ); + virtual void testEnded( const TestInfo &rTest ); + virtual void testEnded( const TestInfo &rTest, + const Failed &rFail ); + virtual void testException( const TestInfo &rTest, + std::exception &e ); + virtual void suiteEnded(); + + private: + class Bu::Json *pReport; + }; + + private: + Report *pReport; }; Bu::Formatter &operator<<( Bu::Formatter &f, const Bu::UnitSuite::Expect &e ); -- cgit v1.2.3