Unit test augmentations and harness.

Added some features to the mkunit program, including cleanup routine
support. Added reporting modes for the UnitSuite class, and it can now
generate machine readable reports. Added a new program, rununits that
runs all unit tests and generates a synopsis of what you really care
about at the end, issues!

2 files changed, 328 insertions, 82 deletions

diff --git a/src/stable/unitsuite.cpp b/src/stable/unitsuite.cpp
index a38c77a..b2544f2 100644
--- a/src/stable/unitsuite.cpp
+++ b/src/stable/unitsuite.cpp
@@ -9,6 +9,7 @@
 #include "bu/file.h"
 #include "bu/sio.h"
 #include "bu/optparser.h"
+#include "bu/json.h"
 #include <stdlib.h>
 #include <time.h>
 
@@ -18,28 +19,33 @@ using namespace Bu;
 
 Bu::UnitSuite::UnitSuite() :
     iOptions( 0 ),
-    iNameWidth( 0 )
+    pReport( NULL )
 {
 }
 
 Bu::UnitSuite::UnitSuite( int iOptions ) :
-    iOptions( iOptions ),
-    iNameWidth( 0 )
+    iOptions( iOptions )
 {
 }
 
 Bu::UnitSuite::~UnitSuite()
 {
+    delete pReport;
 }
 
 int Bu::UnitSuite::run( int argc, char *argv[] )
 {
     bool bCleanup = true;
+    bool bInterop = false;
     OptParser p;
     p.addOption( Bu::slot( this, &Bu::UnitSuite::onListCases ), 'l', "list",
             "List available test cases." );
+    p.addOption( Bu::slot( this, &Bu::UnitSuite::onPrintName ), "print-name",
+            "Print the internal name of this test suite.");
     p.addOption( bCleanup, "no-cleanup", "Don't erase temp files.");
     p.setOverride( "no-cleanup", false );
+    p.addOption( bInterop, "interop", "Output machine parsable json.");
+    p.setOverride( "interop", true );
     p.setNonOption( Bu::slot( this, &Bu::UnitSuite::onAddTest ) );
     p.addHelpOption();
     p.parse( argc, argv );
@@ -56,108 +62,58 @@ int Bu::UnitSuite::run( int argc, char *argv[] )
         lTests = lSub;
     }
 
-    int iEPass = 0;
-    int iEFail = 0;
-    int iUPass = 0;
-    int iUFail = 0;
+    if( bInterop )
+        pReport = new ReportJson();
+    else
+        pReport = new ReportConsole();
+    pReport->suiteStarting( *this, lTests );
+
     for( TestList::iterator i = lTests.begin(); i != lTests.end(); i++ )
     {
-        sio << Fmt( iNameWidth+3, Fmt::Left ).fill('.') << i->sName
-            << sio.flush;
         try
         {
             iStepCount = -1;
             iProgress = 0;
+            if( pReport )
+                pReport->testStarting( *i );
             (this->*(i->fTest))();
-            switch( i->eExpect )
-            {
-                case expectPass:
-                    sio << "pass." << sio.nl;
-                    iEPass++;
-                    break;
-
-                case expectFail:
-                    sio << "unexpected pass." << sio.nl;
-                    iUPass++;
-                    break;
-            }
+            if( pReport )
+                pReport->testEnded( *i );
         }
         catch( Failed &e )
         {
-            switch( i->eExpect )
-            {
-                case expectPass:
-                    sio << "unexpected ";
-                    iUFail++;
-                    break;
-
-                case expectFail:
-                    sio << "expected ";
-                    iEFail++;
-                    break;
-            }
-            if( e.bFile )
-            {
-                sio << "fail in unitTest(" << e.str << "). (" << e.sFile
-                    << ":" << e.nLine << ")." << sio.nl;
-            }
-            else
-            {
-                sio << "fail in unitTest(" << e.str << ")." << sio.nl;
-            }
+            if( pReport )
+                pReport->testEnded( *i, e );
 
             if( (iOptions & optStopOnError) )
+            {
                 return 0;
+            }
         }
         catch( std::exception &e )
         {
-            switch( i->eExpect )
-            {
-                case expectPass:
-                    sio << "unexpected ";
-                    iUFail++;
-                    break;
-
-                case expectFail:
-                    sio << "expected ";
-                    iEFail++;
-                    break;
-            }
-            sio << "fail with unknown exception.  what: " << e.what() << sio.nl;
+            if( pReport )
+                pReport->testException( *i, e );
 
             if( (iOptions & optStopOnError) )
+            {
                 return 0;
+            }
         }
         catch( ... )
         {
-            switch( i->eExpect )
-            {
-                case expectPass:
-                    sio << "unexpected ";
-                    iUFail++;
-                    break;
-
-                case expectFail:
-                    sio << "expected ";
-                    iEFail++;
-                    break;
-            }
             sio << "fail with external exception." << sio.nl;
+            return -1;
 
             if( (iOptions & optStopOnError) )
+            {
                 return 0;
+            }
         }
     }
 
-    sio << sio.nl
-        << "Report:" << sio.nl
-        << "\tTotal tests run:     " << lTests.getSize() << sio.nl
-        << "\tExpected passes:     " << iEPass << sio.nl
-        << "\tExpected failures:   " << iEFail << sio.nl
-        << "\tUnexpected passes:   " << iUPass << sio.nl
-        << "\tUnexpected failures: " << iUFail << sio.nl << sio.nl;
-    if( iUPass == 0 && iUFail == 0 )
-        sio << "\tNothing unexpected." << sio.nl << sio.nl;
+    if( pReport )
+        pReport->suiteEnded();
 
     if( bCleanup )
     {
@@ -165,6 +121,8 @@ int Bu::UnitSuite::run( int argc, char *argv[] )
         {
             unlink( (*i).getStr() );
         }
+
+        cleanup();
     }
 
     return 0;
@@ -191,8 +149,6 @@ void Bu::UnitSuite::add( Test fTest, const Bu::String &sName, Expect e )
     }
     ti.fTest = fTest;
     lTests.append( ti );
-    if( iNameWidth < ti.sName.getSize() )
-        iNameWidth = ti.sName.getSize();
 }
 
 void Bu::UnitSuite::setName( const String &sName )
@@ -200,12 +156,21 @@ void Bu::UnitSuite::setName( const String &sName )
     sSuiteName = sName;
 }
 
+Bu::String Bu::UnitSuite::getName() const
+{
+    return sSuiteName;
+}
+
+void Bu::UnitSuite::cleanup()
+{
+}
+
 void Bu::UnitSuite::dispProgress()
 {
     if( tLastUpdate == time( NULL ) )
         return;
-    sio << Fmt(3) << (iProgress*100/iStepCount) << "%" << "\b\b\b\b"
-        << sio.flush;
+    if( pReport )
+        pReport->updateProgress( iProgress, iStepCount );
     tLastUpdate = time( NULL );
 }
 
@@ -251,6 +216,13 @@ int Bu::UnitSuite::onListCases( StrArray )
     return 0;
 }
 
+int Bu::UnitSuite::onPrintName( StrArray )
+{
+    Bu::println("%1").arg( sSuiteName );
+    exit( 0 );
+    return 0;
+}
+
 int Bu::UnitSuite::onAddTest( StrArray aParam )
 {
     hSelTests.insert( aParam[0], true );
@@ -262,12 +234,213 @@ Bu::Formatter &Bu::operator<<( Bu::Formatter &f, const Bu::UnitSuite::Expect &e
     switch( e )
     {
         case Bu::UnitSuite::expectPass:
-            return f << "expect pass";
+            return f << "pass";
 
         case Bu::UnitSuite::expectFail:
-            return f << "expect fail";
+            return f << "fail";
     }
 
     return f << "**error**";
 }
 
+/////////
+// Bu::UnitSuite::Report
+////
+
+Bu::UnitSuite::Report::Report()
+{
+}
+
+Bu::UnitSuite::Report::~Report()
+{
+}
+
+/////////
+// Bu::UnitSuite::ReportConsole
+////
+
+Bu::UnitSuite::ReportConsole::ReportConsole() :
+    iTestCount( 0 ),
+    iNameWidth( 0 ),
+    iEPass( 0 ),
+    iEFail( 0 ),
+    iUPass( 0 ),
+    iUFail( 0 )
+{
+}
+
+Bu::UnitSuite::ReportConsole::~ReportConsole()
+{
+}
+
+void Bu::UnitSuite::ReportConsole::suiteStarting( const UnitSuite & /*rSuite*/,
+        const TestList &lTests )
+{
+    iTestCount = lTests.getSize();
+    for( TestList::const_iterator i = lTests.begin(); i; i++ )
+    {
+        if( iNameWidth < i->sName.getSize() )
+            iNameWidth = i->sName.getSize();
+    }
+}
+
+void Bu::UnitSuite::ReportConsole::testStarting( const TestInfo &rTest )
+{
+    sio << Fmt( iNameWidth+3, Fmt::Left ).fill('.') << rTest.sName
+        << sio.flush;
+}
+
+void Bu::UnitSuite::ReportConsole::updateProgress( int iProgress,
+        int iStepCount )
+{
+    sio << Fmt(3) << (iProgress*100/iStepCount) << "%" << "\b\b\b\b"
+        << sio.flush;
+}
+
+void Bu::UnitSuite::ReportConsole::testEnded( const TestInfo &rTest )
+{
+    switch( rTest.eExpect )
+    {
+        case expectPass:
+            sio << "pass." << sio.nl;
+            iEPass++;
+            break;
+
+        case expectFail:
+            sio << "unexpected pass." << sio.nl;
+            iUPass++;
+            break;
+    }
+}
+
+void Bu::UnitSuite::ReportConsole::testEnded( const TestInfo &rTest,
+        const Bu::UnitSuite::Failed &rFail )
+{
+    switch( rTest.eExpect )
+    {
+        case expectPass:
+            sio << "unexpected ";
+            iUFail++;
+            break;
+
+        case expectFail:
+            sio << "expected ";
+            iEFail++;
+            break;
+    }
+    if( rFail.bFile )
+    {
+        sio << "fail in unitTest(" << rFail.str << "). (" << rFail.sFile
+            << ":" << rFail.nLine << ")." << sio.nl;
+    }
+    else
+    {
+        sio << "fail in unitTest(" << rFail.str << ")." << sio.nl;
+    }
+}
+
+void Bu::UnitSuite::ReportConsole::testException( const TestInfo &rTest,
+        std::exception &e )
+{
+    switch( rTest.eExpect )
+    {
+        case expectPass:
+            sio << "unexpected ";
+            iUFail++;
+            break;
+
+        case expectFail:
+            sio << "expected ";
+            iEFail++;
+            break;
+    }
+    sio << "fail with unknown exception.  what: " << e.what() << sio.nl;
+}
+
+void Bu::UnitSuite::ReportConsole::suiteEnded()
+{
+    sio << sio.nl
+        << "Report:" << sio.nl
+        << "\tTotal tests run:     " << iTestCount << sio.nl;
+
+    if( iEPass > 0 )
+        sio << "\tExpected passes:     " << iEPass << sio.nl;
+    if( iEFail > 0 )
+        sio << "\tExpected failures:   " << iEFail << sio.nl;
+    if( iUPass > 0 )
+        sio << "\tUnexpected passes:   " << iUPass << sio.nl;
+    if( iUFail > 0 )
+        sio << "\tUnexpected failures: " << iUFail << sio.nl;
+    sio << sio.nl;
+    if( iUPass == 0 && iUFail == 0 )
+        sio << "\tNothing unexpected." << sio.nl << sio.nl;
+}
+
+
+/////////
+// Bu::UnitSuite::Report
+////
+
+Bu::UnitSuite::ReportJson::ReportJson() :
+    pReport( new Bu::Json( Bu::Json::Object ) )
+{
+}
+
+Bu::UnitSuite::ReportJson::~ReportJson()
+{
+    delete pReport;
+}
+
+void Bu::UnitSuite::ReportJson::suiteStarting( const UnitSuite &rSuite,
+        const TestList & /*lTests*/ )
+{
+    pReport->insert("type", "report");
+    pReport->insert("suite", rSuite.getName() );
+    pReport->insertArray("tests");
+}
+
+void Bu::UnitSuite::ReportJson::testStarting( const TestInfo & )
+{
+}
+
+void Bu::UnitSuite::ReportJson::updateProgress( int, int )
+{
+}
+
+void Bu::UnitSuite::ReportJson::testEnded( const TestInfo &rTest )
+{
+    Bu::Json &rOb = (*pReport)["tests"].appendObject();
+    rOb.insert("name", rTest.sName );
+    rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) );
+    rOb.insert("result", "pass");
+}
+
+void Bu::UnitSuite::ReportJson::testEnded( const TestInfo &rTest,
+        const Bu::UnitSuite::Failed &rFail )
+{
+    Bu::Json &rOb = (*pReport)["tests"].appendObject();
+    rOb.insert("name", rTest.sName );
+    rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) );
+    rOb.insert("result", "fail");
+    Bu::Json &rFailOb = rOb.insertObject("fail");
+    rFailOb.insert("action", rFail.str );
+    rFailOb.insert("file", rFail.sFile );
+    rFailOb.insert("line", (double)rFail.nLine );
+}
+
+void Bu::UnitSuite::ReportJson::testException( const TestInfo &rTest,
+        std::exception &e )
+{
+    Bu::Json &rOb = (*pReport)["tests"].appendObject();
+    rOb.insert("name", rTest.sName );
+    rOb.insert("expected", Bu::String("%1").arg( rTest.eExpect ) );
+    rOb.insert("result", "fail");
+    Bu::Json &rFail = rOb.insertObject("fail");
+    rFail.insert("what", e.what() );
+}
+
+void Bu::UnitSuite::ReportJson::suiteEnded()
+{
+    Bu::println("%1").arg( pReport->toString() );
+}
+
diff --git a/src/stable/unitsuite.h b/src/stable/unitsuite.h
index ea5e389..b1e2953 100644
--- a/src/stable/unitsuite.h
+++ b/src/stable/unitsuite.h
@@ -17,6 +17,7 @@
 
 namespace Bu
 {
+    class Json;
     /**
      * Provides a unit testing framework.  This is pretty easy to use, probably
      * the best way to get started is to use ch to generate a template, or use
@@ -96,6 +97,8 @@ namespace Bu
     protected:
         void add( Test fTest, const Bu::String &sName, Expect e=expectPass );
         void setName( const String &sName );
+        String getName() const;
+        virtual void cleanup();
 
         void dispProgress();
         void setStepCount( int iSteps );
@@ -104,6 +107,7 @@ namespace Bu
 
     private:
         int onListCases( Bu::Array<Bu::String> aParam );
+        int onPrintName( Bu::Array<Bu::String> aParam );
         int onAddTest( Bu::Array<Bu::String> aParam );
 
     private:
@@ -128,6 +132,75 @@ namespace Bu
         time_t tLastUpdate;
 
         Bu::Hash<Bu::String, bool> hSelTests;
+
+    public:
+        class Report
+        {
+        public:
+            Report();
+            virtual ~Report();
+
+            virtual void suiteStarting( const UnitSuite &rSuite,
+                    const TestList &lTests )=0;
+            virtual void testStarting( const TestInfo &rTest )=0;
+            virtual void updateProgress( int iProgress, int iStepCount )=0;
+            virtual void testEnded( const TestInfo &rTest )=0;
+            virtual void testEnded( const TestInfo &rTest,
+                    const Failed &rFail )=0;
+            virtual void testException( const TestInfo &rTest,
+                    std::exception &e )=0;
+            virtual void suiteEnded()=0;
+        };
+
+        class ReportConsole : public Report
+        {
+        public:
+            ReportConsole();
+            virtual ~ReportConsole();
+
+            virtual void suiteStarting( const UnitSuite &rSuite,
+                    const TestList &lTests );
+            virtual void testStarting( const TestInfo &rTest );
+            virtual void updateProgress( int iProgress, int iStepCount ); 
+            virtual void testEnded( const TestInfo &rTest );
+            virtual void testEnded( const TestInfo &rTest,
+                    const Failed &rFail );
+            virtual void testException( const TestInfo &rTest,
+                    std::exception &e );
+            virtual void suiteEnded();
+
+        private:
+            int iTestCount;
+            int iNameWidth;
+            int iEPass;
+            int iEFail;
+            int iUPass;
+            int iUFail;
+        };
+
+        class ReportJson : public Report
+        {
+        public:
+            ReportJson();
+            virtual ~ReportJson();
+
+            virtual void suiteStarting( const UnitSuite &rSuite,
+                    const TestList &lTests );
+            virtual void testStarting( const TestInfo &rTest );
+            virtual void updateProgress( int iProgress, int iStepCount ); 
+            virtual void testEnded( const TestInfo &rTest );
+            virtual void testEnded( const TestInfo &rTest,
+                    const Failed &rFail );
+            virtual void testException( const TestInfo &rTest,
+                    std::exception &e );
+            virtual void suiteEnded();
+
+        private:
+            class Bu::Json *pReport;
+        };
+
+    private:
+        Report *pReport;
     };
 
 Bu::Formatter &operator<<( Bu::Formatter &f, const Bu::UnitSuite::Expect &e );