More myriad fixes, it passes all existing unit tests.

3 files changed, 421 insertions, 13 deletions

diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 85daa0d..f9fcd3e 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -12,13 +12,13 @@
 #define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
 { \
     throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
-        "Insufficent data reading myriad data from backing stream."); \
+        "Insufficient data reading myriad data from backing stream."); \
 } (void)0
 
 #define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
 { \
     throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
-        "Insufficent data reading from myriad stream."); \
+        "Insufficient data reading from myriad stream."); \
 } (void)0
 
 namespace Bu
@@ -112,7 +112,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     }
     {
         Bu::MutexLocker l2( mBacking );
-        if( (eMode&Write) && rBacking.isWritable() )
+        if( (eMode&Write) && !rBacking.isWritable() )
         {
             throw Bu::MyriadException( MyriadException::badMode,
                 "Backing stream does not support writing.");
@@ -398,6 +398,8 @@ bool Bu::Myriad::loadMyriad()
 
     lFreeBlocks = hUnusedBlocks.getKeys();
     //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
+    
+    bIsNewStream = false;
 
     return true;
 }
@@ -434,14 +436,11 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
         //Bu::println("      Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
     }
 
-    if( iPreallocateBlocks > iHeaderStreamBlocks )
+    if( iPreallocateBlocks < iHeaderStreamBlocks )
     {
-        rBacking.setSize( iBlockSize*iPreallocateBlocks );
-    }
-    else
-    {
-        rBacking.setSize( iBlockSize*iHeaderStreamBlocks );
+        iPreallocateBlocks = iHeaderStreamBlocks;
     }
+    rBacking.setSize( iBlockSize*iPreallocateBlocks );
 
     //
     // Write Myriad header
@@ -540,14 +539,15 @@ void Bu::Myriad::writeHeader()
             uint32_t uStreamSize = pStream->getSize();
             mbHeader.write( &uStreamId, 4 );
             mbHeader.write( &uStreamSize, 4 );
+            int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize );
             Bu::Array<int32_t> aBlocks = pStream->getBlockList();
             
             //Bu::println("  Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
 
-            for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
+//            for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
+            for( int j = 0; j < iBlocks; j++ )
             {
-                int32_t iIdx = *i;
-                mbHeader.write( &iIdx, 4 );
+                mbHeader.write( &aBlocks[j], 4 );
             }
         }
 
@@ -768,7 +768,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
 
     if( iStart+iSize >= this->iSize )
     {
-        int32_t iDiff = this->iSize-(iStart+iSize);
+        int32_t iDiff = (iStart+iSize)-this->iSize;
         iSize -= iDiff;
         iStart += iDiff;
     }
diff --git a/src/tests/myriad.cpp b/src/tests/myriad.cpp
new file mode 100644
index 0000000..ee4eac4
--- /dev/null
+++ b/src/tests/myriad.cpp
@@ -0,0 +1,23 @@
+#include <bu/myriad.h>
+#include <bu/myriadstream.h>
+#include <bu/string.h>
+#include <bu/sio.h>
+#include <bu/file.h>
+
+int main( int , char *[] )
+{
+    Bu::File fMyriad("test.myr", Bu::File::WriteNew|Bu::File::Read );
+    Bu::Myriad m( fMyriad, 32 );
+
+    Bu::MyriadStream ms = m.create( Bu::Myriad::ReadWrite );
+    ms.setSize( 150 );
+    ms.setPos( 145 );
+    char stuff[10];
+    int32_t iRead = ms.read( stuff, 10 );
+
+    Bu::println("Tried to read 10, expect 5, got %1").arg( iRead );
+
+
+    return 0;
+}
+
diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit
new file mode 100644
index 0000000..f7bea97
--- /dev/null
+++ b/src/unit/myriad.unit
@@ -0,0 +1,385 @@
+// vim: syntax=cpp
+/*
+ * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/string.h"
+#include "bu/file.h"
+#include "bu/myriad.h"
+#include "bu/myriadstream.h"
+#include "bu/array.h"
+
+#include "bu/sio.h"
+#include "bu/archive.h"
+#include "bu/md5.h"
+#include "bu/unitsuite.h"
+
+#include <stdlib.h>
+
+using namespace Bu;
+
+class VerifyObject
+{
+friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo );
+friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo );
+public:
+    VerifyObject( int iUnits ) :
+        iUnits( iUnits ),
+        iBytesWritten( 0 )
+    {
+    }
+
+    virtual ~VerifyObject()
+    {
+    }
+
+    int getBytesWritten()
+    {
+        return iBytesWritten;
+    }
+
+private:
+    int iUnits;
+    mutable int iBytesWritten;
+};
+
+Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo )
+{
+    Md5 sum;
+    ar << vo.iUnits;
+    vo.iBytesWritten = sizeof(int);
+    sum.addData( &vo.iUnits, sizeof(int) );
+    for( int j = 0; j < vo.iUnits; j++ )
+    {
+        int iRand = random()%128;
+//      ar << iRand;
+        Bu::String sDat( iRand );
+        for( int j = 0; j < iRand; j++ )
+            sDat[j] = (char)((uint8_t)(random()%256));
+        ar << sDat;
+        sum.addData( &iRand, sizeof(int) );
+        sum.addData( sDat.getStr(), iRand );
+        vo.iBytesWritten += sizeof(long) + iRand;
+    }
+    Bu::String sRes = sum.getResult();
+    ar << sRes;
+    vo.iBytesWritten += sizeof(long) + sRes.getSize();
+    return ar;
+}
+
+Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo )
+{
+    Md5 sum;
+    ar >> vo.iUnits;
+    sum.addData( &vo.iUnits, sizeof(int) );
+    for( int j = 0; j < vo.iUnits; j++ )
+    {
+        int iRand;
+//      ar >> iRand;
+        Bu::String sStr;
+        ar >> sStr;
+        iRand = sStr.getSize();
+        sum.addData( &iRand, sizeof(int) );
+        sum.addData( sStr.getStr(), iRand );
+    }
+    Bu::String sSum;
+    ar >> sSum;
+    unitTest( sSum == sum.getResult() );
+    int iTooMuch;
+    try
+    {
+        ar >> iTooMuch;
+        unitFailed("should have thrown an exception.");
+    }
+    catch( Bu::ExceptionBase &e )
+    {
+    }
+    return ar;
+}
+
+suite Myriad
+{
+    test setSize
+    {
+        String sFileName("myriad-XXXXXXX");
+
+        File fMyriad = tempFile( sFileName );
+        Myriad m( fMyriad, 32 );
+
+        MyriadStream ms = m.create( Myriad::ReadWrite );
+        ms.setSize( 150 );
+        ms.setPos( 145 );
+        char stuff[10];
+        unitTest( ms.read( stuff, 10 ) == 5 );
+
+        ms.setSize( 12 );
+        unitTest( ms.read( stuff, 10 ) == 0 );
+        unitTest( ms.write( "hello", 5 ) == 5 );
+        unitTest( ms.tell() == 17 );
+
+        ms.setSize( 500 );
+        unitTest( ms.tell() == 17 );
+    }
+
+    void addBlock( Stream &s, bool bAppend=true )
+    {
+        if( bAppend )
+            s.setPosEnd( 0 );
+        int iSize = (random()%1016)+8;
+        s.write( &iSize, 4 );
+        char *buf = new char[iSize-8];
+        for( int j = 0; j < iSize-8; j++ )
+        {
+            buf[j] = (j+iSize)%256;
+        }
+        if( random()%2 == 0 )
+        {
+            s.write( buf, iSize-8 );
+        }
+        else
+        {
+            for( int j = 0; j < iSize-8; )
+            {
+                int iAmnt = (random()%8)+1;
+                if( iAmnt+j > iSize-8 )
+                    iAmnt = iSize-8-j;
+                iAmnt = s.write( buf+j, iAmnt );
+                j += iAmnt;
+            }
+        }
+        delete[] buf;
+        iSize = ~iSize;
+        s.write( &iSize, 4 );
+    }
+
+    void verifyBlock( Stream &s )
+    {
+        int iSize, iInv;
+        if( s.read( &iSize, 4 ) == 0 )
+            return;
+        if( iSize < 8 || iSize > 1024 )
+            throw ExceptionBase("Read bad data, %d", iSize );
+        char *buf = new char[iSize-8];
+        if( s.read( buf, iSize-8 ) < (Bu::size)iSize-8 )
+        {
+            delete[] buf;
+            throw ExceptionBase("Block failed verify (insuffient block data).");
+        }
+        for( int j = 0; j < iSize-8; j++ )
+        {
+            if( buf[j] != (char)((j+iSize)%256) )
+            {
+                char b = buf[j];
+                delete[] buf;
+                throw ExceptionBase("Block failed computed data verify "
+                    "(%02X==%02X).", b, (char)((j+iSize)%256) );
+            }
+        }
+        delete[] buf;
+        if( s.read( &iInv, 4 ) < 4 )
+            throw ExceptionBase("Block failed verify (insufficient data).");
+        if( iInv != ~iSize )
+            throw ExceptionBase("Block failed inversion verify.");
+    }
+    
+    void verifyStream( Stream &s )
+    {
+        s.setPos( 0 );
+        while( !s.isEos() )
+            verifyBlock( s );
+    }
+
+    test stressGrow
+    {
+        String sFileName("myriad-XXXXXXX");
+
+        File fMyriad = tempFile( sFileName );
+        Myriad m( fMyriad, 64 );
+
+        Array<int> aStreams;
+        for( int j = 0; j < 5; j++ )
+        {
+            aStreams.append( m.create( Bu::Myriad::Read ).getId() );
+        }
+
+        srandom( 512 );
+
+        for( int j = 0; j < 2500; j++ )
+        {
+            switch( random()%5 )
+            {
+                case 0:
+                    aStreams.append( m.create( Bu::Myriad::Read ).getId() );
+                    break;
+
+                case 1:
+                    if( aStreams.getSize() > 0 )
+                    {
+                        int iStream = random()%aStreams.getSize();
+                        {
+                            MyriadStream ms = m.open( aStreams[iStream],  Myriad::Read );
+                            verifyStream( ms );
+                        }
+                        m.erase( aStreams[iStream] );
+                        Array<int>::iterator i = aStreams.begin();
+                        for( int k = 0; k < iStream; k++ )
+                            i++;
+                        aStreams.erase( i );
+                    }
+                    break;
+
+                default:
+                    if( aStreams.getSize() == 0 )
+                    {
+                        aStreams.append(
+                            m.create( Bu::Myriad::Read ).getId()
+                            );
+                    }
+                    {
+                        int iStream = random()%aStreams.getSize();
+                        MyriadStream ms = m.open( aStreams[iStream], Myriad::ReadWrite );
+                        addBlock( ms );
+                        verifyStream( ms );
+                    }
+                    break;
+            }
+        }
+
+        for( Array<int>::iterator i = aStreams.begin(); i; i++ )
+        {
+            MyriadStream ms = m.open( *i, Myriad::Read );
+            verifyStream( ms );
+        }
+    }
+    
+    test stressTruncate
+    {
+        String sFileName("myriad-XXXXXXX");
+
+        File fMyriad = tempFile( sFileName );
+        Myriad m( fMyriad, 128 );
+
+        Array<int> aStream;
+
+        for( int j = 0; j < 5; j++ )
+        {
+            aStream.append( m.create( Bu::Myriad::Read ).getId() );
+        }
+        
+        srandom( 1024 );
+
+        char b;
+        for( int iter = 0; iter < 2500; iter++ )
+        {
+            for( Array<int>::iterator i = aStream.begin(); i; i++ )
+            {
+                MyriadStream ms = m.open( *i, Myriad::ReadWrite );
+                addBlock( ms, false );
+                ms.setSize( ms.tell() );
+                unitTest( ms.read( &b, 1 ) == 0 );
+                ms.setPos( 0 );
+                verifyBlock( ms );
+                unitTest( ms.read( &b, 1 ) == 0 );
+            }
+        }
+    }
+    
+    test stressTruncate2
+    {
+        String sFileName("myriad-XXXXXXX");
+
+        Array<int> aStream;
+
+        setStepCount( 5*2500 + 5 );
+
+        {
+            File fMyriad = tempFile( sFileName );
+            Myriad m( fMyriad, 128 );
+
+            for( int j = 0; j < 5; j++ )
+            {
+                aStream.append( m.create( Bu::Myriad::Read ).getId() );
+                incProgress();
+            }
+        }
+        
+        srandom( 1024 );
+
+        char b;
+        for( int iter = 0; iter < 2500; iter++ )
+        {
+            File fMyriad( sFileName, File::ReadWrite );
+            Myriad m( fMyriad );
+            for( Array<int>::iterator i = aStream.begin(); i; i++ )
+            {
+                MyriadStream ms = m.open( *i, Myriad::ReadWrite );
+                addBlock( ms, false );
+                ms.setSize( ms.tell() );
+                unitTest( ms.read( &b, 1 ) == 0 );
+                ms.setPos( 0 );
+                verifyBlock( ms );
+                unitTest( ms.read( &b, 1 ) == 0 );
+                incProgress();
+            }
+        }
+    }
+
+    test stressArchive
+    {
+        String sFileName("myriad-XXXXXX");
+        Array<int> aStream;
+        
+        srandom( 2096 );
+
+        setStepCount( 15*250 + 15 );
+        
+        {
+            File fMyriad = tempFile( sFileName );
+            Myriad m( fMyriad, 1024 );
+
+            for( int j = 0; j < 15; j++ )
+            {
+                MyriadStream ms = m.create( Myriad::Write );
+                int iStream = ms.getId();
+                aStream.append( iStream );
+                VerifyObject vo( random()%1024 );
+                {
+                    Archive ar( ms, Archive::save );
+                    ar << vo;
+                    unitTest( ms.tell() == vo.getBytesWritten() );
+                    ms.setSize( ms.tell() );
+                }
+                unitTest( m.getSize( iStream ) == vo.getBytesWritten() );
+                incProgress();
+            }
+        }
+
+        for( int iter = 0; iter < 250; iter++ )
+        {
+            File fMyriad( sFileName, File::ReadWrite );
+            Myriad m( fMyriad );
+            for( Array<int>::iterator i = aStream.begin(); i; i++ )
+            {
+                VerifyObject vo( random()%1024 );
+                {
+                    MyriadStream ms = m.open( *i, Myriad::Read );
+                    Archive ar( ms, Archive::load );
+                    ar >> vo;
+                }
+                {
+                    MyriadStream ms = m.open( *i, Myriad::WriteNew );
+                    Archive ar( ms, Archive::save );
+                    ar << vo;
+                    unitTest( ms.tell() == vo.getBytesWritten() );
+                    ms.setSize( ms.tell() );
+                }
+                unitTest( m.getSize( *i ) == vo.getBytesWritten() );
+                incProgress();
+            }
+        }
+    }
+}
+