Myriad headers are the right size now.

That was weird. It seems to be working now.

3 files changed, 149 insertions, 45 deletions

diff --git a/src/stable/file.cpp b/src/stable/file.cpp
index 05e4af1..55766ea 100644
--- a/src/stable/file.cpp
+++ b/src/stable/file.cpp
@@ -142,6 +142,16 @@ bool Bu::File::isEos()
 
 bool Bu::File::canRead()
 {
+    return isReadable();
+}
+
+bool Bu::File::canWrite()
+{
+    return isWritable();
+}
+
+bool Bu::File::isReadable()
+{
 #ifdef WIN32
     return true;
 #else
@@ -152,7 +162,7 @@ bool Bu::File::canRead()
 #endif
 }
 
-bool Bu::File::canWrite()
+bool Bu::File::isWritable()
 {
 #ifdef WIN32
     return true;
@@ -164,16 +174,6 @@ bool Bu::File::canWrite()
 #endif
 }
 
-bool Bu::File::isReadable()
-{
-    return true;
-}
-
-bool Bu::File::isWritable()
-{
-    return true;
-}
-
 bool Bu::File::isSeekable()
 {
     return true;
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 44a990a..4edd004 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -15,6 +15,12 @@
         "Insufficent data reading myriad data from backing stream."); \
 } (void)0
 
+#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
+{ \
+    throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
+        "Insufficent data reading from myriad stream."); \
+} (void)0
+
 namespace Bu
 {
     subExceptionDef( MyriadException )
@@ -63,7 +69,9 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode,
     {
         pStream->aBlocks.append( __allocateBlock() );
     }
+    mhStream.lock();
     hStream.insert( pStream->iStream, pStream );
+    mhStream.unlock();
     bStructureChanged = true;
 
     return Bu::MyriadStream( *this, pStream, (Mode)(eMode&ReadWrite) );
@@ -72,7 +80,7 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode,
 Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
         Bu::Myriad::Mode eMode )
 {
-    Bu::MutexLocker l( mAccess );
+    Bu::MutexLocker l( mhStream );
     if( !hStream.has( iStream ) )
     {
         throw Bu::MyriadException( MyriadException::noSuchStream,
@@ -89,15 +97,19 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     return Bu::MyriadStream( *this, hStream.get( iStream ), eMode );
 }
 
-bool Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ )
+void Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ )
 {
-    return false;
 }
 
-bool Bu::Myriad::setSize( Bu::Myriad::StreamId /*iStream*/,
-        int32_t /*iNewSize*/ )
+void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
+        int32_t iNewSize )
 {
-    return false;
+    Stream *pStream;
+    {
+        Bu::MutexLocker l( mhStream );
+        pStream = hStream.get( iStream );
+    }
+    pStream->setSize( iNewSize );
 }
 
 Bu::String Bu::Myriad::getLocation() const
@@ -108,6 +120,72 @@ Bu::String Bu::Myriad::getLocation() const
         .arg( 1 ).arg( iBlockSize ).arg( rBacking.getLocation() );
 }
 
+int32_t Bu::Myriad::getBlockSize() const
+{
+    Bu::MutexLocker l( mAccess );
+    return iBlockSize;
+}
+
+int32_t Bu::Myriad::getTotalBlocks() const
+{
+    Bu::MutexLocker l( mAccess );
+    return iBlockCount;
+}
+
+int32_t Bu::Myriad::getUsedBlocks() const
+{
+    Bu::MutexLocker l( mAccess );
+    return iBlockCount-lFreeBlocks.getSize();
+}
+
+int32_t Bu::Myriad::getFreeBlocks() const
+{
+    Bu::MutexLocker l( mAccess );
+    return lFreeBlocks.getSize();
+}
+
+int32_t Bu::Myriad::getTotalStreams() const
+{
+    Bu::MutexLocker l( mhStream );
+    return hStream.getSize();
+}
+
+int32_t Bu::Myriad::getTotalUsedBytes() const
+{
+    Bu::MutexLocker l( mhStream );
+    int32_t iTotal = 0;
+    for( StreamHash::const_iterator i = hStream.begin(); i; i++ )
+    {
+        iTotal += i.getValue()->getSize();
+    }
+
+    return iTotal;
+}
+
+int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
+{
+    if( iAssumeBlockSize < 0 )
+    {
+        iAssumeBlockSize = getBlockSize();
+    }
+    int32_t iTotal = 0;
+    {
+        Bu::MutexLocker l( mhStream );
+        for( StreamHash::const_iterator i = hStream.begin(); i; i++ )
+        {
+            if( (i.getValue()->getSize()%iAssumeBlockSize) > 0 )
+                iTotal += iBlockSize-(i.getValue()->getSize()%iAssumeBlockSize);
+        }
+    }
+
+    {
+        Bu::MutexLocker l( mAccess );
+        iTotal += lFreeBlocks.getSize()*iBlockSize;
+    }
+
+    return iTotal;
+}
+
 bool Bu::Myriad::loadMyriad()
 {
     Bu::println("Load myriad!");
@@ -129,11 +207,25 @@ bool Bu::Myriad::loadMyriad()
     }
     MyriadRead( &uBitsPerInt, 1 );
     if( uBitsPerInt != 32 )
-{
+    {
         throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
                 "Only 32 bits per int are supported at this time.");
     }
     MyriadRead( &iBlockSize, 4 );
+
+    iBlockCount = rBacking.getSize()/iBlockSize;
+    if( (rBacking.getSize()%iBlockSize) != 0 )
+    {
+        throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
+                "Backing stream is not cleanly divisibly by the block size.");
+    }
+
+    Bu::Hash<int32_t,bool> hUnusedBlocks;
+    for( int32_t j = 0; j < iBlockCount; j++ )
+    {
+        hUnusedBlocks.insert( j, true );
+    }
+
     int iStreamCount;
     MyriadRead( &iStreamCount, 4 );
 
@@ -162,6 +254,7 @@ bool Bu::Myriad::loadMyriad()
     {
         int32_t iBlockIndex;
         MyriadRead( &iBlockIndex, 4 );
+        hUnusedBlocks.erase( iBlockIndex );
         pHeaderStream->aBlocks.append( iBlockIndex );
         if( rBacking.tell()+4 <= (j+1)*iBlockSize )
             break;
@@ -172,7 +265,8 @@ bool Bu::Myriad::loadMyriad()
     while( pHeaderStream->aBlocks.getSize() < iHeaderStreamBlocks )
     {
         int32_t iBlockIndex;
-        sHeader.read( &iBlockIndex, 4 );
+        ReqRead( sHeader, &iBlockIndex, 4 );
+        hUnusedBlocks.erase( iBlockIndex );
         pHeaderStream->aBlocks.append( iBlockIndex );
     }
 
@@ -180,14 +274,15 @@ bool Bu::Myriad::loadMyriad()
     for( int j = 1; j < iStreamCount; j++ )
     {
         int32_t iStreamBytes;
-        sHeader.read( &iStream, 4 );
-        sHeader.read( &iStreamBytes, 4 );
+        ReqRead( sHeader, &iStream, 4 );
+        ReqRead( sHeader, &iStreamBytes, 4 );
         Stream *pStream = new Stream( *this, iStream, iStreamBytes );
         int32_t iBlocks = blkDiv(iStreamBytes, iBlockSize );
         for( int k = 0; k < iBlocks; k++ )
         {
             int32_t iBlockIndex;
-            sHeader.read( &iBlockIndex, 4 );
+            ReqRead( sHeader, &iBlockIndex, 4 );
+            hUnusedBlocks.erase( iBlockIndex );
             pStream->aBlocks.append( iBlockIndex );
         }
         hStream.insert( iStream, pStream );
@@ -195,6 +290,9 @@ bool Bu::Myriad::loadMyriad()
             iLastUsedIndex = iStream;
     }
 
+    lFreeBlocks = hUnusedBlocks.getKeys();
+    Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
+
     return true;
 }
 
@@ -211,23 +309,6 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
                 "Backing stream contains data, but not a myriad structure.");
     }
 
-/*
-    struct {
-        char sMagicCode[4];
-        uint8_t uVer;
-        uint8_t uBitsPerInt;
-        uint32_t uBlockSize;
-        uint32_t uStreamCount;
-    } sHeader;
-
-    struct {
-        uint32_t uStreamId;
-        uint32_t uStreamSize;
-    } sStreamHeader;
-    
-    Bu::println("sHeader = %1, sStreamHeader = %2").arg( sizeof(sHeader) ).arg( sizeof(sStreamHeader) );
-*/
-
     // Start with the bytes for the file header and initial stream header
     int iHeaderStreamBytes
         = 14 // Base header
@@ -299,6 +380,7 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
         lFreeBlocks.append( j );
     }
     iLastUsedIndex = 0;
+    iBlockCount = iPreallocateBlocks;
 }
 
 void Bu::Myriad::writeHeader()
@@ -309,6 +391,7 @@ void Bu::Myriad::writeHeader()
     Bu::MemBuf mbHeader;
     {
         Bu::MutexLocker l( mAccess );
+        Bu::MutexLocker l2( mhStream );
 
         int32_t iHdrStreamSize = __calcHeaderSize();
         // Maybe just do stream surgery here.
@@ -401,12 +484,15 @@ int32_t Bu::Myriad::__calcHeaderSize()
         }
     }
 
+    Bu::println("HeaderCalc:");
+    Bu::println("  Base (no header stream): %1").arg( iHdrSize );
     int32_t iNewSize = iHdrSize;
     int32_t iOldSize;
 
     do {
         iOldSize = iNewSize;
-        iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize);
+        iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4;
+        Bu::println("  Recomp: %1").arg( iNewSize );
     } while( iOldSize != iNewSize );
 
     return iNewSize;
@@ -467,13 +553,13 @@ void Bu::Myriad::blankBlock( int32_t iBlockId )
 
 void Bu::Myriad::openStream( StreamId id )
 {
-    Bu::MutexLocker l( mAccess );
+    Bu::MutexLocker l( mhStream );
     hStream.get( id )->open();
 }
 
 void Bu::Myriad::closeStream( StreamId id )
 {
-    Bu::MutexLocker l( mAccess );
+    Bu::MutexLocker l( mhStream );
     hStream.get( id )->close();
 }
 
@@ -579,6 +665,17 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
 {
     int32_t iRead = 0;
     Bu::MutexLocker l( mAccess );
+
+    if( iStart >= this->iSize )
+        return 0;
+
+    if( iStart+iSize >= this->iSize )
+    {
+        int32_t iDiff = this->iSize-(iStart+iSize);
+        iSize -= iDiff;
+        iStart += iDiff;
+    }
+
     while( iSize > 0 )
     {
         int32_t iBlock = aBlocks[iStart/rParent.iBlockSize];
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 7cf6041..58f700a 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -73,9 +73,16 @@ namespace Bu
 
         MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
         MyriadStream open( StreamId iStream, Mode eMode );
-        bool erase( StreamId iStream );
-        bool setSize( StreamId iStream, int32_t iNewSize );
+        void erase( StreamId iStream );
+        void setSize( StreamId iStream, int32_t iNewSize );
         Bu::String getLocation() const;
+        int32_t getBlockSize() const;
+        int32_t getTotalBlocks() const;
+        int32_t getUsedBlocks() const;
+        int32_t getFreeBlocks() const;
+        int32_t getTotalStreams() const;
+        int32_t getTotalUsedBytes() const;
+        int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
 
     private:
         bool loadMyriad();
@@ -151,7 +158,6 @@ namespace Bu
             int32_t iSize;
             Bu::Array<int32_t> aBlocks;
             int32_t iOpenCount;
-            bool bStructureChanged;
         };
 
     private:
@@ -164,6 +170,7 @@ namespace Bu
         int32_t iBlockCount;
         bool bIsNewStream;
         bool bStructureChanged;
+        mutable Bu::Mutex mhStream;
         StreamHash hStream;
         IndexList lFreeBlocks;
         StreamId iLastUsedIndex;