5 files changed, 214 insertions, 49 deletions

diff --git a/src/stable/mutex.h b/src/stable/mutex.h
index d9e8910..8034974 100644
--- a/src/stable/mutex.h
+++ b/src/stable/mutex.h
@@ -33,7 +33,7 @@ namespace Bu
          * wait for the mutex to unlock, the odds of which are usually farily
          * low at deconstruction time.
          */
-        ~Mutex();
+        virtual ~Mutex();
 
         /**
          * Lock the mutex.  This causes all future calls to lock on this
@@ -44,13 +44,13 @@ namespace Bu
          * computation within a locked section.  This can cause thread-deadlock
          * and your program may hang.
          */
-        int lock();
+        virtual int lock();
 
         /**
          * Unlock the mutex.  This allows the next thread that asked for a lock
          * to lock the mutex and continue with execution.
          */
-        int unlock();
+        virtual int unlock();
 
         /**
          * Try to lock the mutex.  This is the option to go with if you cannot
@@ -58,7 +58,7 @@ namespace Bu
          * will attempt to lock the mutex, if the mutex is already locked this
          * function returns immediately with an error code.
          */
-        int trylock();
+        virtual int trylock();
 
     protected:
         pthread_mutex_t mutex;  /**< The internal mutex reference. */
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 1bf2301..492676e 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -12,13 +12,13 @@
 #define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
 { \
     throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
-        "Insufficent data reading myriad data from backing stream."); \
+        "Insufficient data reading myriad data from backing stream."); \
 } (void)0
 
 #define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
 { \
     throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
-        "Insufficent data reading from myriad stream."); \
+        "Insufficient data reading from myriad stream."); \
 } (void)0
 
 namespace Bu
@@ -74,7 +74,7 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode,
     mhStream.unlock();
     bStructureChanged = true;
 
-    return Bu::MyriadStream( *this, pStream, (Mode)(eMode&ReadWrite) );
+    return Bu::MyriadStream( *this, pStream, eMode&ReadWrite );
 }
 
 Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
@@ -84,7 +84,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     Bu::MutexLocker l( mhStream );
     if( (eMode&Create) )
     {
-        if( !hStream.has( iStream ) )
+        if( hStream.has( iStream ) )
         {
             if( (eMode&Exclusive) )
             {
@@ -112,7 +112,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     }
     {
         Bu::MutexLocker l2( mBacking );
-        if( (eMode&Write) && rBacking.isWritable() )
+        if( (eMode&Write) && !rBacking.isWritable() )
         {
             throw Bu::MyriadException( MyriadException::badMode,
                 "Backing stream does not support writing.");
@@ -129,8 +129,43 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     return Bu::MyriadStream( *this, pStream, eMode );
 }
 
-void Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ )
+Bu::Myriad::StreamId Bu::Myriad::allocate()
 {
+    Bu::MutexLocker l( mAccess );
+
+    Stream *pStream = new Stream( *this, ++iLastUsedIndex, 0 );
+    mhStream.lock();
+    hStream.insert( pStream->iStream, pStream );
+    mhStream.unlock();
+    bStructureChanged = true;
+
+    return pStream->iStream;
+}
+
+void Bu::Myriad::erase( Bu::Myriad::StreamId iStream )
+{
+    // For now, let's prevent you from erasing a stream if it's open.
+    Bu::MutexLocker l( mhStream );
+    if( !hStream.has( iStream ) )
+    {
+        throw Bu::MyriadException( Bu::MyriadException::noSuchStream,
+                "No such stream exists.");
+    }
+    Stream *pStream = hStream.get( iStream );
+    Bu::MutexLocker sl( pStream->mAccess );
+    if( pStream->iOpenCount > 0 )
+    {
+        throw Bu::MyriadException( Bu::MyriadException::streamOpen,
+                "Cannot currently erase a stream while it is open.");
+    }
+
+    for( Bu::Array<int32_t>::iterator i = pStream->aBlocks.begin(); i; i++ )
+    {
+        releaseBlock( *i, false );
+    }
+    pStream->aBlocks.clear();
+    hStream.erase( iStream );
+    delete pStream;
 }
 
 void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
@@ -144,6 +179,18 @@ void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
     pStream->setSize( iNewSize );
 }
 
+int32_t Bu::Myriad::getSize( StreamId iStream ) const
+{
+    Bu::MutexLocker l( mhStream );
+    return hStream.get( iStream )->getSize();
+}
+
+bool Bu::Myriad::exists( StreamId iStream ) const
+{
+    Bu::MutexLocker l( mhStream );
+    return hStream.has( iStream );
+}
+
 Bu::String Bu::Myriad::getLocation() const
 {
     Bu::MutexLocker l( mAccess );
@@ -218,9 +265,62 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
     return iTotal;
 }
 
+Bu::Myriad::StreamIdList Bu::Myriad::getStreamList() const
+{
+    mhStream.lock();
+    StreamIdList lIds = hStream.getKeys();
+    mhStream.unlock();
+    lIds.sort();
+    if( lIds.first() == 0 )
+    {
+        lIds.eraseFirst();
+    }
+    return lIds;
+}
+
+Bu::BitString Bu::Myriad::buildBlockUseMap() const
+{
+    Bu::MutexLocker l( mAccess );
+    Bu::BitString bsMap( iBlockCount );
+    bsMap.fill();
+    for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
+    {
+        bsMap.setBit( *i, false );
+    }
+    return bsMap;
+}
+
+Bu::Myriad::StreamIdArray Bu::Myriad::buildBlockMap() const
+{
+    Bu::MutexLocker l( mAccess );
+    StreamIdArray bm( iBlockCount );
+    for( int j = 0; j < iBlockCount; j++ )
+    {
+        bm.append( -1 );
+    }
+    Bu::MutexLocker l2( mhStream );
+    for( StreamHash::const_iterator iStream = hStream.begin();
+         iStream; iStream++ )
+    {
+        int32_t iId = iStream.getKey();
+        Stream *pStream = iStream.getValue();
+        for( Bu::Array<int32_t>::const_iterator iBlock =
+             pStream->aBlocks.begin(); iBlock; iBlock++ )
+        {
+            bm[*iBlock] = iId;
+        }
+    }
+    return bm;
+}
+
+void Bu::Myriad::sync()
+{
+    writeHeader();
+}
+
 bool Bu::Myriad::loadMyriad()
 {
-    Bu::println("Load myriad!");
+    //Bu::println("Load myriad!");
     char sMagicCode[4];
     rBacking.setPos( 0 );
     MyriadRead( sMagicCode, 4 );
@@ -323,7 +423,9 @@ bool Bu::Myriad::loadMyriad()
     }
 
     lFreeBlocks = hUnusedBlocks.getKeys();
-    Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
+    //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
+    
+    bIsNewStream = false;
 
     return true;
 }
@@ -350,32 +452,21 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
     // plus one block index.
     int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
 
-    Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
-        .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
-        .arg( iHeaderStreamBlocks )
-        .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
-        .arg( iHeaderStreamBlocks*iBlockSize );
+    //Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
     while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
            > iHeaderStreamBlocks*iBlockSize )
     {
         iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize);
         if( iHeaderStreamBlocks > 100 )
             break;
-        Bu::println("      Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
-            .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
-             .arg( iHeaderStreamBlocks )
-            .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
-            .arg( iHeaderStreamBlocks*iBlockSize );
+        //Bu::println("      Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
     }
 
-    if( iPreallocateBlocks > iHeaderStreamBlocks )
+    if( iPreallocateBlocks < iHeaderStreamBlocks )
     {
-        rBacking.setSize( iBlockSize*iPreallocateBlocks );
-    }
-    else
-    {
-        rBacking.setSize( iBlockSize*iHeaderStreamBlocks );
+        iPreallocateBlocks = iHeaderStreamBlocks;
     }
+    rBacking.setSize( iBlockSize*iPreallocateBlocks );
 
     //
     // Write Myriad header
@@ -417,12 +508,12 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
 
 void Bu::Myriad::writeHeader()
 {
+    Bu::MutexLocker l( mAccess );
     if( !rBacking.isWritable() )
         return;
-    Bu::println("Writing stream breakdown:");
+    //Bu::println("Writing stream breakdown:");
     Bu::MemBuf mbHeader;
     {
-        Bu::MutexLocker l( mAccess );
         Bu::MutexLocker l2( mhStream );
 
         int32_t iHdrStreamSize = __calcHeaderSize();
@@ -449,7 +540,7 @@ void Bu::Myriad::writeHeader()
             psHeader->iSize = iHdrStreamSize;
         }
 
-        Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
+        //Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
         
         uint8_t uVer = 1;
         uint8_t uBpi = 32;
@@ -474,17 +565,15 @@ void Bu::Myriad::writeHeader()
             uint32_t uStreamSize = pStream->getSize();
             mbHeader.write( &uStreamId, 4 );
             mbHeader.write( &uStreamSize, 4 );
+            int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize );
             Bu::Array<int32_t> aBlocks = pStream->getBlockList();
             
-            Bu::println("  Stream %1 is %2 bytes %3 blocks (%4 blocks computed)")
-                .arg( *i ).arg( uStreamSize )
-                .arg( aBlocks.getSize() )
-                .arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
+            //Bu::println("  Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
 
-            for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
+//            for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
+            for( int j = 0; j < iBlocks; j++ )
             {
-                int32_t iIdx = *i;
-                mbHeader.write( &iIdx, 4 );
+                mbHeader.write( &aBlocks[j], 4 );
             }
         }
 
@@ -517,15 +606,15 @@ int32_t Bu::Myriad::__calcHeaderSize()
         }
     }
 
-    Bu::println("HeaderCalc:");
-    Bu::println("  Base (no header stream): %1").arg( iHdrSize );
+    //Bu::println("HeaderCalc:");
+    //Bu::println("  Base (no header stream): %1").arg( iHdrSize );
     int32_t iNewSize = iHdrSize;
     int32_t iOldSize;
 
     do {
         iOldSize = iNewSize;
         iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4;
-        Bu::println("  Recomp: %1").arg( iNewSize );
+        //Bu::println("  Recomp: %1").arg( iNewSize );
     } while( iOldSize != iNewSize );
 
     return iNewSize;
@@ -705,9 +794,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
 
     if( iStart+iSize >= this->iSize )
     {
-        int32_t iDiff = this->iSize-(iStart+iSize);
-        iSize -= iDiff;
-        iStart += iDiff;
+        iSize = this->iSize-iStart;
     }
 
     while( iSize > 0 )
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 58f700a..5accd1e 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -7,6 +7,8 @@
 #include "bu/array.h"
 #include "bu/hash.h"
 
+#include "bu/bitstring.h"
+
 namespace Bu
 {
     class MyriadStream;
@@ -25,14 +27,25 @@ namespace Bu
             invalidParameter,
             invalidBackingStream,
             badMode,
+            streamOpen,
         };
     subExceptionDeclEnd();
 
+    /**
+     * Myriad Stream Multiplexer. This is a system that allows you to store
+     * many streams within a single backing stream. This is great for databases,
+     * caching, etc. It's fairly lightweight, and allows all streams to grow
+     * dynamically using a block-allocation scheme.  This is used extensively
+     * by the caching system and MyriadFs as well as other systems within
+     * libbu++.
+     */
     class Myriad
     {
     public:
         typedef int32_t StreamId;
-        enum Mode {
+        typedef Bu::Array<StreamId> StreamIdArray;
+        typedef Bu::List<StreamId> StreamIdList;
+        enum Mode : int32_t {
             None        = 0x00,
 
             // Flags
@@ -42,7 +55,7 @@ namespace Bu
             Truncate    = 0x08, ///< Truncate file if it does exist
             Append      = 0x10, ///< Start writing at end of file
             //NonBlock    = 0x20, ///< Open file in non-blocking mode
-            Exclusive   = 0x44, ///< Create file, if it exists then fail
+            Exclusive   = 0x40, ///< Create file, if it exists then fail
 
             // Helpful mixes
             ReadWrite   = 0x03, ///< Open for reading and writing
@@ -52,8 +65,9 @@ namespace Bu
 
     public:
         /**
-         * Open existing Myriad stream, or initialize a new one if it doesn't
-         * exist.
+         * Open existing Myriad container, or initialize a new one if the
+         * backing stream is empty. If other data is already in the provided
+         * backing stream an error is thrown.
          *
          * Myriad format V0
          *   0 -   3:  Myriad_MAGIC_CODE (0ad3fa84)
@@ -68,13 +82,40 @@ namespace Bu
          *   4 -   7:  Size of stream in bytes
          *   8 - ...:  List of blocks in stream (4 bytes per block
          */
-        Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1, int32_t iPreallocateBlocks=-1 );
+        Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1,
+                int32_t iPreallocateBlocks=-1 );
         virtual ~Myriad();
 
+        /**
+         * Creates a new stream open in the specified eMode and, optionally,
+         * preallocates the specificed amount of space. The stream is zero
+         * bytes even if space is preallocated. The open stream is returned,
+         * ready for use. Use this if you don't care what the id is of the
+         * newly created stream.
+         */
         MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
+
+        /**
+         * Open an existing stream or create a new stream with the specified
+         * id (iStream) with the specified eMode. This respects the normal file
+         * modes, see Bu::Myriad::Mode for details.
+         */
         MyriadStream open( StreamId iStream, Mode eMode );
+
+        /**
+         * Allocate a new stream but do not open it, just ensure it exists and
+         * return the id of the newly allocated stream.
+         */
+        StreamId allocate();
+
+        /**
+         * Erase the stream specified by iStream. This only can work when the
+         * stream is not open at the moment.
+         */
         void erase( StreamId iStream );
         void setSize( StreamId iStream, int32_t iNewSize );
+        int32_t getSize( StreamId iStream ) const;
+        bool exists( StreamId iStream ) const;
         Bu::String getLocation() const;
         int32_t getBlockSize() const;
         int32_t getTotalBlocks() const;
@@ -83,6 +124,22 @@ namespace Bu
         int32_t getTotalStreams() const;
         int32_t getTotalUsedBytes() const;
         int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
+        Bu::BitString buildBlockUseMap() const;
+        StreamIdArray buildBlockMap() const;
+        
+        /**
+         * Lists all stream ids that you are allowed to open. Technically there
+         * is always a zero stream, but it is used by Myriad for stream/block
+         * accounting. It works like a normal stream but you should not open
+         * it.
+         */
+        StreamIdList getStreamList() const;
+
+        /**
+         * Flush all caches to the backing stream, write all structural and
+         * header changes.
+         */
+        void sync();
 
     private:
         bool loadMyriad();
@@ -175,6 +232,20 @@ namespace Bu
         IndexList lFreeBlocks;
         StreamId iLastUsedIndex;
     };
+    constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b )
+    {
+        return static_cast<Myriad::Mode>(
+            static_cast<std::underlying_type<Myriad::Mode>::type>(a) &
+            static_cast<std::underlying_type<Myriad::Mode>::type>(b)
+            );
+    }
+    constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b )
+    {
+        return static_cast<Myriad::Mode>(
+            static_cast<std::underlying_type<Myriad::Mode>::type>(a) |
+            static_cast<std::underlying_type<Myriad::Mode>::type>(b)
+            );
+    }
 };
 
 #endif
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
index 9ea2e17..eaf91a5 100644
--- a/src/stable/myriadstream.cpp
+++ b/src/stable/myriadstream.cpp
@@ -166,3 +166,9 @@ Bu::String Bu::MyriadStream::getLocation() const
     return pStream->getLocation();
 }
 
+Bu::Myriad::StreamId Bu::MyriadStream::getId() const
+{
+    Bu::MutexLocker l( mAccess );
+    return pStream->getStreamId();
+}
+
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
index b86dbd7..27a15d5 100644
--- a/src/stable/myriadstream.h
+++ b/src/stable/myriadstream.h
@@ -39,6 +39,7 @@ namespace Bu
         virtual size getSize() const;
         virtual size getBlockSize() const;
         virtual Bu::String getLocation() const;
+        Myriad::StreamId getId() const;
 
     private:
         mutable Bu::Mutex mAccess;