Erasing streams works, stream map building works.

6 files changed, 119 insertions, 41 deletions

diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index eba1ebf..85daa0d 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -252,6 +252,41 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
     return iTotal;
 }
 
+Bu::BitString Bu::Myriad::buildBlockUseMap() const
+{
+    Bu::MutexLocker l( mAccess );
+    Bu::BitString bsMap( iBlockCount );
+    bsMap.fill();
+    for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
+    {
+        bsMap.setBit( *i, false );
+    }
+    return bsMap;
+}
+
+Bu::Array<int32_t> Bu::Myriad::buildBlockMap() const
+{
+    Bu::MutexLocker l( mAccess );
+    Bu::Array<int32_t> bm( iBlockCount );
+    for( int j = 0; j < iBlockCount; j++ )
+    {
+        bm.append( -1 );
+    }
+    Bu::MutexLocker l2( mhStream );
+    for( StreamHash::const_iterator iStream = hStream.begin();
+         iStream; iStream++ )
+    {
+        int32_t iId = iStream.getKey();
+        Stream *pStream = iStream.getValue();
+        for( Bu::Array<int32_t>::const_iterator iBlock =
+             pStream->aBlocks.begin(); iBlock; iBlock++ )
+        {
+            bm[*iBlock] = iId;
+        }
+    }
+    return bm;
+}
+
 void Bu::Myriad::sync()
 {
     writeHeader();
@@ -259,7 +294,7 @@ void Bu::Myriad::sync()
 
 bool Bu::Myriad::loadMyriad()
 {
-    Bu::println("Load myriad!");
+    //Bu::println("Load myriad!");
     char sMagicCode[4];
     rBacking.setPos( 0 );
     MyriadRead( sMagicCode, 4 );
@@ -362,7 +397,7 @@ bool Bu::Myriad::loadMyriad()
     }
 
     lFreeBlocks = hUnusedBlocks.getKeys();
-    Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
+    //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
 
     return true;
 }
@@ -389,22 +424,14 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
     // plus one block index.
     int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
 
-    Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
-        .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
-        .arg( iHeaderStreamBlocks )
-        .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
-        .arg( iHeaderStreamBlocks*iBlockSize );
+    //Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
     while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
            > iHeaderStreamBlocks*iBlockSize )
     {
         iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize);
         if( iHeaderStreamBlocks > 100 )
             break;
-        Bu::println("      Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
-            .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
-             .arg( iHeaderStreamBlocks )
-            .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
-            .arg( iHeaderStreamBlocks*iBlockSize );
+        //Bu::println("      Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
     }
 
     if( iPreallocateBlocks > iHeaderStreamBlocks )
@@ -459,7 +486,7 @@ void Bu::Myriad::writeHeader()
     Bu::MutexLocker l( mAccess );
     if( !rBacking.isWritable() )
         return;
-    Bu::println("Writing stream breakdown:");
+    //Bu::println("Writing stream breakdown:");
     Bu::MemBuf mbHeader;
     {
         Bu::MutexLocker l2( mhStream );
@@ -488,7 +515,7 @@ void Bu::Myriad::writeHeader()
             psHeader->iSize = iHdrStreamSize;
         }
 
-        Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
+        //Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
         
         uint8_t uVer = 1;
         uint8_t uBpi = 32;
@@ -515,10 +542,7 @@ void Bu::Myriad::writeHeader()
             mbHeader.write( &uStreamSize, 4 );
             Bu::Array<int32_t> aBlocks = pStream->getBlockList();
             
-            Bu::println("  Stream %1 is %2 bytes %3 blocks (%4 blocks computed)")
-                .arg( *i ).arg( uStreamSize )
-                .arg( aBlocks.getSize() )
-                .arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
+            //Bu::println("  Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
 
             for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
             {
@@ -556,15 +580,15 @@ int32_t Bu::Myriad::__calcHeaderSize()
         }
     }
 
-    Bu::println("HeaderCalc:");
-    Bu::println("  Base (no header stream): %1").arg( iHdrSize );
+    //Bu::println("HeaderCalc:");
+    //Bu::println("  Base (no header stream): %1").arg( iHdrSize );
     int32_t iNewSize = iHdrSize;
     int32_t iOldSize;
 
     do {
         iOldSize = iNewSize;
         iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4;
-        Bu::println("  Recomp: %1").arg( iNewSize );
+        //Bu::println("  Recomp: %1").arg( iNewSize );
     } while( iOldSize != iNewSize );
 
     return iNewSize;
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 60c5a39..7f84c2a 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -7,6 +7,8 @@
 #include "bu/array.h"
 #include "bu/hash.h"
 
+#include "bu/bitstring.h"
+
 namespace Bu
 {
     class MyriadStream;
@@ -87,6 +89,8 @@ namespace Bu
         int32_t getTotalStreams() const;
         int32_t getTotalUsedBytes() const;
         int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
+        Bu::BitString buildBlockUseMap() const;
+        Bu::Array<int32_t> buildBlockMap() const;
 
         /**
          * Flush all caches to the backing stream, write all structural and
diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp
index 9d24741..cce309d 100644
--- a/src/tests/bigmyriad.cpp
+++ b/src/tests/bigmyriad.cpp
@@ -5,14 +5,15 @@
 int main()
 {
     Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create );
-    Bu::Myriad m( f, 256, 12 );
+    Bu::Myriad m( f, 1024, 12 );
 
-    char *buf = new char[1024*1024*10];
+#define SSIZE (1024*8) // 1024*1024*10
+    char *buf = new char[SSIZE];
 
     for( int j = 0; j < 25; j++ )
     {
-        memset( buf, j, 1024*1024*10 );
-        m.create( Bu::Myriad::Write ).write( buf, 1024*1024*10 );
+        memset( buf, j, SSIZE );
+        m.create( Bu::Myriad::Write ).write( buf, SSIZE );
 //        m.sync();
         printf("\r%03d%%", (j+1)*100/25 );
         fflush( stdout );
diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp
index 346cd85..0dd9840 100644
--- a/src/tools/myriad.cpp
+++ b/src/tools/myriad.cpp
@@ -94,6 +94,49 @@ Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
     return f;
 }
 
+void printMap( const Bu::BitString &bs )
+{
+    for( int j = 0; j < bs.getSize(); j++ )
+    {
+        if( j>0 && (j%50) == 0 )
+            Bu::println("");
+        if( bs.getBit( j ) )
+            Bu::print("#");
+        else
+            Bu::print("-");
+    }
+    Bu::println("\n");
+}
+
+void printMap( const Bu::Array<int32_t> &bm )
+{
+    int iBigest = 0;
+    for( int j = 0; j < bm.getSize(); j++ )
+    {
+        if( iBigest < bm[j] )
+            iBigest = bm[j];
+    }
+    int iWidth = Bu::String("%1").arg( iBigest ).end().getSize();
+    Bu::String sEmpty;
+    for( int j = 0; j < iWidth; j++ )
+    {
+        sEmpty += '-';
+    }
+    int iBreakAt = 60/(iWidth+1);
+    for( int j = 0; j < bm.getSize(); j++ )
+    {
+        if( j>0 && (j%iBreakAt) == 0 )
+            Bu::println("");
+
+        if( bm[j] < 0 )
+            Bu::print("%1 ").arg( sEmpty, Bu::Fmt(2).right().fill(' '));
+        else
+            Bu::print("%1 ").arg( bm[j], Bu::Fmt(2).right().fill(' '));
+        
+    }
+    Bu::println("\n");
+}
+
 int main( int argc, char *argv[] )
 {
     Options opts( argc, argv );
@@ -168,6 +211,8 @@ int main( int argc, char *argv[] )
                 File fOut( opts.sFile, File::Write|File::Read );
                 Myriad m( fOut );
                 m.erase( opts.iStream );
+                printMap( m.buildBlockUseMap() );
+                printMap( m.buildBlockMap() );
             }
             break;
 
@@ -273,17 +318,8 @@ int main( int argc, char *argv[] )
             {
                 File fIn( opts.sFile, File::Write|File::Read );
                 Myriad m( fIn );
-/*                Bu::BitString bs = m.getBlocksUsed();
-                for( int j = 0; j < bs.getSize(); j++ )
-                {
-                    if( j>0 && (j%50) == 0 )
-                        Bu::println("");
-                    if( bs.getBit( j ) )
-                        Bu::print("#");
-                    else
-                        Bu::print("-");
-                }*/
-                Bu::println("\n");
+                printMap( m.buildBlockUseMap() );
+                printMap( m.buildBlockMap() );
             }
             break;
 
@@ -295,4 +331,3 @@ int main( int argc, char *argv[] )
 
     return 0;
 }
-
diff --git a/src/unstable/bitstring.cpp b/src/unstable/bitstring.cpp
index 21c1316..b80c073 100644
--- a/src/unstable/bitstring.cpp
+++ b/src/unstable/bitstring.cpp
@@ -209,7 +209,7 @@ void Bu::BitString::flipBit( long iBit )
     caData[iBit/8] ^= (1<<(iBit%8));
 }
 
-bool Bu::BitString::getBit( long iBit )
+bool Bu::BitString::getBit( long iBit ) const
 {
     if( iBit >= iBits || iBit < 0 ) return false;
     if( (caData[iBit/8] & (1<<(iBit%8))) == 0 )
@@ -224,7 +224,7 @@ long Bu::BitString::getBitLength()
     return iBits;
 }
 
-long Bu::BitString::getSize()
+long Bu::BitString::getSize() const
 {
     return iBits;
 }
@@ -311,6 +311,14 @@ void Bu::BitString::clear()
     }
 }
 
+void Bu::BitString::fill()
+{
+    if( caData != NULL )
+    {
+        memset( caData, 0xff, iBytes );
+    }
+}
+
 bool Bu::BitString::setBitLength( long iLength, bool bClear )
 {
     return setSize( iLength, bClear );
diff --git a/src/unstable/bitstring.h b/src/unstable/bitstring.h
index afc22fb..70ba822 100644
--- a/src/unstable/bitstring.h
+++ b/src/unstable/bitstring.h
@@ -88,7 +88,7 @@ namespace Bu
          *@param iBit The index of the bit to test.
          *@returns True for a 1, false for a 0.
          */
-        bool getBit( long iBit );
+        bool getBit( long iBit ) const;
 
         /**
          * Inverts the entire BitString, in effect this calls flipBit on every
@@ -106,7 +106,7 @@ namespace Bu
         DEPRECATED
         long getBitLength();
 
-        long getSize();
+        long getSize() const;
 
         /**
          * Sets the entire BitString to zeros, but it does it very quickly.
@@ -115,6 +115,12 @@ namespace Bu
         void clear();
 
         /**
+         * Sets the entire BitString to ones, but it does it very quickly.
+         * This operation runs in O(N).
+         */
+        void fill();
+
+        /**
          * Gets another BitString that is autonomous of the current one
          * (contains a copy of the memory, not a pointer) and contains a subset
          * of the data in the current BitString.  This is an inclusive