From 0befcc026be9e4f6c40f8771c7f26f767ebddbf7 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 30 Sep 2024 11:50:17 -0700 Subject: Erasing streams works, stream map building works. --- src/stable/myriad.cpp | 66 +++++++++++++++++++++++++++++++--------------- src/stable/myriad.h | 4 +++ src/tests/bigmyriad.cpp | 9 ++++--- src/tools/myriad.cpp | 59 ++++++++++++++++++++++++++++++++--------- src/unstable/bitstring.cpp | 12 +++++++-- src/unstable/bitstring.h | 10 +++++-- 6 files changed, 119 insertions(+), 41 deletions(-) diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp index eba1ebf..85daa0d 100644 --- a/src/stable/myriad.cpp +++ b/src/stable/myriad.cpp @@ -252,6 +252,41 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const return iTotal; } +Bu::BitString Bu::Myriad::buildBlockUseMap() const +{ + Bu::MutexLocker l( mAccess ); + Bu::BitString bsMap( iBlockCount ); + bsMap.fill(); + for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ ) + { + bsMap.setBit( *i, false ); + } + return bsMap; +} + +Bu::Array Bu::Myriad::buildBlockMap() const +{ + Bu::MutexLocker l( mAccess ); + Bu::Array bm( iBlockCount ); + for( int j = 0; j < iBlockCount; j++ ) + { + bm.append( -1 ); + } + Bu::MutexLocker l2( mhStream ); + for( StreamHash::const_iterator iStream = hStream.begin(); + iStream; iStream++ ) + { + int32_t iId = iStream.getKey(); + Stream *pStream = iStream.getValue(); + for( Bu::Array::const_iterator iBlock = + pStream->aBlocks.begin(); iBlock; iBlock++ ) + { + bm[*iBlock] = iId; + } + } + return bm; +} + void Bu::Myriad::sync() { writeHeader(); @@ -259,7 +294,7 @@ void Bu::Myriad::sync() bool Bu::Myriad::loadMyriad() { - Bu::println("Load myriad!"); + //Bu::println("Load myriad!"); char sMagicCode[4]; rBacking.setPos( 0 ); MyriadRead( sMagicCode, 4 ); @@ -362,7 +397,7 @@ bool Bu::Myriad::loadMyriad() } lFreeBlocks = hUnusedBlocks.getKeys(); - Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); + //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); return true; } @@ -389,22 +424,14 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) // plus one block index. int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize ); - Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") - .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ) - .arg( iHeaderStreamBlocks ) - .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ) - .arg( iHeaderStreamBlocks*iBlockSize ); + //Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize ); while( iHeaderStreamBytes+(iHeaderStreamBlocks*4) > iHeaderStreamBlocks*iBlockSize ) { iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize); if( iHeaderStreamBlocks > 100 ) break; - Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") - .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ) - .arg( iHeaderStreamBlocks ) - .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ) - .arg( iHeaderStreamBlocks*iBlockSize ); + //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize ); } if( iPreallocateBlocks > iHeaderStreamBlocks ) @@ -459,7 +486,7 @@ void Bu::Myriad::writeHeader() Bu::MutexLocker l( mAccess ); if( !rBacking.isWritable() ) return; - Bu::println("Writing stream breakdown:"); + //Bu::println("Writing stream breakdown:"); Bu::MemBuf mbHeader; { Bu::MutexLocker l2( mhStream ); @@ -488,7 +515,7 @@ void Bu::Myriad::writeHeader() psHeader->iSize = iHdrStreamSize; } - Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize ); + //Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize ); uint8_t uVer = 1; uint8_t uBpi = 32; @@ -515,10 +542,7 @@ void Bu::Myriad::writeHeader() mbHeader.write( &uStreamSize, 4 ); Bu::Array aBlocks = pStream->getBlockList(); - Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)") - .arg( *i ).arg( uStreamSize ) - .arg( aBlocks.getSize() ) - .arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) ); + //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) ); for( Bu::Array::iterator i = aBlocks.begin(); i; i++ ) { @@ -556,15 +580,15 @@ int32_t Bu::Myriad::__calcHeaderSize() } } - Bu::println("HeaderCalc:"); - Bu::println(" Base (no header stream): %1").arg( iHdrSize ); + //Bu::println("HeaderCalc:"); + //Bu::println(" Base (no header stream): %1").arg( iHdrSize ); int32_t iNewSize = iHdrSize; int32_t iOldSize; do { iOldSize = iNewSize; iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4; - Bu::println(" Recomp: %1").arg( iNewSize ); + //Bu::println(" Recomp: %1").arg( iNewSize ); } while( iOldSize != iNewSize ); return iNewSize; diff --git a/src/stable/myriad.h b/src/stable/myriad.h index 60c5a39..7f84c2a 100644 --- a/src/stable/myriad.h +++ b/src/stable/myriad.h @@ -7,6 +7,8 @@ #include "bu/array.h" #include "bu/hash.h" +#include "bu/bitstring.h" + namespace Bu { class MyriadStream; @@ -87,6 +89,8 @@ namespace Bu int32_t getTotalStreams() const; int32_t getTotalUsedBytes() const; int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const; + Bu::BitString buildBlockUseMap() const; + Bu::Array buildBlockMap() const; /** * Flush all caches to the backing stream, write all structural and diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp index 9d24741..cce309d 100644 --- a/src/tests/bigmyriad.cpp +++ b/src/tests/bigmyriad.cpp @@ -5,14 +5,15 @@ int main() { Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create ); - Bu::Myriad m( f, 256, 12 ); + Bu::Myriad m( f, 1024, 12 ); - char *buf = new char[1024*1024*10]; +#define SSIZE (1024*8) // 1024*1024*10 + char *buf = new char[SSIZE]; for( int j = 0; j < 25; j++ ) { - memset( buf, j, 1024*1024*10 ); - m.create( Bu::Myriad::Write ).write( buf, 1024*1024*10 ); + memset( buf, j, SSIZE ); + m.create( Bu::Myriad::Write ).write( buf, SSIZE ); // m.sync(); printf("\r%03d%%", (j+1)*100/25 ); fflush( stdout ); diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp index 346cd85..0dd9840 100644 --- a/src/tools/myriad.cpp +++ b/src/tools/myriad.cpp @@ -94,6 +94,49 @@ Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ ) return f; } +void printMap( const Bu::BitString &bs ) +{ + for( int j = 0; j < bs.getSize(); j++ ) + { + if( j>0 && (j%50) == 0 ) + Bu::println(""); + if( bs.getBit( j ) ) + Bu::print("#"); + else + Bu::print("-"); + } + Bu::println("\n"); +} + +void printMap( const Bu::Array &bm ) +{ + int iBigest = 0; + for( int j = 0; j < bm.getSize(); j++ ) + { + if( iBigest < bm[j] ) + iBigest = bm[j]; + } + int iWidth = Bu::String("%1").arg( iBigest ).end().getSize(); + Bu::String sEmpty; + for( int j = 0; j < iWidth; j++ ) + { + sEmpty += '-'; + } + int iBreakAt = 60/(iWidth+1); + for( int j = 0; j < bm.getSize(); j++ ) + { + if( j>0 && (j%iBreakAt) == 0 ) + Bu::println(""); + + if( bm[j] < 0 ) + Bu::print("%1 ").arg( sEmpty, Bu::Fmt(2).right().fill(' ')); + else + Bu::print("%1 ").arg( bm[j], Bu::Fmt(2).right().fill(' ')); + + } + Bu::println("\n"); +} + int main( int argc, char *argv[] ) { Options opts( argc, argv ); @@ -168,6 +211,8 @@ int main( int argc, char *argv[] ) File fOut( opts.sFile, File::Write|File::Read ); Myriad m( fOut ); m.erase( opts.iStream ); + printMap( m.buildBlockUseMap() ); + printMap( m.buildBlockMap() ); } break; @@ -273,17 +318,8 @@ int main( int argc, char *argv[] ) { File fIn( opts.sFile, File::Write|File::Read ); Myriad m( fIn ); -/* Bu::BitString bs = m.getBlocksUsed(); - for( int j = 0; j < bs.getSize(); j++ ) - { - if( j>0 && (j%50) == 0 ) - Bu::println(""); - if( bs.getBit( j ) ) - Bu::print("#"); - else - Bu::print("-"); - }*/ - Bu::println("\n"); + printMap( m.buildBlockUseMap() ); + printMap( m.buildBlockMap() ); } break; @@ -295,4 +331,3 @@ int main( int argc, char *argv[] ) return 0; } - diff --git a/src/unstable/bitstring.cpp b/src/unstable/bitstring.cpp index 21c1316..b80c073 100644 --- a/src/unstable/bitstring.cpp +++ b/src/unstable/bitstring.cpp @@ -209,7 +209,7 @@ void Bu::BitString::flipBit( long iBit ) caData[iBit/8] ^= (1<<(iBit%8)); } -bool Bu::BitString::getBit( long iBit ) +bool Bu::BitString::getBit( long iBit ) const { if( iBit >= iBits || iBit < 0 ) return false; if( (caData[iBit/8] & (1<<(iBit%8))) == 0 ) @@ -224,7 +224,7 @@ long Bu::BitString::getBitLength() return iBits; } -long Bu::BitString::getSize() +long Bu::BitString::getSize() const { return iBits; } @@ -311,6 +311,14 @@ void Bu::BitString::clear() } } +void Bu::BitString::fill() +{ + if( caData != NULL ) + { + memset( caData, 0xff, iBytes ); + } +} + bool Bu::BitString::setBitLength( long iLength, bool bClear ) { return setSize( iLength, bClear ); diff --git a/src/unstable/bitstring.h b/src/unstable/bitstring.h index afc22fb..70ba822 100644 --- a/src/unstable/bitstring.h +++ b/src/unstable/bitstring.h @@ -88,7 +88,7 @@ namespace Bu *@param iBit The index of the bit to test. *@returns True for a 1, false for a 0. */ - bool getBit( long iBit ); + bool getBit( long iBit ) const; /** * Inverts the entire BitString, in effect this calls flipBit on every @@ -106,7 +106,7 @@ namespace Bu DEPRECATED long getBitLength(); - long getSize(); + long getSize() const; /** * Sets the entire BitString to zeros, but it does it very quickly. @@ -114,6 +114,12 @@ namespace Bu */ void clear(); + /** + * Sets the entire BitString to ones, but it does it very quickly. + * This operation runs in O(N). + */ + void fill(); + /** * Gets another BitString that is autonomous of the current one * (contains a copy of the memory, not a pointer) and contains a subset -- cgit v1.2.3