From cad3e4d5d87dd713e27779d1f4a20821485b9de2 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Wed, 18 Sep 2024 12:55:05 -0700 Subject: Myriad headers are the right size now. That was weird. It seems to be working now. --- src/stable/file.cpp | 22 +++---- src/stable/myriad.cpp | 159 ++++++++++++++++++++++++++++++++++++++++---------- src/stable/myriad.h | 13 ++++- 3 files changed, 149 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/src/stable/file.cpp b/src/stable/file.cpp index 05e4af1..55766ea 100644 --- a/src/stable/file.cpp +++ b/src/stable/file.cpp @@ -141,6 +141,16 @@ bool Bu::File::isEos() } bool Bu::File::canRead() +{ + return isReadable(); +} + +bool Bu::File::canWrite() +{ + return isWritable(); +} + +bool Bu::File::isReadable() { #ifdef WIN32 return true; @@ -152,7 +162,7 @@ bool Bu::File::canRead() #endif } -bool Bu::File::canWrite() +bool Bu::File::isWritable() { #ifdef WIN32 return true; @@ -164,16 +174,6 @@ bool Bu::File::canWrite() #endif } -bool Bu::File::isReadable() -{ - return true; -} - -bool Bu::File::isWritable() -{ - return true; -} - bool Bu::File::isSeekable() { return true; diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp index 44a990a..4edd004 100644 --- a/src/stable/myriad.cpp +++ b/src/stable/myriad.cpp @@ -15,6 +15,12 @@ "Insufficent data reading myriad data from backing stream."); \ } (void)0 +#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \ +{ \ + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ + "Insufficent data reading from myriad stream."); \ +} (void)0 + namespace Bu { subExceptionDef( MyriadException ) @@ -63,7 +69,9 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode, { pStream->aBlocks.append( __allocateBlock() ); } + mhStream.lock(); hStream.insert( pStream->iStream, pStream ); + mhStream.unlock(); bStructureChanged = true; return Bu::MyriadStream( *this, pStream, (Mode)(eMode&ReadWrite) ); @@ -72,7 +80,7 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode, Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, Bu::Myriad::Mode eMode ) { - Bu::MutexLocker l( mAccess ); + Bu::MutexLocker l( mhStream ); if( !hStream.has( iStream ) ) { throw Bu::MyriadException( MyriadException::noSuchStream, @@ -89,15 +97,19 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, return Bu::MyriadStream( *this, hStream.get( iStream ), eMode ); } -bool Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ ) +void Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ ) { - return false; } -bool Bu::Myriad::setSize( Bu::Myriad::StreamId /*iStream*/, - int32_t /*iNewSize*/ ) +void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream, + int32_t iNewSize ) { - return false; + Stream *pStream; + { + Bu::MutexLocker l( mhStream ); + pStream = hStream.get( iStream ); + } + pStream->setSize( iNewSize ); } Bu::String Bu::Myriad::getLocation() const @@ -108,6 +120,72 @@ Bu::String Bu::Myriad::getLocation() const .arg( 1 ).arg( iBlockSize ).arg( rBacking.getLocation() ); } +int32_t Bu::Myriad::getBlockSize() const +{ + Bu::MutexLocker l( mAccess ); + return iBlockSize; +} + +int32_t Bu::Myriad::getTotalBlocks() const +{ + Bu::MutexLocker l( mAccess ); + return iBlockCount; +} + +int32_t Bu::Myriad::getUsedBlocks() const +{ + Bu::MutexLocker l( mAccess ); + return iBlockCount-lFreeBlocks.getSize(); +} + +int32_t Bu::Myriad::getFreeBlocks() const +{ + Bu::MutexLocker l( mAccess ); + return lFreeBlocks.getSize(); +} + +int32_t Bu::Myriad::getTotalStreams() const +{ + Bu::MutexLocker l( mhStream ); + return hStream.getSize(); +} + +int32_t Bu::Myriad::getTotalUsedBytes() const +{ + Bu::MutexLocker l( mhStream ); + int32_t iTotal = 0; + for( StreamHash::const_iterator i = hStream.begin(); i; i++ ) + { + iTotal += i.getValue()->getSize(); + } + + return iTotal; +} + +int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const +{ + if( iAssumeBlockSize < 0 ) + { + iAssumeBlockSize = getBlockSize(); + } + int32_t iTotal = 0; + { + Bu::MutexLocker l( mhStream ); + for( StreamHash::const_iterator i = hStream.begin(); i; i++ ) + { + if( (i.getValue()->getSize()%iAssumeBlockSize) > 0 ) + iTotal += iBlockSize-(i.getValue()->getSize()%iAssumeBlockSize); + } + } + + { + Bu::MutexLocker l( mAccess ); + iTotal += lFreeBlocks.getSize()*iBlockSize; + } + + return iTotal; +} + bool Bu::Myriad::loadMyriad() { Bu::println("Load myriad!"); @@ -129,11 +207,25 @@ bool Bu::Myriad::loadMyriad() } MyriadRead( &uBitsPerInt, 1 ); if( uBitsPerInt != 32 ) -{ + { throw Bu::MyriadException( Bu::MyriadException::invalidFormat, "Only 32 bits per int are supported at this time."); } MyriadRead( &iBlockSize, 4 ); + + iBlockCount = rBacking.getSize()/iBlockSize; + if( (rBacking.getSize()%iBlockSize) != 0 ) + { + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "Backing stream is not cleanly divisibly by the block size."); + } + + Bu::Hash hUnusedBlocks; + for( int32_t j = 0; j < iBlockCount; j++ ) + { + hUnusedBlocks.insert( j, true ); + } + int iStreamCount; MyriadRead( &iStreamCount, 4 ); @@ -162,6 +254,7 @@ bool Bu::Myriad::loadMyriad() { int32_t iBlockIndex; MyriadRead( &iBlockIndex, 4 ); + hUnusedBlocks.erase( iBlockIndex ); pHeaderStream->aBlocks.append( iBlockIndex ); if( rBacking.tell()+4 <= (j+1)*iBlockSize ) break; @@ -172,7 +265,8 @@ bool Bu::Myriad::loadMyriad() while( pHeaderStream->aBlocks.getSize() < iHeaderStreamBlocks ) { int32_t iBlockIndex; - sHeader.read( &iBlockIndex, 4 ); + ReqRead( sHeader, &iBlockIndex, 4 ); + hUnusedBlocks.erase( iBlockIndex ); pHeaderStream->aBlocks.append( iBlockIndex ); } @@ -180,14 +274,15 @@ bool Bu::Myriad::loadMyriad() for( int j = 1; j < iStreamCount; j++ ) { int32_t iStreamBytes; - sHeader.read( &iStream, 4 ); - sHeader.read( &iStreamBytes, 4 ); + ReqRead( sHeader, &iStream, 4 ); + ReqRead( sHeader, &iStreamBytes, 4 ); Stream *pStream = new Stream( *this, iStream, iStreamBytes ); int32_t iBlocks = blkDiv(iStreamBytes, iBlockSize ); for( int k = 0; k < iBlocks; k++ ) { int32_t iBlockIndex; - sHeader.read( &iBlockIndex, 4 ); + ReqRead( sHeader, &iBlockIndex, 4 ); + hUnusedBlocks.erase( iBlockIndex ); pStream->aBlocks.append( iBlockIndex ); } hStream.insert( iStream, pStream ); @@ -195,6 +290,9 @@ bool Bu::Myriad::loadMyriad() iLastUsedIndex = iStream; } + lFreeBlocks = hUnusedBlocks.getKeys(); + Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); + return true; } @@ -211,23 +309,6 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) "Backing stream contains data, but not a myriad structure."); } -/* - struct { - char sMagicCode[4]; - uint8_t uVer; - uint8_t uBitsPerInt; - uint32_t uBlockSize; - uint32_t uStreamCount; - } sHeader; - - struct { - uint32_t uStreamId; - uint32_t uStreamSize; - } sStreamHeader; - - Bu::println("sHeader = %1, sStreamHeader = %2").arg( sizeof(sHeader) ).arg( sizeof(sStreamHeader) ); -*/ - // Start with the bytes for the file header and initial stream header int iHeaderStreamBytes = 14 // Base header @@ -299,6 +380,7 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) lFreeBlocks.append( j ); } iLastUsedIndex = 0; + iBlockCount = iPreallocateBlocks; } void Bu::Myriad::writeHeader() @@ -309,6 +391,7 @@ void Bu::Myriad::writeHeader() Bu::MemBuf mbHeader; { Bu::MutexLocker l( mAccess ); + Bu::MutexLocker l2( mhStream ); int32_t iHdrStreamSize = __calcHeaderSize(); // Maybe just do stream surgery here. @@ -401,12 +484,15 @@ int32_t Bu::Myriad::__calcHeaderSize() } } + Bu::println("HeaderCalc:"); + Bu::println(" Base (no header stream): %1").arg( iHdrSize ); int32_t iNewSize = iHdrSize; int32_t iOldSize; do { iOldSize = iNewSize; - iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize); + iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4; + Bu::println(" Recomp: %1").arg( iNewSize ); } while( iOldSize != iNewSize ); return iNewSize; @@ -467,13 +553,13 @@ void Bu::Myriad::blankBlock( int32_t iBlockId ) void Bu::Myriad::openStream( StreamId id ) { - Bu::MutexLocker l( mAccess ); + Bu::MutexLocker l( mhStream ); hStream.get( id )->open(); } void Bu::Myriad::closeStream( StreamId id ) { - Bu::MutexLocker l( mAccess ); + Bu::MutexLocker l( mhStream ); hStream.get( id )->close(); } @@ -579,6 +665,17 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget, { int32_t iRead = 0; Bu::MutexLocker l( mAccess ); + + if( iStart >= this->iSize ) + return 0; + + if( iStart+iSize >= this->iSize ) + { + int32_t iDiff = this->iSize-(iStart+iSize); + iSize -= iDiff; + iStart += iDiff; + } + while( iSize > 0 ) { int32_t iBlock = aBlocks[iStart/rParent.iBlockSize]; diff --git a/src/stable/myriad.h b/src/stable/myriad.h index 7cf6041..58f700a 100644 --- a/src/stable/myriad.h +++ b/src/stable/myriad.h @@ -73,9 +73,16 @@ namespace Bu MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 ); MyriadStream open( StreamId iStream, Mode eMode ); - bool erase( StreamId iStream ); - bool setSize( StreamId iStream, int32_t iNewSize ); + void erase( StreamId iStream ); + void setSize( StreamId iStream, int32_t iNewSize ); Bu::String getLocation() const; + int32_t getBlockSize() const; + int32_t getTotalBlocks() const; + int32_t getUsedBlocks() const; + int32_t getFreeBlocks() const; + int32_t getTotalStreams() const; + int32_t getTotalUsedBytes() const; + int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const; private: bool loadMyriad(); @@ -151,7 +158,6 @@ namespace Bu int32_t iSize; Bu::Array aBlocks; int32_t iOpenCount; - bool bStructureChanged; }; private: @@ -164,6 +170,7 @@ namespace Bu int32_t iBlockCount; bool bIsNewStream; bool bStructureChanged; + mutable Bu::Mutex mhStream; StreamHash hStream; IndexList lFreeBlocks; StreamId iLastUsedIndex; -- cgit v1.2.3