From 00651aeaf50f8481a2c894f9462cd3b8eb6971d6 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 1 Oct 2024 11:21:13 -0700 Subject: More myriad fixes, it passes all existing unit tests. --- src/stable/myriad.cpp | 26 ++-- src/tests/myriad.cpp | 23 +++ src/unit/myriad.unit | 385 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 13 deletions(-) create mode 100644 src/tests/myriad.cpp create mode 100644 src/unit/myriad.unit diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp index 85daa0d..f9fcd3e 100644 --- a/src/stable/myriad.cpp +++ b/src/stable/myriad.cpp @@ -12,13 +12,13 @@ #define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \ { \ throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ - "Insufficent data reading myriad data from backing stream."); \ + "Insufficient data reading myriad data from backing stream."); \ } (void)0 #define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \ { \ throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ - "Insufficent data reading from myriad stream."); \ + "Insufficient data reading from myriad stream."); \ } (void)0 namespace Bu @@ -112,7 +112,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, } { Bu::MutexLocker l2( mBacking ); - if( (eMode&Write) && rBacking.isWritable() ) + if( (eMode&Write) && !rBacking.isWritable() ) { throw Bu::MyriadException( MyriadException::badMode, "Backing stream does not support writing."); @@ -398,6 +398,8 @@ bool Bu::Myriad::loadMyriad() lFreeBlocks = hUnusedBlocks.getKeys(); //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); + + bIsNewStream = false; return true; } @@ -434,14 +436,11 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize ); } - if( iPreallocateBlocks > iHeaderStreamBlocks ) + if( iPreallocateBlocks < iHeaderStreamBlocks ) { - rBacking.setSize( iBlockSize*iPreallocateBlocks ); - } - else - { - rBacking.setSize( iBlockSize*iHeaderStreamBlocks ); + iPreallocateBlocks = iHeaderStreamBlocks; } + rBacking.setSize( iBlockSize*iPreallocateBlocks ); // // Write Myriad header @@ -540,14 +539,15 @@ void Bu::Myriad::writeHeader() uint32_t uStreamSize = pStream->getSize(); mbHeader.write( &uStreamId, 4 ); mbHeader.write( &uStreamSize, 4 ); + int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize ); Bu::Array aBlocks = pStream->getBlockList(); //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) ); - for( Bu::Array::iterator i = aBlocks.begin(); i; i++ ) +// for( Bu::Array::iterator i = aBlocks.begin(); i; i++ ) + for( int j = 0; j < iBlocks; j++ ) { - int32_t iIdx = *i; - mbHeader.write( &iIdx, 4 ); + mbHeader.write( &aBlocks[j], 4 ); } } @@ -768,7 +768,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget, if( iStart+iSize >= this->iSize ) { - int32_t iDiff = this->iSize-(iStart+iSize); + int32_t iDiff = (iStart+iSize)-this->iSize; iSize -= iDiff; iStart += iDiff; } diff --git a/src/tests/myriad.cpp b/src/tests/myriad.cpp new file mode 100644 index 0000000..ee4eac4 --- /dev/null +++ b/src/tests/myriad.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include +#include + +int main( int , char *[] ) +{ + Bu::File fMyriad("test.myr", Bu::File::WriteNew|Bu::File::Read ); + Bu::Myriad m( fMyriad, 32 ); + + Bu::MyriadStream ms = m.create( Bu::Myriad::ReadWrite ); + ms.setSize( 150 ); + ms.setPos( 145 ); + char stuff[10]; + int32_t iRead = ms.read( stuff, 10 ); + + Bu::println("Tried to read 10, expect 5, got %1").arg( iRead ); + + + return 0; +} + diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit new file mode 100644 index 0000000..f7bea97 --- /dev/null +++ b/src/unit/myriad.unit @@ -0,0 +1,385 @@ +// vim: syntax=cpp +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/string.h" +#include "bu/file.h" +#include "bu/myriad.h" +#include "bu/myriadstream.h" +#include "bu/array.h" + +#include "bu/sio.h" +#include "bu/archive.h" +#include "bu/md5.h" +#include "bu/unitsuite.h" + +#include + +using namespace Bu; + +class VerifyObject +{ +friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo ); +friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo ); +public: + VerifyObject( int iUnits ) : + iUnits( iUnits ), + iBytesWritten( 0 ) + { + } + + virtual ~VerifyObject() + { + } + + int getBytesWritten() + { + return iBytesWritten; + } + +private: + int iUnits; + mutable int iBytesWritten; +}; + +Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo ) +{ + Md5 sum; + ar << vo.iUnits; + vo.iBytesWritten = sizeof(int); + sum.addData( &vo.iUnits, sizeof(int) ); + for( int j = 0; j < vo.iUnits; j++ ) + { + int iRand = random()%128; +// ar << iRand; + Bu::String sDat( iRand ); + for( int j = 0; j < iRand; j++ ) + sDat[j] = (char)((uint8_t)(random()%256)); + ar << sDat; + sum.addData( &iRand, sizeof(int) ); + sum.addData( sDat.getStr(), iRand ); + vo.iBytesWritten += sizeof(long) + iRand; + } + Bu::String sRes = sum.getResult(); + ar << sRes; + vo.iBytesWritten += sizeof(long) + sRes.getSize(); + return ar; +} + +Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo ) +{ + Md5 sum; + ar >> vo.iUnits; + sum.addData( &vo.iUnits, sizeof(int) ); + for( int j = 0; j < vo.iUnits; j++ ) + { + int iRand; +// ar >> iRand; + Bu::String sStr; + ar >> sStr; + iRand = sStr.getSize(); + sum.addData( &iRand, sizeof(int) ); + sum.addData( sStr.getStr(), iRand ); + } + Bu::String sSum; + ar >> sSum; + unitTest( sSum == sum.getResult() ); + int iTooMuch; + try + { + ar >> iTooMuch; + unitFailed("should have thrown an exception."); + } + catch( Bu::ExceptionBase &e ) + { + } + return ar; +} + +suite Myriad +{ + test setSize + { + String sFileName("myriad-XXXXXXX"); + + File fMyriad = tempFile( sFileName ); + Myriad m( fMyriad, 32 ); + + MyriadStream ms = m.create( Myriad::ReadWrite ); + ms.setSize( 150 ); + ms.setPos( 145 ); + char stuff[10]; + unitTest( ms.read( stuff, 10 ) == 5 ); + + ms.setSize( 12 ); + unitTest( ms.read( stuff, 10 ) == 0 ); + unitTest( ms.write( "hello", 5 ) == 5 ); + unitTest( ms.tell() == 17 ); + + ms.setSize( 500 ); + unitTest( ms.tell() == 17 ); + } + + void addBlock( Stream &s, bool bAppend=true ) + { + if( bAppend ) + s.setPosEnd( 0 ); + int iSize = (random()%1016)+8; + s.write( &iSize, 4 ); + char *buf = new char[iSize-8]; + for( int j = 0; j < iSize-8; j++ ) + { + buf[j] = (j+iSize)%256; + } + if( random()%2 == 0 ) + { + s.write( buf, iSize-8 ); + } + else + { + for( int j = 0; j < iSize-8; ) + { + int iAmnt = (random()%8)+1; + if( iAmnt+j > iSize-8 ) + iAmnt = iSize-8-j; + iAmnt = s.write( buf+j, iAmnt ); + j += iAmnt; + } + } + delete[] buf; + iSize = ~iSize; + s.write( &iSize, 4 ); + } + + void verifyBlock( Stream &s ) + { + int iSize, iInv; + if( s.read( &iSize, 4 ) == 0 ) + return; + if( iSize < 8 || iSize > 1024 ) + throw ExceptionBase("Read bad data, %d", iSize ); + char *buf = new char[iSize-8]; + if( s.read( buf, iSize-8 ) < (Bu::size)iSize-8 ) + { + delete[] buf; + throw ExceptionBase("Block failed verify (insuffient block data)."); + } + for( int j = 0; j < iSize-8; j++ ) + { + if( buf[j] != (char)((j+iSize)%256) ) + { + char b = buf[j]; + delete[] buf; + throw ExceptionBase("Block failed computed data verify " + "(%02X==%02X).", b, (char)((j+iSize)%256) ); + } + } + delete[] buf; + if( s.read( &iInv, 4 ) < 4 ) + throw ExceptionBase("Block failed verify (insufficient data)."); + if( iInv != ~iSize ) + throw ExceptionBase("Block failed inversion verify."); + } + + void verifyStream( Stream &s ) + { + s.setPos( 0 ); + while( !s.isEos() ) + verifyBlock( s ); + } + + test stressGrow + { + String sFileName("myriad-XXXXXXX"); + + File fMyriad = tempFile( sFileName ); + Myriad m( fMyriad, 64 ); + + Array aStreams; + for( int j = 0; j < 5; j++ ) + { + aStreams.append( m.create( Bu::Myriad::Read ).getId() ); + } + + srandom( 512 ); + + for( int j = 0; j < 2500; j++ ) + { + switch( random()%5 ) + { + case 0: + aStreams.append( m.create( Bu::Myriad::Read ).getId() ); + break; + + case 1: + if( aStreams.getSize() > 0 ) + { + int iStream = random()%aStreams.getSize(); + { + MyriadStream ms = m.open( aStreams[iStream], Myriad::Read ); + verifyStream( ms ); + } + m.erase( aStreams[iStream] ); + Array::iterator i = aStreams.begin(); + for( int k = 0; k < iStream; k++ ) + i++; + aStreams.erase( i ); + } + break; + + default: + if( aStreams.getSize() == 0 ) + { + aStreams.append( + m.create( Bu::Myriad::Read ).getId() + ); + } + { + int iStream = random()%aStreams.getSize(); + MyriadStream ms = m.open( aStreams[iStream], Myriad::ReadWrite ); + addBlock( ms ); + verifyStream( ms ); + } + break; + } + } + + for( Array::iterator i = aStreams.begin(); i; i++ ) + { + MyriadStream ms = m.open( *i, Myriad::Read ); + verifyStream( ms ); + } + } + + test stressTruncate + { + String sFileName("myriad-XXXXXXX"); + + File fMyriad = tempFile( sFileName ); + Myriad m( fMyriad, 128 ); + + Array aStream; + + for( int j = 0; j < 5; j++ ) + { + aStream.append( m.create( Bu::Myriad::Read ).getId() ); + } + + srandom( 1024 ); + + char b; + for( int iter = 0; iter < 2500; iter++ ) + { + for( Array::iterator i = aStream.begin(); i; i++ ) + { + MyriadStream ms = m.open( *i, Myriad::ReadWrite ); + addBlock( ms, false ); + ms.setSize( ms.tell() ); + unitTest( ms.read( &b, 1 ) == 0 ); + ms.setPos( 0 ); + verifyBlock( ms ); + unitTest( ms.read( &b, 1 ) == 0 ); + } + } + } + + test stressTruncate2 + { + String sFileName("myriad-XXXXXXX"); + + Array aStream; + + setStepCount( 5*2500 + 5 ); + + { + File fMyriad = tempFile( sFileName ); + Myriad m( fMyriad, 128 ); + + for( int j = 0; j < 5; j++ ) + { + aStream.append( m.create( Bu::Myriad::Read ).getId() ); + incProgress(); + } + } + + srandom( 1024 ); + + char b; + for( int iter = 0; iter < 2500; iter++ ) + { + File fMyriad( sFileName, File::ReadWrite ); + Myriad m( fMyriad ); + for( Array::iterator i = aStream.begin(); i; i++ ) + { + MyriadStream ms = m.open( *i, Myriad::ReadWrite ); + addBlock( ms, false ); + ms.setSize( ms.tell() ); + unitTest( ms.read( &b, 1 ) == 0 ); + ms.setPos( 0 ); + verifyBlock( ms ); + unitTest( ms.read( &b, 1 ) == 0 ); + incProgress(); + } + } + } + + test stressArchive + { + String sFileName("myriad-XXXXXX"); + Array aStream; + + srandom( 2096 ); + + setStepCount( 15*250 + 15 ); + + { + File fMyriad = tempFile( sFileName ); + Myriad m( fMyriad, 1024 ); + + for( int j = 0; j < 15; j++ ) + { + MyriadStream ms = m.create( Myriad::Write ); + int iStream = ms.getId(); + aStream.append( iStream ); + VerifyObject vo( random()%1024 ); + { + Archive ar( ms, Archive::save ); + ar << vo; + unitTest( ms.tell() == vo.getBytesWritten() ); + ms.setSize( ms.tell() ); + } + unitTest( m.getSize( iStream ) == vo.getBytesWritten() ); + incProgress(); + } + } + + for( int iter = 0; iter < 250; iter++ ) + { + File fMyriad( sFileName, File::ReadWrite ); + Myriad m( fMyriad ); + for( Array::iterator i = aStream.begin(); i; i++ ) + { + VerifyObject vo( random()%1024 ); + { + MyriadStream ms = m.open( *i, Myriad::Read ); + Archive ar( ms, Archive::load ); + ar >> vo; + } + { + MyriadStream ms = m.open( *i, Myriad::WriteNew ); + Archive ar( ms, Archive::save ); + ar << vo; + unitTest( ms.tell() == vo.getBytesWritten() ); + ms.setSize( ms.tell() ); + } + unitTest( m.getSize( *i ) == vo.getBytesWritten() ); + incProgress(); + } + } + } +} + -- cgit v1.2.3