From 41c9581b48f055f6559335ffc0316f27ed1b3657 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 12 Apr 2010 07:37:09 +0000 Subject: Myriad is getting pretty close, just have to finish the writing code and probably tweak the header init. --- src/myriad.cpp | 69 +++++++++++++++++----- src/myriad.h | 41 ++++++++++---- src/myriadstream.cpp | 157 ++++++++++++++++++++++++++------------------------- src/myriadstream.h | 12 ++-- src/tools/myriad.cpp | 112 +++++++++++++++++++++++++++++------- 5 files changed, 261 insertions(+), 130 deletions(-) (limited to 'src') diff --git a/src/myriad.cpp b/src/myriad.cpp index d3914df..a1a5c38 100644 --- a/src/myriad.cpp +++ b/src/myriad.cpp @@ -77,14 +77,18 @@ void Bu::Myriad::initialize() iBlocks = iSize/iBlockSize; sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl; - + + // Don't do this, just read the damn header. + sio << "Myriad: Don't do this, just read the damn header (line 82)" + << sio.nl; + int iHeaderSize = 14 + 8 + 4; - int iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); + int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize ); while( iHeaderSize > iHeaderBlocks*iBlockSize ) { - iHeaderSize = 14 + 8 + 4*iHeaderBlocks; iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); + iHeaderSize = 14 + 8 + 4*iHeaderBlocks; } sio << "Myriad: iHeaderSize=" << iHeaderSize @@ -102,7 +106,8 @@ void Bu::Myriad::initialize() sStore.read( &s.iId, 4 ); sStore.read( &s.iSize, 4 ); int iSBlocks = blkDiv(s.iSize, iBlockSize); - sio << "Myriad: - Stream::iId=" << s.iId << ", Stream::iSize=" << s.iSize + sio << "Myriad: - Stream::iId=" << s.iId + << ", Stream::iSize=" << s.iSize << ", Stream::aBlocks=" << iSBlocks << ", sStore.tell()=" << sStore.tell() << sio.nl; for( int k = 0; k < iSBlocks; k++ ) @@ -133,7 +138,7 @@ void Bu::Myriad::initialize() } } - sio << bsBlockUsed.toString() << sio.nl; + sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; //printf("%d blocks, %db each, %db block offset\n", // iBlocks, iBlockSize, iBlockStart ); @@ -143,7 +148,7 @@ void Bu::Myriad::initialize() void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) { int iHeaderSize = 14 + 8 + 4; - int iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); + int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize ); char cBuf = 1; int iBuf = 0; @@ -152,13 +157,13 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) while( iHeaderSize > iHeaderBlocks*iBlockSize ) { - iHeaderSize = 14 + 8 + 4*iHeaderBlocks; iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); + iHeaderSize = 14 + 8 + 4*iHeaderBlocks; } iPreAllocate += iHeaderBlocks; - sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" + sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; bsBlockUsed.setSize( iPreAllocate, true ); @@ -204,6 +209,7 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) this->iBlocks = iPreAllocate; pStr->iSize = sStore.tell(); + sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl; //hStreams.insert( 0, BlockArray( 0 ) ); } @@ -229,9 +235,9 @@ int Bu::Myriad::createStream( int iPreAllocate ) for( int j = 0; j < iPreAllocate; j++ ) { int iFreeBlock = findEmptyBlock(); - sio << "Myriad: Adding block " << j << sio.nl; - pStr->aBlocks.append( j ); - bsBlockUsed.setBit( j ); + sio << "Myriad: Adding block " << iFreeBlock << sio.nl; + pStr->aBlocks.append( iFreeBlock ); + bsBlockUsed.setBit( iFreeBlock ); } return 0; @@ -261,9 +267,9 @@ void Bu::Myriad::deleteStream( int /*iID*/ ) { } -Bu::MyriadStream Bu::Myriad::openStream( int iID ) +Bu::MyriadStream Bu::Myriad::openStream( int iId ) { - return MyriadStream( *this, iID ); + return MyriadStream( *this, findStream( iId ) ); } int Bu::Myriad::getBlockSize() @@ -281,4 +287,41 @@ int Bu::Myriad::getNumUsedBlocks() return iUsed; } +Bu::Myriad::Stream *Bu::Myriad::findStream( int iId ) +{ + for( StreamArray::iterator i = aStreams.begin(); i; i++ ) + { + if( (*i)->iId == iId ) + return *i; + } + + return NULL; +} + +Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock ) +{ + sio << "Myriad: Reading block " << iBlock << ", bytes " + << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl; + Block *pBlock = new Block; + pBlock->pData = new char[iBlockSize]; + sStore.setPos( iBlockSize * iBlock ); + sStore.read( pBlock->pData, iBlockSize ); + pBlock->bChanged = false; + pBlock->iBlockIndex = iBlock; + + return pBlock; +} + +void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock ) +{ + sio << "Myriad: Releasing block " << pBlock->iBlockIndex << sio.nl; + if( pBlock->bChanged ) + { + sio << "Myriad: - Block changed, writing back to stream." << sio.nl; + sStore.setPos( iBlockSize * pBlock->iBlockIndex ); + sStore.write( pBlock->pData, iBlockSize ); + } + delete[] pBlock->pData; + delete pBlock; +} diff --git a/src/myriad.h b/src/myriad.h index f6d5e51..0344057 100644 --- a/src/myriad.h +++ b/src/myriad.h @@ -30,7 +30,7 @@ namespace Bu * Header format is as follows: * * MMMMvBssssSSSS* - * M = Magic number + * M = Magic number (FFC399BD) * v = version number * B = Bits per int * s = Blocksize in bytes @@ -95,12 +95,12 @@ namespace Bu /** * Delete a stream that's already within the Myriad. */ - void deleteStream( int iID ); + void deleteStream( int iId ); /** * Return a new Stream object assosiated with the given stream ID. */ - MyriadStream openStream( int iID ); + MyriadStream openStream( int iId ); int getBlockSize(); int getNumBlocks(); @@ -118,25 +118,42 @@ namespace Bu { blockUnused = 0xFFFFFFFFUL }; + + typedef Bu::Array BlockArray; + class Stream + { + public: + int iId; + int iSize; + BlockArray aBlocks; + }; + typedef Bu::Array StreamArray; + + class Block + { + public: + char *pData; + bool bChanged; + int iBlockIndex; + }; void updateHeader(); int findEmptyBlock(); + /** + *@todo Change this to use a binary search, it's nicer. + */ + Stream *findStream( int iId ); + + Block *getBlock( int iBlock ); + void releaseBlock( Block *pBlock ); + private: Bu::Stream &sStore; int iBlockSize; int iBlocks; int iUsed; Bu::BitString bsBlockUsed; - typedef Bu::Array BlockArray; - class Stream - { - public: - int iId; - int iSize; - BlockArray aBlocks; - }; - typedef Bu::Array StreamArray; StreamArray aStreams; }; }; diff --git a/src/myriadstream.cpp b/src/myriadstream.cpp index 04f98ed..0e6fc89 100644 --- a/src/myriadstream.cpp +++ b/src/myriadstream.cpp @@ -6,19 +6,22 @@ */ #include "bu/myriadstream.h" +#include "bu/sio.h" + +using Bu::sio; +using Bu::Fmt; #include -Bu::MyriadStream::MyriadStream( Myriad &rMyriad, uint32_t uStream ) : +Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad, + Bu::Myriad::Stream *pStream ) : rMyriad( rMyriad ), - uStream( uStream ), + pStream( pStream ), pCurBlock( NULL ), - uCurBlock( uStream ), - uSize( 0 ), - uBlockSize( rMyriad.iBlockSize ), - uPos( 0 ) + iPos( 0 ) { - //printf("MyriadStream::allocated\n"); + sio << "MyriadStream: Created, iId=" << pStream->iId << ", iSize=" + << pStream->iSize << sio.nl; //pCurBlock = rMyriad.newBlock(); //rMyriad.getBlock( uStream, pCurBlock ); //uSize = pCurBlock->uBytesUsed; @@ -26,7 +29,8 @@ Bu::MyriadStream::MyriadStream( Myriad &rMyriad, uint32_t uStream ) : Bu::MyriadStream::~MyriadStream() { - //printf("Destroying stream?\n"); + if( pCurBlock ) + rMyriad.releaseBlock( pCurBlock ); //rMyriad.updateStreamSize( uStream, uSize ); //rMyriad.deleteBlock( pCurBlock ); } @@ -37,57 +41,54 @@ void Bu::MyriadStream::close() size_t Bu::MyriadStream::read( void *pBuf, size_t nBytes ) { - /* - if( nBytes == 0 ) + sio << "MyriadStream: Read: Started, asked to read " << nBytes << "b." + << sio.nl; + if( nBytes <= 0 ) return 0; - if( nBytes + uPos > uSize ) - nBytes = uSize - uPos; - if( (uPos%uBlockSize)+nBytes < uBlockSize ) + if( nBytes > pStream->iSize-iPos ) + nBytes = pStream->iSize-iPos; + int iLeft = nBytes; + sio << "MyriadStream: Read: Started, going to read " << nBytes << "b." + << sio.nl; + if( pCurBlock == NULL ) { - size_t iRead = nBytes; - if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) ) - iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); - memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead ); - //printf("read buffill: %ub, %u-%u/%u -> %d-%d/%d (a:%u)", - // iRead, uPos, uPos+iRead-1, uSize, 0, iRead-1, nBytes, uCurBlock ); - uPos += iRead; - //printf(" -- %u\n", uPos%uBlockSize ); - //printf("ra: block %u = %ub:%u (%ub total)\n", - // uCurBlock, uPos, nBytes, uSize ); - - // This can't happen, if we're right on a boundery, it goes to the - // other case - //if( uPos%uBlockSize == 0 ) - // uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock, false ); - return iRead; + sio << "MyriadStream: Read: No block loaded, loading initial block." + << sio.nl; + pCurBlock = rMyriad.getBlock( + pStream->aBlocks[iPos/rMyriad.iBlockSize] + ); } - else + while( iLeft > 0 ) { - size_t nTotal = 0; - for(;;) + int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize]; + if( pCurBlock->iBlockIndex != iCurBlock ) { - uint32_t iRead = nBytes; - if( iRead > uBlockSize-(uPos%uBlockSize) ) - iRead = uBlockSize-(uPos%uBlockSize); - if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) ) - iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); - memcpy( ((char *)pBuf)+nTotal, - pCurBlock->pData+(uPos%uBlockSize), iRead ); - //printf(" read buffill: %ub, %u-%u/%u -> %d-%d/%d (b:%u)\n", - // iRead, uPos, uPos+iRead-1, uSize, - // nTotal, nTotal+nBytes-1, nBytes, uCurBlock ); - uPos += iRead; - nBytes -= iRead; - nTotal += iRead; - //printf("rb: block %u = %ub:%u (%ub total)\n", - // uCurBlock, uPos, iRead, uSize ); - if( uPos%uBlockSize == 0 ) - uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock, false ); - if( nBytes == 0 || uPos >= uSize ) - return nTotal; + sio << "MyriadStream: Read: Loading new block " << iCurBlock << "." + << sio.nl; + rMyriad.releaseBlock( pCurBlock ); + pCurBlock = rMyriad.getBlock( iCurBlock ); } - }*/ - return 0; + + int iAmnt = Bu::min( + rMyriad.iBlockSize - iPos%rMyriad.iBlockSize, + iLeft + ); +// if( iLeft > iAmnt ) +// iAmnt = iLeft; + sio << "MyriadStream: Read: Copying out bytes: " + << (iPos%rMyriad.iBlockSize) << " - " + << iAmnt + << ", " << iLeft << "b left." << sio.nl; + memcpy( + pBuf, + pCurBlock->pData+iPos%rMyriad.iBlockSize, + iAmnt + ); + iPos += iAmnt; + pBuf = &((char *)pBuf)[iAmnt]; + iLeft -= iAmnt; + } + return nBytes; } size_t Bu::MyriadStream::write( const void *pBuf, size_t nBytes ) @@ -101,22 +102,22 @@ size_t Bu::MyriadStream::write( const void *pBuf, size_t nBytes ) uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock ); } */ /* - if( (uPos%uBlockSize)+nBytes < uBlockSize ) + if( (iPos%uBlockSize)+nBytes < uBlockSize ) { - //printf("wa: %u:%u:%u:%u -> ", uPos, uPos%uBlockSize, uSize, pCurBlock->uBytesUsed ); - memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes ); + //printf("wa: %u:%u:%u:%u -> ", iPos, iPos%uBlockSize, uSize, pCurBlock->uBytesUsed ); + memcpy( pCurBlock->pData+(iPos%uBlockSize), pBuf, nBytes ); //printf("write buffill: %ub, %u-%u/%u -> %d-%d/%d (a:%u:%u)\n", // nBytes, 0, nBytes-1, nBytes, - // uPos, uPos+nBytes-1, uSize, uCurBlock, + // iPos, iPos+nBytes-1, uSize, uCurBlock, // pCurBlock->uBytesUsed ); - if( (uPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed ) - pCurBlock->uBytesUsed = (uPos%uBlockSize)+nBytes; + if( (iPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed ) + pCurBlock->uBytesUsed = (iPos%uBlockSize)+nBytes; rMyriad.setBlock( uCurBlock, pCurBlock ); - uPos += nBytes; - if( uPos > uSize ) - uSize = uPos; + iPos += nBytes; + if( iPos > uSize ) + uSize = iPos; //printf("block %u = %ub (%ub total) %d:%u\n", - // uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, uPos ); + // uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, iPos ); return nBytes; } else @@ -124,27 +125,27 @@ size_t Bu::MyriadStream::write( const void *pBuf, size_t nBytes ) size_t nTotal = 0; for(;;) { - uint32_t uNow = uBlockSize-(uPos%uBlockSize); - //printf("uNow: %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, uPos, uBlockSize, nBytes ); + uint32_t uNow = uBlockSize-(iPos%uBlockSize); + //printf("uNow: %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, iPos, uBlockSize, nBytes ); if( nBytes < uNow ) uNow = nBytes; - memcpy( pCurBlock->pData+(uPos%uBlockSize), + memcpy( pCurBlock->pData+(iPos%uBlockSize), &((char *)pBuf)[nTotal], uNow ); //printf("write buffill: %ub, %u-%u/%u -> %d-%d/%d (b:%u:%u)\n", // uNow, nTotal, nTotal+uNow-1, nBytes, - // uPos, uPos+uNow-1, uSize, uCurBlock, pCurBlock->uBytesUsed ); - if( (uPos%uBlockSize)+uNow > pCurBlock->uBytesUsed ) - pCurBlock->uBytesUsed = (uPos%uBlockSize)+uNow; + // iPos, iPos+uNow-1, uSize, uCurBlock, pCurBlock->uBytesUsed ); + if( (iPos%uBlockSize)+uNow > pCurBlock->uBytesUsed ) + pCurBlock->uBytesUsed = (iPos%uBlockSize)+uNow; rMyriad.setBlock( uCurBlock, pCurBlock ); - uPos += uNow; - if( uPos > uSize ) - uSize = uPos; + iPos += uNow; + if( iPos > uSize ) + uSize = iPos; nTotal += uNow; nBytes -= uNow; //printf("wb: block %u = %ub (%ub total)\n", // uCurBlock, pCurBlock->uBytesUsed, uSize ); //if( pCurBlock->uBytesUsed == uBlockSize ) - if( uPos%uBlockSize == 0 ) + if( iPos%uBlockSize == 0 ) uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock ); if( nBytes == 0 ) return nTotal; @@ -155,27 +156,27 @@ size_t Bu::MyriadStream::write( const void *pBuf, size_t nBytes ) long Bu::MyriadStream::tell() { - return uPos; + return iPos; } void Bu::MyriadStream::seek( long offset ) { - uPos += offset; + iPos += offset; } void Bu::MyriadStream::setPos( long pos ) { - uPos = pos; + iPos = pos; } void Bu::MyriadStream::setPosEnd( long pos ) { - uPos = uSize-pos-1; + iPos = pStream->iSize-pos-1; } bool Bu::MyriadStream::isEos() { - return true; + return iPos >= pStream->iSize; } bool Bu::MyriadStream::isOpen() diff --git a/src/myriadstream.h b/src/myriadstream.h index 29ab777..e2e3ba7 100644 --- a/src/myriadstream.h +++ b/src/myriadstream.h @@ -20,7 +20,7 @@ namespace Bu /** * These can only be created by the Myriad class. */ - MyriadStream( Myriad &rMyriad, uint32_t uStream ); + MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream ); public: virtual ~MyriadStream(); @@ -46,12 +46,10 @@ namespace Bu private: Myriad &rMyriad; - uint32_t uStream; - char *pCurBlock; - uint32_t uCurBlock; - uint32_t uSize; - uint32_t uBlockSize; - uint32_t uPos; + Myriad::Stream *pStream; + Myriad::Block *pCurBlock; + int iBlockSize; + int iPos; }; }; diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp index b3067d2..95b4503 100644 --- a/src/tools/myriad.cpp +++ b/src/tools/myriad.cpp @@ -19,6 +19,8 @@ enum Mode { modeCreate, modeInfo, + modeStreamNew, + modeStreamRead, modeNone }; @@ -29,22 +31,32 @@ public: Options( int argc, char *argv[] ) : eMode( modeNone ), iBlockSize( 64 ), - iPreallocate( 0 ) + iPreallocate( 0 ), + iStream( 0 ) { addHelpBanner("Mode of operation:"); - addOption( eMode, 'c', "create", "Create a new NIDS file." ); - addOption( eMode, "info", "Display some info about a NIDS file." ); + addOption( eMode, 'c', "create", + "Create a new Myriad file." ); + addOption( eMode, 'i', "info", + "Display some info about a Myriad file." ); + addOption( eMode, 'n', "new", + "Create a new sub-stream in a Myriad file."); + addOption( eMode, 'r', "read", + "Read a stream from a Myriad file."); addHelpOption(); addHelpBanner("\nGeneral options:"); addOption( iBlockSize, 'b', "block-size", "Set the block size." ); addOption( iPreallocate, 'p', "preallocate", "Number of blocks to preallocate." ); - addOption( sOutput, 'o', "output", "Set the output filename." ); - addOption( sInput, 'i', "input", "Set the input filename." ); + addOption( sFile, 'f', "file", "Set the Myriad filename." ); + addOption( iStream, 's', "stream", "Substream to work with."); + addOption( sSrc, "src", "Source file for copying into a Myriad file."); setOverride( "create", "create" ); setOverride( "info", "info" ); + setOverride( "new", "new" ); + setOverride( "read", "read" ); parse( argc, argv ); } @@ -52,17 +64,22 @@ public: Mode eMode; int iBlockSize; int iPreallocate; - Bu::FString sOutput; - Bu::FString sInput; + int iStream; + Bu::FString sFile; + Bu::FString sSrc; }; Bu::Formatter &operator>>( Bu::Formatter &f, Mode &m ) { Bu::FString sTok = f.readToken(); - if( sTok == "create" || sTok == "c" ) + if( sTok == "create" ) m = modeCreate; else if( sTok == "info" ) m = modeInfo; + else if( sTok == "new" ) + m = modeStreamNew; + else if( sTok == "read" ) + m = modeStreamRead; else m = modeNone; return f; @@ -75,32 +92,87 @@ int main( int argc, char *argv[] ) switch( opts.eMode ) { case modeCreate: - if( !opts.sOutput.isSet() ) + if( !opts.sFile.isSet() ) { - sio << "Please specify an output file to create a stream for." - << sio.nl; + sio << "Please specify a file to create." << sio.nl; return 0; } else { - File fOut( opts.sOutput, File::WriteNew ); - Myriad n( fOut ); - n.initialize( opts.iBlockSize, opts.iPreallocate ); + File fOut( opts.sFile, File::WriteNew ); + Myriad m( fOut ); + m.initialize( opts.iBlockSize, opts.iPreallocate ); } break; case modeInfo: - if( !opts.sInput.isSet() ) + if( !opts.sFile.isSet() ) { - sio << "Please specify an input file to display info about." - << sio.nl; + sio << "Please specify a file to display info about." << sio.nl; return 0; } else { - File fIn( opts.sInput, File::Read ); - Myriad n( fIn ); - n.initialize(); + File fIn( opts.sFile, File::Read ); + Myriad m( fIn ); + m.initialize(); + } + break; + + case modeStreamNew: + if( !opts.sFile.isSet() ) + { + sio << "Please specify a file manipulate." << sio.nl; + return 0; + } + else + { + File fOut( opts.sFile, File::Write|File::Read ); + Myriad m( fOut ); + m.initialize(); + m.createStream( opts.iPreallocate ); + } + break; + + case modeStreamRead: + if( !opts.sFile.isSet() ) + { + sio << "Please specify a file manipulate." << sio.nl; + return 0; + } + else + { + File fOut( opts.sFile, File::Read ); + Myriad m( fOut ); + m.initialize(); + MyriadStream s = m.openStream( opts.iStream ); + sio << "Stream " << opts.iStream << ":" << sio.nl; + char buf[8]; + int iPos = 0; + while( !s.isEos() ) + { + size_t sAmnt = s.read( buf, 8 ); + sio << Fmt(5) << iPos << ": "; + iPos += sAmnt; + for( size_t j = 0; j < sAmnt; j++ ) + { + sio << Fmt::hex(2) << (int)((unsigned char)buf[j]) + << " "; + } + for( size_t j = sAmnt; j < 8; j++ ) + { + sio << "-- "; + } + sio << "| "; + for( size_t j = 0; j < sAmnt; j++ ) + { + if( buf[j] >= 32 && buf[j] <= 126 ) + sio << buf[j] << " "; + else + sio << " "; + } + sio << sio.nl; + } } break; -- cgit v1.2.3