From 9098237f5bb16b204a5ea999b702e5eb170f68ac Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 27 Jan 2009 15:25:46 +0000 Subject: Corrected some larger read/write issues in corner cases that I hit suprisingly often within nids. There's still a problem somewhere, but I'll find it. Also, even after having the file class canRead and canWrite functions work properly, and using them before trying to write to a nids to update info, we never ever write anything, so something is still wrong there. For now, all utilities that open a nids stream read-only will crash when it closes. Pretty minor really. --- src/cachestorenids.h | 10 ++++ src/file.cpp | 8 ++++ src/nids.cpp | 40 +++++++++++++++- src/nids.h | 7 +++ src/nidsstream.cpp | 65 +++++++++++++++++++------- src/tests/nidstool.cpp | 124 +++++++++++++++++++++++++++++++++++++++++++++++++ src/tests/rh.cpp | 52 +++++++++++++++++++++ 7 files changed, 286 insertions(+), 20 deletions(-) create mode 100644 src/tests/nidstool.cpp create mode 100644 src/tests/rh.cpp (limited to 'src') diff --git a/src/cachestorenids.h b/src/cachestorenids.h index 62d1555..bd0fcc7 100644 --- a/src/cachestorenids.h +++ b/src/cachestorenids.h @@ -7,6 +7,10 @@ #include "bu/nidsstream.h" #include "bu/cachestore.h" +#include "bu/file.h" + +static int iCnt = 0; + namespace Bu { template @@ -44,6 +48,12 @@ namespace Bu NidsStream ns = nStore.openStream( 0 ); Bu::Archive ar( ns, Bu::Archive::save ); ar << hId; + + Bu::FString sName; + sName.format("hash-%d.%02d", time( NULL ), iCnt++ ); + Bu::File sTmp( sName, Bu::File::Write|Bu::File::Create ); + Bu::Archive artmp( sTmp, Bu::Archive::save ); + artmp << hId; } virtual obtype *load( const keytype &key ) diff --git a/src/file.cpp b/src/file.cpp index 7c18a06..20ff5c9 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -110,12 +110,20 @@ bool Bu::File::isEOS() bool Bu::File::canRead() { +#ifdef WIN32 return true; +#else + return (fcntl( fd, F_GETFL, 0 )&O_RDONLY) == O_RDONLY; +#endif } bool Bu::File::canWrite() { +#ifdef WIN32 return true; +#else + return (fcntl( fd, F_GETFL, 0 )&O_WRONLY) == O_WRONLY; +#endif } bool Bu::File::isReadable() diff --git a/src/nids.cpp b/src/nids.cpp index 06895ac..00d5f2b 100644 --- a/src/nids.cpp +++ b/src/nids.cpp @@ -21,18 +21,19 @@ Bu::Nids::Nids( Bu::Stream &sStore ) : sStore( sStore ), iBlockSize( 0 ), iBlocks( 0 ), - iBlockStart( -1 ) + iBlockStart( -1 ), + iUsed( 0 ) { } Bu::Nids::~Nids() { + updateHeader(); } void Bu::Nids::initialize() { unsigned char buf[4]; - int iUsed; if( sStore.read( buf, 4 ) < 4 ) throw NidsException("Input stream appears to be empty."); if( memcmp( buf, NIDS_MAGIC_CODE, 4 ) ) @@ -111,6 +112,15 @@ void Bu::Nids::initialize( int iBlockSize, int iPreAllocate ) delete[] (char *)block; } +void Bu::Nids::updateHeader() +{ +// if( !sStore.canWrite() ) +// return; + sStore.setPos( 10 ); // Skip the magic number, version, bpi, block size + sStore.write( &iBlocks, 4 ); + sStore.write( &iUsed, 4 ); +} + void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock, uint32_t uPrevBlock, bool bNew ) { @@ -129,6 +139,7 @@ void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock, sStore.write( buf, iSize ); delete[] buf; } + iUsed++; } uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock, @@ -171,6 +182,27 @@ int Bu::Nids::getBlockSize() { return iBlockSize; } + +int Bu::Nids::getNumBlocks() +{ + return iBlocks; +} + +int Bu::Nids::getNumUsedBlocks() +{ + return iUsed; +} + +int Bu::Nids::getBlockStart() +{ + return iBlockStart; +} + +int Bu::Nids::getBlockOverhead() +{ + return sizeof(Block); +} + /* void Bu::Nids::extendStream( int iID, int iBlockCount ) { @@ -190,6 +222,8 @@ void Bu::Nids::setBlock( uint32_t uIndex, Bu::Nids::Block *pBlock ) void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize ) { +// if( !sStore.canWrite() ) +// return; sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 ); sStore.write( &uSize, 4 ); } @@ -206,6 +240,8 @@ uint32_t Bu::Nids::getNextBlock( uint32_t uIndex, sStore.setPos( iBlockStart + (iBlockSize*uIndex)+1*4 ); sStore.write( &uNew, 4 ); getBlock( uNew, pBlock ); + printf("Allocated new block (%u) for stream %u.\n", + uNew, pBlock->uFirstBlock ); } else { diff --git a/src/nids.h b/src/nids.h index d297fa1..63d7061 100644 --- a/src/nids.h +++ b/src/nids.h @@ -65,6 +65,10 @@ namespace Bu NidsStream openStream( int iID ); int getBlockSize(); + int getNumBlocks(); + int getNumUsedBlocks(); + int getBlockStart(); + int getBlockOverhead(); private: typedef struct Block @@ -92,6 +96,8 @@ namespace Bu uint32_t getNextBlock( uint32_t uIndex, struct Nids::Block *pBlock, bool bCreate=true); + void updateHeader(); + // Block allocation routines Block *newBlock(); void deleteBlock( Block *pBlock ); @@ -101,6 +107,7 @@ namespace Bu int iBlockSize; int iBlocks; int iBlockStart; + int iUsed; Bu::BitString bsBlockUsed; }; }; diff --git a/src/nidsstream.cpp b/src/nidsstream.cpp index 4f692c6..fff73f6 100644 --- a/src/nidsstream.cpp +++ b/src/nidsstream.cpp @@ -45,15 +45,27 @@ void Bu::NidsStream::close() size_t Bu::NidsStream::read( void *pBuf, size_t nBytes ) { - if( uPos%uBlockSize+nBytes < uBlockSize ) + if( nBytes == 0 ) + return 0; + if( nBytes + uPos > uSize ) + nBytes = uSize - uPos; + if( (uPos%uBlockSize)+nBytes < uBlockSize ) { size_t iRead = nBytes; if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) ) iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead ); + //printf("buffill: %ub, %u-%u/%u -> %d-%d/%d (a:%u)", + // iRead, uPos, uPos+iRead-1, uSize, 0, iRead-1, nBytes, uCurBlock ); uPos += iRead; - //printf("a: block %u = %ub (%ub total)\n", - // uCurBlock, pCurBlock->uBytesUsed, uSize ); + //printf(" -- %u\n", uPos%uBlockSize ); + //printf("ra: block %u = %ub:%u (%ub total)\n", + // uCurBlock, uPos, nBytes, uSize ); + + // This can't happen, if we're right on a boundery, it goes to the + // other case + //if( uPos%uBlockSize == 0 ) + // uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false ); return iRead; } else @@ -68,15 +80,18 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes ) iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); memcpy( ((char *)pBuf)+nTotal, pCurBlock->pData+(uPos%uBlockSize), iRead ); + //printf("buffill: %ub, %u-%u/%u -> %d-%d/%d (b:%u)\n", + // iRead, uPos, uPos+iRead-1, uSize, + // nTotal, nTotal+nBytes-1, nBytes, uCurBlock ); uPos += iRead; nBytes -= iRead; nTotal += iRead; - //printf("r: block %u = %ub/%ub (%ub total)\n", - // uCurBlock, iRead, pCurBlock->uBytesUsed, uSize ); - if( nBytes == 0 || uPos == uSize ) - return nTotal; - if( nTotal%uBlockSize == 0 ) + //printf("rb: block %u = %ub:%u (%ub total)\n", + // uCurBlock, uPos, iRead, uSize ); + if( uPos%uBlockSize == 0 ) uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false ); + if( nBytes == 0 || uPos >= uSize ) + return nTotal; } } return 0; @@ -84,15 +99,26 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes ) size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes ) { - if( uPos%uBlockSize+nBytes < uBlockSize ) + if( nBytes == 0 ) + return 0; +/* if( pCurBlock->uBytesUsed >= uBlockSize ) { + // We're at the end of our current block, allocate another before we do + // anything. + uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock ); + } */ + if( (uPos%uBlockSize)+nBytes < uBlockSize ) + { + //printf("wa: %u:%u:%u:%u -> ", uPos, uPos%uBlockSize, uSize, pCurBlock->uBytesUsed ); memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes ); - pCurBlock->uBytesUsed += nBytes; + if( (uPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed ) + pCurBlock->uBytesUsed = (uPos%uBlockSize)+nBytes; rNids.setBlock( uCurBlock, pCurBlock ); uPos += nBytes; - uSize += nBytes; - //printf("a: block %u = %ub (%ub total)\n", - // uCurBlock, pCurBlock->uBytesUsed, uSize ); + if( uPos > uSize ) + uSize = uPos; + //printf("block %u = %ub (%ub total) %d:%u\n", + // uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, uPos ); return nBytes; } else @@ -101,22 +127,25 @@ size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes ) for(;;) { uint32_t uNow = uBlockSize-(uPos%uBlockSize); + //printf("uNow: %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, uPos, uBlockSize, nBytes ); if( nBytes < uNow ) uNow = nBytes; memcpy( pCurBlock->pData+(uPos%uBlockSize), &((char *)pBuf)[nTotal], uNow ); - pCurBlock->uBytesUsed += uNow; + if( (uPos%uBlockSize)+uNow > pCurBlock->uBytesUsed ) + pCurBlock->uBytesUsed = (uPos%uBlockSize)+uNow; rNids.setBlock( uCurBlock, pCurBlock ); - uSize += uNow; uPos += uNow; + if( uPos > uSize ) + uSize = uPos; nTotal += uNow; nBytes -= uNow; - //printf("b: block %u = %ub (%ub total)\n", + //printf("wb: block %u = %ub (%ub total)\n", // uCurBlock, pCurBlock->uBytesUsed, uSize ); - if( nBytes == 0 ) - return nTotal; if( pCurBlock->uBytesUsed == uBlockSize ) uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock ); + if( nBytes == 0 ) + return nTotal; } } } diff --git a/src/tests/nidstool.cpp b/src/tests/nidstool.cpp new file mode 100644 index 0000000..546534e --- /dev/null +++ b/src/tests/nidstool.cpp @@ -0,0 +1,124 @@ +#include "bu/file.h" +#include "bu/nids.h" +#include "bu/nidsstream.h" +#include "bu/paramproc.h" + +#include + +class Param : public Bu::ParamProc +{ +public: + Param( int argc, char *argv[] ) + { + addHelpBanner("nidstool - Do stuff with nids files.\n\n"); + addParam("info", 'i', mkproc(Param::procInfo), + "Print some info about the file."); + addParam("dump", 'd', mkproc(Param::procDump), + "Dump a stream to a file"); + addParam("help", 'h', mkproc(Bu::ParamProc::help), "This help."); + process( argc, argv ); + } + + virtual ~Param() + { + } + + int procInfo( int argc, char *argv[] ) + { + if( argc < 1 ) + { + printf("You must provide a file name.\n"); + exit( 1 ); + } + + Bu::File fIn( argv[0], Bu::File::Read ); + Bu::Nids n( fIn ); + n.initialize(); + + printf("Block size: %db\n", n.getBlockSize() ); + printf("Block count: %d\n", n.getNumBlocks() ); + printf("Blocks used: %d (%d%%)\n", n.getNumUsedBlocks(), + n.getNumUsedBlocks()*100/n.getNumBlocks() ); + printf("Block start: %db\n", n.getBlockStart() ); + printf("Block overhead: %db\n", n.getBlockOverhead() ); + printf("Block storage: %db (%d%%)\n", + n.getBlockSize()-n.getBlockOverhead(), + (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() ); + + if( argc >= 2 ) + { + typedef struct Block + { + uint32_t uFirstBlock; + uint32_t uNextBlock; + uint32_t uPrevBlock; + uint32_t uBytesUsed; + uint32_t uReserved; + } Block; + + uint32_t uStream = strtoul( argv[1], NULL, 0 ); + uint32_t uBlock = uStream; + + Block b; + + for(;;) + { + fIn.setPos( n.getBlockStart()+n.getBlockSize()*uBlock ); + fIn.read( &b, sizeof(Block) ); + printf("Stream %u: block %u, next %u, prev %u, %ub used.\n", + uStream, uBlock, b.uNextBlock, b.uPrevBlock, b.uBytesUsed + ); + if( b.uNextBlock == 0xFFFFFFFFUL ) + break; + uBlock = b.uNextBlock; + } + printf("Stream End.\n"); + + return 2; + } + + return 1; + } + + int procDump( int argc, char *argv[] ) + { + if( argc < 3 ) + { + printf("You must provide a nids file, a stream id, and an output " + "file.\n"); + exit( 1 ); + } + + Bu::File fIn( argv[0], Bu::File::Read ); + Bu::Nids n( fIn ); + n.initialize(); + + int iStream = strtol( argv[1], NULL, 0 ); + Bu::NidsStream sIn = n.openStream( iStream ); + + Bu::File fOut( argv[2], Bu::File::Write|Bu::File::Create ); + int iTotal = 0; + char buf[100]; + for(;;) + { + int iRead = sIn.read( buf, 100 ); + iTotal += fOut.write( buf, iRead ); + if( iRead < 100 ) + break; + } + + printf("Wrote %db from stream %d in %s to %s.\n", + iTotal, iStream, argv[0], argv[2] ); + return 3; + } + +}; + + +int main( int argc, char *argv[] ) +{ + Param p( argc, argv ); + + return 0; +} + diff --git a/src/tests/rh.cpp b/src/tests/rh.cpp new file mode 100644 index 0000000..70abcb7 --- /dev/null +++ b/src/tests/rh.cpp @@ -0,0 +1,52 @@ +#include "bu/file.h" +#include "bu/hash.h" +#include "bu/archive.h" +#include "bu/fstring.h" +#include "bu/nids.h" +#include "bu/nidsstream.h" + +int main( int argc, char *argv[] ) +{ + if( argv[1][0] == 'r' ) + { + typedef Bu::Hash Hsh; + + Bu::File fIn( argv[2], Bu::File::Read ); + Bu::Archive ar( fIn, Bu::Archive::load ); + + Hsh h; + ar >> h; + + printf("Read %d.\n", h.getSize() ); + for( Hsh::iterator i = h.begin(); i != h.end(); i++ ) + { + printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() ); + } + + printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() ); + } + else if( argv[1][0] == 'n' ) + { + typedef Bu::Hash Hsh; + + Bu::File fIn( argv[2], Bu::File::Read ); + Bu::Nids n( fIn ); + n.initialize(); + Bu::NidsStream sIn = n.openStream( 0 ); + Bu::Archive ar( sIn, Bu::Archive::load ); + + Hsh h; + ar >> h; + + printf("Read %d.\n", h.getSize() ); + for( Hsh::iterator i = h.begin(); i != h.end(); i++ ) + { + printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() ); + } + + printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() ); + } + + return 0; +} + -- cgit v1.2.3