From 67ec9d667ab0c3f2258f6f69308d0731e74a74d0 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 27 Jan 2009 21:45:03 +0000 Subject: Nids is even better, all fixed, no problems. And you can define you're own creator functions for the cache store...soon, you'll also be able to define you're own loader/writer functions, but the default will still work exactly like this. I also did more work on nidstool, I think I may actually have to create a tools dir that will just compile some executables for the libbu++ root, because this thing is handy. You can get info on the system, trace streams' blocks, and I'm working on an analysis function that will help you figure out how to optomize your nids files. Plus, it'll have a function soon for re-writing a nids stream, which will let you change the block size, defragment, and remove unused blocks. --- src/cachestorenids.h | 13 +++--- src/nids.cpp | 16 +++---- src/nids.h | 8 ++-- src/tests/nidstool.cpp | 115 ++++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 114 insertions(+), 38 deletions(-) diff --git a/src/cachestorenids.h b/src/cachestorenids.h index 9288008..a5fa402 100644 --- a/src/cachestorenids.h +++ b/src/cachestorenids.h @@ -9,13 +9,17 @@ #include "bu/file.h" -static int iCnt = 0; - namespace Bu { template keytype __cacheGetKey( const obtype *pObj ); + template + obtype *__cacheStoreNidsAlloc( const keytype &key ) + { + return new obtype(); + } + template class CacheStoreNids : public CacheStore { @@ -55,7 +59,7 @@ namespace Bu int iStream = hId.get( key ); NidsStream ns = nStore.openStream( iStream ); Bu::Archive ar( ns, Bu::Archive::load ); - obtype *pOb = new obtype(); + obtype *pOb = __cacheStoreNidsAlloc( key ); ar >> (*pOb); return pOb; } @@ -76,8 +80,7 @@ namespace Bu hId.insert( key, iStream ); NidsStream ns = nStore.openStream( iStream ); Bu::Archive ar( ns, Bu::Archive::save ); - obtype *pOb = new obtype(); - ar << (*pOb); + ar << (*pSrc); return key; } diff --git a/src/nids.cpp b/src/nids.cpp index 23c11b0..26a7964 100644 --- a/src/nids.cpp +++ b/src/nids.cpp @@ -104,7 +104,7 @@ void Bu::Nids::initialize( int iBlockSize, int iPreAllocate ) Block *block = (Block *)new char[iBlockSize]; memset( block, 0, iBlockSize ); - block->uFirstBlock = block->uNextBlock = block->uPrevBlock = blockUnused; + block->uFirstBlock = block->uNextBlock /*=block->uPrevBlock*/ = blockUnused; for( int j = 0; j < iPreAllocate; j++ ) { sStore.write( block, iBlockSize ); @@ -122,9 +122,9 @@ void Bu::Nids::updateHeader() } void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock, - uint32_t uPrevBlock, bool bNew ) + /*uint32_t uPrevBlock,*/ bool bNew ) { - Block b = { uPos, blockUnused, uPrevBlock, 0, 0, { } }; + Block b = { uPos, blockUnused, /*uPrevBlock, 0,*/ 0, { } }; if( uFirstBlock != blockUnused ) b.uFirstBlock = uFirstBlock; bsBlockUsed.setBit( uPos ); @@ -142,27 +142,27 @@ void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock, iUsed++; } -uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock, +uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, /*uint32_t uPrevBlock,*/ int /*iPreAllocate*/ ) { for( int j = 0; j < iBlocks; j++ ) { if( !bsBlockUsed.getBit( j ) ) { - initBlock( j, uFirstBlock, uPrevBlock ); + initBlock( j, uFirstBlock/*, uPrevBlock*/ ); return j; } } // Oh, we don't have any blocks left...allocate a new one. iBlocks++; bsBlockUsed.setSize( iBlocks, false ); - initBlock( iBlocks-1, uFirstBlock, uPrevBlock, true ); + initBlock( iBlocks-1, uFirstBlock/*, uPrevBlock*/, true ); return iBlocks-1; } int Bu::Nids::createStream( int iPreAllocate ) { - return createBlock( blockUnused, blockUnused, iPreAllocate ); + return createBlock( blockUnused, /*blockUnused,*/ iPreAllocate ); } void Bu::Nids::deleteStream( int /*iID*/ ) @@ -224,7 +224,7 @@ void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize ) { if( !sStore.canWrite() ) return; - sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 ); + sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*2 ); sStore.write( &uSize, 4 ); } diff --git a/src/nids.h b/src/nids.h index 63d7061..262d9c1 100644 --- a/src/nids.h +++ b/src/nids.h @@ -75,9 +75,9 @@ namespace Bu { uint32_t uFirstBlock; uint32_t uNextBlock; - uint32_t uPrevBlock; + // uint32_t uPrevBlock; uint32_t uBytesUsed; - uint32_t uReserved; + // uint32_t uReserved; unsigned char pData[0]; } Block; @@ -87,8 +87,8 @@ namespace Bu }; void initBlock( uint32_t uPos, uint32_t uFirstBlock, - uint32_t uPrevBlock, bool bNew=false ); - uint32_t createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock, + /*uint32_t uPrevBlock,*/ bool bNew=false ); + uint32_t createBlock( uint32_t uFirstBlock, /*uint32_t uPrevBlock,*/ int iPreAllocate=1 ); void getBlock( uint32_t uIndex, struct Nids::Block *pBlock ); void setBlock( uint32_t uIndex, struct Nids::Block *pBlock ); diff --git a/src/tests/nidstool.cpp b/src/tests/nidstool.cpp index 546534e..1becba5 100644 --- a/src/tests/nidstool.cpp +++ b/src/tests/nidstool.cpp @@ -5,6 +5,13 @@ #include +typedef struct Block +{ + uint32_t uFirstBlock; + uint32_t uNextBlock; + uint32_t uBytesUsed; +} Block; + class Param : public Bu::ParamProc { public: @@ -14,7 +21,9 @@ public: addParam("info", 'i', mkproc(Param::procInfo), "Print some info about the file."); addParam("dump", 'd', mkproc(Param::procDump), - "Dump a stream to a file"); + "Dump a stream to a file."); + addParam("analyze", 'a', mkproc(Param::procAnalyze), + "Analyze a nids file."); addParam("help", 'h', mkproc(Bu::ParamProc::help), "This help."); process( argc, argv ); } @@ -23,6 +32,20 @@ public: { } + void printInfo( Bu::Nids &n ) + { + printf("File info:\n"); + printf(" Header overhead: %db\n", n.getBlockStart() ); + printf(" Block size: %db\n", n.getBlockSize() ); + printf(" Block count: %d\n", n.getNumBlocks() ); + printf(" Blocks used: %d (%d%%)\n", n.getNumUsedBlocks(), + n.getNumUsedBlocks()*100/n.getNumBlocks() ); + printf(" Block overhead: %db\n", n.getBlockOverhead() ); + printf(" Block storage: %db (%d%%)\n", + n.getBlockSize()-n.getBlockOverhead(), + (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() ); + } + int procInfo( int argc, char *argv[] ) { if( argc < 1 ) @@ -35,27 +58,10 @@ public: Bu::Nids n( fIn ); n.initialize(); - printf("Block size: %db\n", n.getBlockSize() ); - printf("Block count: %d\n", n.getNumBlocks() ); - printf("Blocks used: %d (%d%%)\n", n.getNumUsedBlocks(), - n.getNumUsedBlocks()*100/n.getNumBlocks() ); - printf("Block start: %db\n", n.getBlockStart() ); - printf("Block overhead: %db\n", n.getBlockOverhead() ); - printf("Block storage: %db (%d%%)\n", - n.getBlockSize()-n.getBlockOverhead(), - (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() ); + printInfo( n ); if( argc >= 2 ) { - typedef struct Block - { - uint32_t uFirstBlock; - uint32_t uNextBlock; - uint32_t uPrevBlock; - uint32_t uBytesUsed; - uint32_t uReserved; - } Block; - uint32_t uStream = strtoul( argv[1], NULL, 0 ); uint32_t uBlock = uStream; @@ -65,8 +71,8 @@ public: { fIn.setPos( n.getBlockStart()+n.getBlockSize()*uBlock ); fIn.read( &b, sizeof(Block) ); - printf("Stream %u: block %u, next %u, prev %u, %ub used.\n", - uStream, uBlock, b.uNextBlock, b.uPrevBlock, b.uBytesUsed + printf("Stream %u: block %u, next %u, %ub used.\n", + uStream, uBlock, b.uNextBlock, b.uBytesUsed ); if( b.uNextBlock == 0xFFFFFFFFUL ) break; @@ -112,6 +118,73 @@ public: return 3; } + int procAnalyze( int argc, char *argv[] ) + { + if( argc < 1 ) + { + printf("You must provide a file name.\n"); + exit( 1 ); + } + + Bu::File fIn( argv[0], Bu::File::Read ); + Bu::Nids n( fIn ); + n.initialize(); + + printInfo( n ); + + int iStreamCnt = 0; + int iStreamTotal = 0; + int iOneBlock = 0; + uint32_t iLargest = 0; + uint32_t iSmallest = 0; + int iWaste = 0; + int iUsable = n.getBlockSize()-n.getBlockOverhead(); + Block b; + for( int j = 0; j < n.getNumBlocks(); j++ ) + { + fIn.setPos( n.getBlockStart()+n.getBlockSize()*j ); + fIn.read( &b, sizeof(Block) ); + if( b.uFirstBlock != (uint32_t)j ) + continue; + + iStreamCnt++; + iStreamTotal += b.uBytesUsed; + + if( b.uNextBlock == 0xFFFFFFFFUL ) + { + iOneBlock++; + iWaste += iUsable - b.uBytesUsed; + } + else + { + iWaste += iUsable - (b.uBytesUsed%iUsable); + } + + if( j == 0 ) + { + iSmallest = iLargest = b.uBytesUsed; + } + else + { + if( iLargest < b.uBytesUsed ) + iLargest = b.uBytesUsed; + if( iSmallest > b.uBytesUsed ) + iSmallest = b.uBytesUsed; + } + } + printf("Steam analysis:\n"); + printf(" Stream count: %d\n", iStreamCnt ); + printf(" Stream size: %db/%db/%db (min/avr/max)\n", + iSmallest, iStreamTotal/iStreamCnt, iLargest ); + printf(" One-block streams: %d (%d%%)\n", + iOneBlock, iOneBlock*100/iStreamCnt ); + printf(" Total wasted space: %db (%d%%)\n", + iWaste, iWaste*100/iStreamTotal ); + printf(" Avr blocks-per-stream: %f%%\n", + (float)n.getNumBlocks()/(float)iStreamCnt ); + + return 1; + } }; -- cgit v1.2.3