Nids is even better, all fixed, no problems. And you can define you're own

creator functions for the cache store...soon, you'll also be able to define
you're own loader/writer functions, but the default will still work exactly
like this.

I also did more work on nidstool, I think I may actually have to create a
tools dir that will just compile some executables for the libbu++ root, because
this thing is handy.  You can get info on the system, trace streams' blocks,
and I'm working on an analysis function that will help you figure out how to
optomize your nids files.  Plus, it'll have a function soon for re-writing a
nids stream, which will let you change the block size, defragment, and remove
unused blocks.

4 files changed, 114 insertions, 38 deletions

diff --git a/src/cachestorenids.h b/src/cachestorenids.h
index 9288008..a5fa402 100644
--- a/src/cachestorenids.h
+++ b/src/cachestorenids.h
@@ -9,14 +9,18 @@
 
 #include "bu/file.h"
 
-static int iCnt = 0;
-
 namespace Bu
 {
         template<class obtype, class keytype>
         keytype __cacheGetKey( const obtype *pObj );
 
         template<class obtype, class keytype>
+        obtype *__cacheStoreNidsAlloc( const keytype &key )
+        {
+                return new obtype();
+        }
+
+        template<class obtype, class keytype>
         class CacheStoreNids : public CacheStore<obtype, keytype>
         {
         public:
@@ -55,7 +59,7 @@ namespace Bu
                         int iStream = hId.get( key );
                         NidsStream ns = nStore.openStream( iStream );
                         Bu::Archive ar( ns, Bu::Archive::load );
-                        obtype *pOb = new obtype();
+                        obtype *pOb = __cacheStoreNidsAlloc<obtype, keytype>( key );
                         ar >> (*pOb);
                         return pOb;
                 }
@@ -76,8 +80,7 @@ namespace Bu
                         hId.insert( key, iStream );
                         NidsStream ns = nStore.openStream( iStream );
                         Bu::Archive ar( ns, Bu::Archive::save );
-                        obtype *pOb = new obtype();
-                        ar << (*pOb);
+                        ar << (*pSrc);
                         return key;
                 }
 
diff --git a/src/nids.cpp b/src/nids.cpp
index 23c11b0..26a7964 100644
--- a/src/nids.cpp
+++ b/src/nids.cpp
@@ -104,7 +104,7 @@ void Bu::Nids::initialize( int iBlockSize, int iPreAllocate )
 
         Block *block = (Block *)new char[iBlockSize];
         memset( block, 0, iBlockSize );
-        block->uFirstBlock = block->uNextBlock = block->uPrevBlock = blockUnused;
+        block->uFirstBlock = block->uNextBlock /*=block->uPrevBlock*/ = blockUnused;
         for( int j = 0; j < iPreAllocate; j++ )
         {
                 sStore.write( block, iBlockSize );
@@ -122,9 +122,9 @@ void Bu::Nids::updateHeader()
 }
 
 void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
-        uint32_t uPrevBlock, bool bNew )
+        /*uint32_t uPrevBlock,*/ bool bNew )
 {
-        Block b = { uPos, blockUnused, uPrevBlock, 0, 0, { } };
+        Block b = { uPos, blockUnused, /*uPrevBlock, 0,*/ 0, { } };
         if( uFirstBlock != blockUnused )
                 b.uFirstBlock = uFirstBlock;
         bsBlockUsed.setBit( uPos );
@@ -142,27 +142,27 @@ void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
         iUsed++;
 }
 
-uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock,
+uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, /*uint32_t uPrevBlock,*/
         int /*iPreAllocate*/ )
 {
         for( int j = 0; j < iBlocks; j++ )
         {
                 if( !bsBlockUsed.getBit( j ) )
                 {
-                        initBlock( j, uFirstBlock, uPrevBlock );
+                        initBlock( j, uFirstBlock/*, uPrevBlock*/ );
                         return j;
                 }
         }
         // Oh, we don't have any blocks left...allocate a new one.
         iBlocks++;
         bsBlockUsed.setSize( iBlocks, false );
-        initBlock( iBlocks-1, uFirstBlock, uPrevBlock, true );
+        initBlock( iBlocks-1, uFirstBlock/*, uPrevBlock*/, true );
         return iBlocks-1;
 }
 
 int Bu::Nids::createStream( int iPreAllocate )
 {
-        return createBlock( blockUnused, blockUnused, iPreAllocate );
+        return createBlock( blockUnused, /*blockUnused,*/ iPreAllocate );
 }
 
 void Bu::Nids::deleteStream( int /*iID*/ )
@@ -224,7 +224,7 @@ void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize )
 {
         if( !sStore.canWrite() )
                 return;
-        sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 );
+        sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*2 );
         sStore.write( &uSize, 4 );
 }
 
diff --git a/src/nids.h b/src/nids.h
index 63d7061..262d9c1 100644
--- a/src/nids.h
+++ b/src/nids.h
@@ -75,9 +75,9 @@ namespace Bu
                 {
                         uint32_t uFirstBlock;
                         uint32_t uNextBlock;
-                        uint32_t uPrevBlock;
+                        // uint32_t uPrevBlock;
                         uint32_t uBytesUsed;
-                        uint32_t uReserved;
+                        // uint32_t uReserved;
                         unsigned char pData[0];
                 } Block;
 
@@ -87,8 +87,8 @@ namespace Bu
                 };
 
                 void initBlock( uint32_t uPos, uint32_t uFirstBlock,
-                        uint32_t uPrevBlock, bool bNew=false );
-                uint32_t createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock,
+                        /*uint32_t uPrevBlock,*/ bool bNew=false );
+                uint32_t createBlock( uint32_t uFirstBlock, /*uint32_t uPrevBlock,*/
                         int iPreAllocate=1 );
                 void getBlock( uint32_t uIndex, struct Nids::Block *pBlock );
                 void setBlock( uint32_t uIndex, struct Nids::Block *pBlock );
diff --git a/src/tests/nidstool.cpp b/src/tests/nidstool.cpp
index 546534e..1becba5 100644
--- a/src/tests/nidstool.cpp
+++ b/src/tests/nidstool.cpp
@@ -5,6 +5,13 @@
 
 #include <stdlib.h>
 
+typedef struct Block
+{
+        uint32_t uFirstBlock;
+        uint32_t uNextBlock;
+        uint32_t uBytesUsed;
+} Block;
+
 class Param : public Bu::ParamProc
 {
 public:
@@ -14,7 +21,9 @@ public:
                 addParam("info", 'i', mkproc(Param::procInfo),
                         "Print some info about the file.");
                 addParam("dump", 'd', mkproc(Param::procDump),
-                        "Dump a stream to a file");
+                        "Dump a stream to a file.");
+                addParam("analyze", 'a', mkproc(Param::procAnalyze),
+                        "Analyze a nids file.");
                 addParam("help", 'h', mkproc(Bu::ParamProc::help), "This help.");
                 process( argc, argv );
         }
@@ -23,6 +32,20 @@ public:
         {
         }
 
+        void printInfo( Bu::Nids &n )
+        {
+                printf("File info:\n");
+                printf("  Header overhead: %db\n", n.getBlockStart() );
+                printf("  Block size:      %db\n", n.getBlockSize() );
+                printf("  Block count:     %d\n", n.getNumBlocks() );
+                printf("  Blocks used:     %d (%d%%)\n", n.getNumUsedBlocks(),
+                        n.getNumUsedBlocks()*100/n.getNumBlocks() );
+                printf("  Block overhead:  %db\n", n.getBlockOverhead() );
+                printf("  Block storage:   %db (%d%%)\n",
+                        n.getBlockSize()-n.getBlockOverhead(),
+                        (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() );
+        }
+
         int procInfo( int argc, char *argv[] )
         {
                 if( argc < 1 )
@@ -35,27 +58,10 @@ public:
                 Bu::Nids n( fIn );
                 n.initialize();
 
-                printf("Block size:     %db\n", n.getBlockSize() );
-                printf("Block count:    %d\n", n.getNumBlocks() );
-                printf("Blocks used:    %d (%d%%)\n", n.getNumUsedBlocks(),
-                        n.getNumUsedBlocks()*100/n.getNumBlocks() );
-                printf("Block start:    %db\n", n.getBlockStart() );
-                printf("Block overhead: %db\n", n.getBlockOverhead() );
-                printf("Block storage:  %db (%d%%)\n",
-                        n.getBlockSize()-n.getBlockOverhead(),
-                        (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() );
+                printInfo( n );
 
                 if( argc >= 2 )
                 {
-                        typedef struct Block
-                        {
-                                uint32_t uFirstBlock;
-                                uint32_t uNextBlock;
-                                uint32_t uPrevBlock;
-                                uint32_t uBytesUsed;
-                                uint32_t uReserved;
-                        } Block;
-
                         uint32_t uStream = strtoul( argv[1], NULL, 0 );
                         uint32_t uBlock = uStream;
 
@@ -65,8 +71,8 @@ public:
                         {
                                 fIn.setPos( n.getBlockStart()+n.getBlockSize()*uBlock );
                                 fIn.read( &b, sizeof(Block) );
-                                printf("Stream %u:  block %u, next %u, prev %u, %ub used.\n",
-                                        uStream, uBlock, b.uNextBlock, b.uPrevBlock, b.uBytesUsed
+                                printf("Stream %u:  block %u, next %u, %ub used.\n",
+                                        uStream, uBlock, b.uNextBlock, b.uBytesUsed
                                         );
                                 if( b.uNextBlock == 0xFFFFFFFFUL )
                                         break;
@@ -112,6 +118,73 @@ public:
                 return 3;
         }
 
+        int procAnalyze( int argc, char *argv[] )
+        {
+                if( argc < 1 )
+                {
+                        printf("You must provide a file name.\n");
+                        exit( 1 );
+                }
+
+                Bu::File fIn( argv[0], Bu::File::Read );
+                Bu::Nids n( fIn );
+                n.initialize();
+
+                printInfo( n );
+
+                int iStreamCnt = 0;
+                int iStreamTotal = 0;
+                int iOneBlock = 0;
+                uint32_t iLargest = 0;
+                uint32_t iSmallest = 0;
+                int iWaste = 0;
+                int iUsable = n.getBlockSize()-n.getBlockOverhead();
+                Block b;
+                for( int j = 0; j < n.getNumBlocks(); j++ )
+                {
+                        fIn.setPos( n.getBlockStart()+n.getBlockSize()*j );
+                        fIn.read( &b, sizeof(Block) );
+                        if( b.uFirstBlock != (uint32_t)j )
+                                continue;
+                        
+                        iStreamCnt++;
+                        iStreamTotal += b.uBytesUsed;
+
+                        if( b.uNextBlock == 0xFFFFFFFFUL )
+                        {
+                                iOneBlock++;
+                                iWaste += iUsable - b.uBytesUsed;
+                        }
+                        else
+                        {
+                                iWaste += iUsable - (b.uBytesUsed%iUsable);
+                        }
+                        
+                        if( j == 0 )
+                        {
+                                iSmallest = iLargest = b.uBytesUsed;
+                        }
+                        else
+                        {
+                                if( iLargest < b.uBytesUsed )
+                                        iLargest = b.uBytesUsed;
+                                if( iSmallest > b.uBytesUsed )
+                                        iSmallest = b.uBytesUsed;
+                        }
+                }
+                printf("Steam analysis:\n");
+                printf("  Stream count:          %d\n", iStreamCnt );
+                printf("  Stream size:           %db/%db/%db (min/avr/max)\n",
+                        iSmallest, iStreamTotal/iStreamCnt, iLargest );
+                printf("  One-block streams:     %d (%d%%)\n",
+                        iOneBlock, iOneBlock*100/iStreamCnt );
+                printf("  Total wasted space:    %db (%d%%)\n",
+                        iWaste, iWaste*100/iStreamTotal );
+                printf("  Avr blocks-per-stream: %f%%\n",
+                        (float)n.getNumBlocks()/(float)iStreamCnt );
+
+                return 1;
+        }
 };