Corrected some larger read/write issues in corner cases that I hit suprisingly

often within nids.  There's still a problem somewhere, but I'll find it.

Also, even after having the file class canRead and canWrite functions work
properly, and using them before trying to write to a nids to update info, we
never ever write anything, so something is still wrong there.  For now, all
utilities that open a nids stream read-only will crash when it closes.  Pretty
minor really.

7 files changed, 286 insertions, 20 deletions

diff --git a/src/cachestorenids.h b/src/cachestorenids.h
index 62d1555..bd0fcc7 100644
--- a/src/cachestorenids.h
+++ b/src/cachestorenids.h
@@ -7,6 +7,10 @@
 #include "bu/nidsstream.h"
 #include "bu/cachestore.h"
 
+#include "bu/file.h"
+
+static int iCnt = 0;
+
 namespace Bu
 {
         template<class obtype, class keytype>
@@ -44,6 +48,12 @@ namespace Bu
                         NidsStream ns = nStore.openStream( 0 );
                         Bu::Archive ar( ns, Bu::Archive::save );
                         ar << hId;
+
+                        Bu::FString sName;
+                        sName.format("hash-%d.%02d", time( NULL ), iCnt++ );
+                        Bu::File sTmp( sName, Bu::File::Write|Bu::File::Create );
+                        Bu::Archive artmp( sTmp, Bu::Archive::save );
+                        artmp << hId;
                 }
                 
                 virtual obtype *load( const keytype &key )
diff --git a/src/file.cpp b/src/file.cpp
index 7c18a06..20ff5c9 100644
--- a/src/file.cpp
+++ b/src/file.cpp
@@ -110,12 +110,20 @@ bool Bu::File::isEOS()
 
 bool Bu::File::canRead()
 {
+#ifdef WIN32
         return true;
+#else
+        return (fcntl( fd, F_GETFL, 0 )&O_RDONLY) == O_RDONLY;
+#endif
 }
 
 bool Bu::File::canWrite()
 {
+#ifdef WIN32
         return true;
+#else
+        return (fcntl( fd, F_GETFL, 0 )&O_WRONLY) == O_WRONLY;
+#endif
 }
 
 bool Bu::File::isReadable()
diff --git a/src/nids.cpp b/src/nids.cpp
index 06895ac..00d5f2b 100644
--- a/src/nids.cpp
+++ b/src/nids.cpp
@@ -21,18 +21,19 @@ Bu::Nids::Nids( Bu::Stream &sStore ) :
         sStore( sStore ),
         iBlockSize( 0 ),
         iBlocks( 0 ),
-        iBlockStart( -1 )
+        iBlockStart( -1 ),
+        iUsed( 0 )
 {
 }
 
 Bu::Nids::~Nids()
 {
+        updateHeader();
 }
 
 void Bu::Nids::initialize()
 {
         unsigned char buf[4];
-        int iUsed;
         if( sStore.read( buf, 4 ) < 4 )
                 throw NidsException("Input stream appears to be empty.");
         if( memcmp( buf, NIDS_MAGIC_CODE, 4 ) )
@@ -111,6 +112,15 @@ void Bu::Nids::initialize( int iBlockSize, int iPreAllocate )
         delete[] (char *)block;
 }
 
+void Bu::Nids::updateHeader()
+{
+//      if( !sStore.canWrite() )
+//              return;
+        sStore.setPos( 10 ); // Skip the magic number, version, bpi, block size
+        sStore.write( &iBlocks, 4 );
+        sStore.write( &iUsed, 4 );
+}
+
 void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
         uint32_t uPrevBlock, bool bNew )
 {
@@ -129,6 +139,7 @@ void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
                 sStore.write( buf, iSize );
                 delete[] buf;
         }
+        iUsed++;
 }
 
 uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock,
@@ -171,6 +182,27 @@ int Bu::Nids::getBlockSize()
 {
         return iBlockSize;
 }
+
+int Bu::Nids::getNumBlocks()
+{
+        return iBlocks;
+}
+
+int Bu::Nids::getNumUsedBlocks()
+{
+        return iUsed;
+}
+
+int Bu::Nids::getBlockStart()
+{
+        return iBlockStart;
+}
+
+int Bu::Nids::getBlockOverhead()
+{
+        return sizeof(Block);
+}
+
 /*
 void Bu::Nids::extendStream( int iID, int iBlockCount )
 {
@@ -190,6 +222,8 @@ void Bu::Nids::setBlock( uint32_t uIndex, Bu::Nids::Block *pBlock )
 
 void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize )
 {
+//      if( !sStore.canWrite() )
+//              return;
         sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 );
         sStore.write( &uSize, 4 );
 }
@@ -206,6 +240,8 @@ uint32_t Bu::Nids::getNextBlock( uint32_t uIndex,
                         sStore.setPos( iBlockStart + (iBlockSize*uIndex)+1*4 );
                         sStore.write( &uNew, 4 );
                         getBlock( uNew, pBlock );
+                        printf("Allocated new block (%u) for stream %u.\n",
+                                uNew, pBlock->uFirstBlock );
                 }
                 else
                 {
diff --git a/src/nids.h b/src/nids.h
index d297fa1..63d7061 100644
--- a/src/nids.h
+++ b/src/nids.h
@@ -65,6 +65,10 @@ namespace Bu
                 NidsStream openStream( int iID );
 
                 int getBlockSize();
+                int getNumBlocks();
+                int getNumUsedBlocks();
+                int getBlockStart();
+                int getBlockOverhead();
 
         private:
                 typedef struct Block
@@ -92,6 +96,8 @@ namespace Bu
                 uint32_t getNextBlock( uint32_t uIndex, struct Nids::Block *pBlock,
                         bool bCreate=true);
 
+                void updateHeader();
+
                 // Block allocation routines
                 Block *newBlock();
                 void deleteBlock( Block *pBlock );
@@ -101,6 +107,7 @@ namespace Bu
                 int iBlockSize;
                 int iBlocks;
                 int iBlockStart;
+                int iUsed;
                 Bu::BitString bsBlockUsed;
         };
 };
diff --git a/src/nidsstream.cpp b/src/nidsstream.cpp
index 4f692c6..fff73f6 100644
--- a/src/nidsstream.cpp
+++ b/src/nidsstream.cpp
@@ -45,15 +45,27 @@ void Bu::NidsStream::close()
 
 size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
 {
-        if( uPos%uBlockSize+nBytes < uBlockSize )
+        if( nBytes == 0 )
+                return 0;
+        if( nBytes + uPos > uSize )
+                nBytes = uSize - uPos;
+        if( (uPos%uBlockSize)+nBytes < uBlockSize )
         {
                 size_t iRead = nBytes;
                 if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) )
                         iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
                 memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead );
+                //printf("buffill:  %ub, %u-%u/%u -> %d-%d/%d (a:%u)",
+                //      iRead, uPos, uPos+iRead-1, uSize, 0, iRead-1, nBytes, uCurBlock );
                 uPos += iRead;
-                //printf("a: block %u = %ub (%ub total)\n",
-                //      uCurBlock, pCurBlock->uBytesUsed, uSize );
+                //printf(" -- %u\n", uPos%uBlockSize );
+                //printf("ra: block %u = %ub:%u (%ub total)\n",
+                //      uCurBlock, uPos, nBytes, uSize );
+
+                // This can't happen, if we're right on a boundery, it goes to the
+                // other case
+                //if( uPos%uBlockSize == 0 )
+                //      uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false );
                 return iRead;
         }
         else
@@ -68,15 +80,18 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
                                 iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
                         memcpy( ((char *)pBuf)+nTotal,
                                 pCurBlock->pData+(uPos%uBlockSize), iRead );
+                        //printf("buffill:  %ub, %u-%u/%u -> %d-%d/%d (b:%u)\n",
+                        //      iRead, uPos, uPos+iRead-1, uSize,
+                        //      nTotal, nTotal+nBytes-1, nBytes, uCurBlock );
                         uPos += iRead;
                         nBytes -= iRead;
                         nTotal += iRead;
-                        //printf("r: block %u = %ub/%ub (%ub total)\n",
-                        //      uCurBlock, iRead, pCurBlock->uBytesUsed, uSize );
-                        if( nBytes == 0 || uPos == uSize )
-                                return nTotal;
-                        if( nTotal%uBlockSize == 0 )
+                        //printf("rb: block %u = %ub:%u (%ub total)\n",
+                        //      uCurBlock, uPos, iRead, uSize );
+                        if( uPos%uBlockSize == 0 )
                                 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false );
+                        if( nBytes == 0 || uPos >= uSize )
+                                return nTotal;
                 }
         }
         return 0;
@@ -84,15 +99,26 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
 
 size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes )
 {
-        if( uPos%uBlockSize+nBytes < uBlockSize )
+        if( nBytes == 0 )
+                return 0;
+/*      if( pCurBlock->uBytesUsed >= uBlockSize )
         {
+                // We're at the end of our current block, allocate another before we do
+                // anything.
+                uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock );
+        } */
+        if( (uPos%uBlockSize)+nBytes < uBlockSize )
+        {
+                //printf("wa: %u:%u:%u:%u -> ", uPos, uPos%uBlockSize, uSize, pCurBlock->uBytesUsed );
                 memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes );
-                pCurBlock->uBytesUsed += nBytes;
+                if( (uPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed )
+                        pCurBlock->uBytesUsed = (uPos%uBlockSize)+nBytes;
                 rNids.setBlock( uCurBlock, pCurBlock );
                 uPos += nBytes;
-                uSize += nBytes;
-                //printf("a: block %u = %ub (%ub total)\n",
-                //      uCurBlock, pCurBlock->uBytesUsed, uSize );
+                if( uPos > uSize )
+                        uSize = uPos;
+                //printf("block %u = %ub (%ub total) %d:%u\n",
+                //      uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, uPos );
                 return nBytes;
         }
         else
@@ -101,22 +127,25 @@ size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes )
                 for(;;)
                 {
                         uint32_t uNow = uBlockSize-(uPos%uBlockSize);
+                        //printf("uNow:  %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, uPos, uBlockSize, nBytes );
                         if( nBytes < uNow )
                                 uNow = nBytes;
                         memcpy( pCurBlock->pData+(uPos%uBlockSize),
                                 &((char *)pBuf)[nTotal], uNow );
-                        pCurBlock->uBytesUsed += uNow;
+                        if( (uPos%uBlockSize)+uNow > pCurBlock->uBytesUsed )
+                                pCurBlock->uBytesUsed = (uPos%uBlockSize)+uNow;
                         rNids.setBlock( uCurBlock, pCurBlock );
-                        uSize += uNow;
                         uPos += uNow;
+                        if( uPos > uSize )
+                                uSize = uPos;
                         nTotal += uNow;
                         nBytes -= uNow;
-                        //printf("b: block %u = %ub (%ub total)\n",
+                        //printf("wb: block %u = %ub (%ub total)\n",
                         //      uCurBlock, pCurBlock->uBytesUsed, uSize );
-                        if( nBytes == 0 )
-                                return nTotal;
                         if( pCurBlock->uBytesUsed == uBlockSize )
                                 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock );
+                        if( nBytes == 0 )
+                                return nTotal;
                 }
         }
 }
diff --git a/src/tests/nidstool.cpp b/src/tests/nidstool.cpp
new file mode 100644
index 0000000..546534e
--- /dev/null
+++ b/src/tests/nidstool.cpp
@@ -0,0 +1,124 @@
+#include "bu/file.h"
+#include "bu/nids.h"
+#include "bu/nidsstream.h"
+#include "bu/paramproc.h"
+
+#include <stdlib.h>
+
+class Param : public Bu::ParamProc
+{
+public:
+        Param( int argc, char *argv[] )
+        {
+                addHelpBanner("nidstool - Do stuff with nids files.\n\n");
+                addParam("info", 'i', mkproc(Param::procInfo),
+                        "Print some info about the file.");
+                addParam("dump", 'd', mkproc(Param::procDump),
+                        "Dump a stream to a file");
+                addParam("help", 'h', mkproc(Bu::ParamProc::help), "This help.");
+                process( argc, argv );
+        }
+
+        virtual ~Param()
+        {
+        }
+
+        int procInfo( int argc, char *argv[] )
+        {
+                if( argc < 1 )
+                {
+                        printf("You must provide a file name.\n");
+                        exit( 1 );
+                }
+
+                Bu::File fIn( argv[0], Bu::File::Read );
+                Bu::Nids n( fIn );
+                n.initialize();
+
+                printf("Block size:     %db\n", n.getBlockSize() );
+                printf("Block count:    %d\n", n.getNumBlocks() );
+                printf("Blocks used:    %d (%d%%)\n", n.getNumUsedBlocks(),
+                        n.getNumUsedBlocks()*100/n.getNumBlocks() );
+                printf("Block start:    %db\n", n.getBlockStart() );
+                printf("Block overhead: %db\n", n.getBlockOverhead() );
+                printf("Block storage:  %db (%d%%)\n",
+                        n.getBlockSize()-n.getBlockOverhead(),
+                        (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() );
+
+                if( argc >= 2 )
+                {
+                        typedef struct Block
+                        {
+                                uint32_t uFirstBlock;
+                                uint32_t uNextBlock;
+                                uint32_t uPrevBlock;
+                                uint32_t uBytesUsed;
+                                uint32_t uReserved;
+                        } Block;
+
+                        uint32_t uStream = strtoul( argv[1], NULL, 0 );
+                        uint32_t uBlock = uStream;
+
+                        Block b;
+
+                        for(;;)
+                        {
+                                fIn.setPos( n.getBlockStart()+n.getBlockSize()*uBlock );
+                                fIn.read( &b, sizeof(Block) );
+                                printf("Stream %u:  block %u, next %u, prev %u, %ub used.\n",
+                                        uStream, uBlock, b.uNextBlock, b.uPrevBlock, b.uBytesUsed
+                                        );
+                                if( b.uNextBlock == 0xFFFFFFFFUL )
+                                        break;
+                                uBlock = b.uNextBlock;
+                        }
+                        printf("Stream End.\n");
+
+                        return 2;
+                }
+
+                return 1;
+        }
+
+        int procDump( int argc, char *argv[] )
+        {
+                if( argc < 3 )
+                {
+                        printf("You must provide a nids file, a stream id, and an output "
+                                "file.\n");
+                        exit( 1 );
+                }
+
+                Bu::File fIn( argv[0], Bu::File::Read );
+                Bu::Nids n( fIn );
+                n.initialize();
+
+                int iStream = strtol( argv[1], NULL, 0 );
+                Bu::NidsStream sIn = n.openStream( iStream );
+
+                Bu::File fOut( argv[2], Bu::File::Write|Bu::File::Create );
+                int iTotal = 0;
+                char buf[100];
+                for(;;)
+                {
+                        int iRead = sIn.read( buf, 100 );
+                        iTotal += fOut.write( buf, iRead );
+                        if( iRead < 100 )
+                                break;
+                }
+
+                printf("Wrote %db from stream %d in %s to %s.\n",
+                        iTotal, iStream, argv[0], argv[2] );
+                return 3;
+        }
+
+};
+
+
+int main( int argc, char *argv[] )
+{
+        Param p( argc, argv );
+
+        return 0;
+}
+
diff --git a/src/tests/rh.cpp b/src/tests/rh.cpp
new file mode 100644
index 0000000..70abcb7
--- /dev/null
+++ b/src/tests/rh.cpp
@@ -0,0 +1,52 @@
+#include "bu/file.h"
+#include "bu/hash.h"
+#include "bu/archive.h"
+#include "bu/fstring.h"
+#include "bu/nids.h"
+#include "bu/nidsstream.h"
+
+int main( int argc, char *argv[] )
+{
+        if( argv[1][0] == 'r' )
+        {
+                typedef Bu::Hash<Bu::FString, int> Hsh;
+
+                Bu::File fIn( argv[2], Bu::File::Read );
+                Bu::Archive ar( fIn, Bu::Archive::load );
+
+                Hsh h;
+                ar >> h;
+
+                printf("Read %d.\n", h.getSize() );
+                for( Hsh::iterator i = h.begin(); i != h.end(); i++ )
+                {
+                        printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() );
+                }
+
+                printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() );
+        }
+        else if( argv[1][0] == 'n' )
+        {
+                typedef Bu::Hash<Bu::FString, int> Hsh;
+
+                Bu::File fIn( argv[2], Bu::File::Read );
+                Bu::Nids n( fIn );
+                n.initialize();
+                Bu::NidsStream sIn = n.openStream( 0 );
+                Bu::Archive ar( sIn, Bu::Archive::load );
+
+                Hsh h;
+                ar >> h;
+
+                printf("Read %d.\n", h.getSize() );
+                for( Hsh::iterator i = h.begin(); i != h.end(); i++ )
+                {
+                        printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() );
+                }
+
+                printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() );
+        }
+
+        return 0;
+}
+