Changed the name of nids to Myriad, I like it, but I'm not getting rid of nids

until I can safely migrate to Myriad.

10 files changed, 1010 insertions, 83 deletions

diff --git a/src/array.h b/src/array.h
index 6279382..af41c33 100644
--- a/src/array.h
+++ b/src/array.h
@@ -11,10 +11,95 @@
 #include <memory>
 #include "bu/exceptionbase.h"
 #include "bu/archivebase.h"
+#include "bu/sharedcore.h"
 
 namespace Bu
 {
         subExceptionDecl( ArrayException )
+
+        template<typename value, int inc, typename valuealloc>
+        class ArrayCore
+        {
+        public:
+                ArrayCore() :
+                        pData( NULL ),
+                        iSize( 0 ),
+                        iCapacity( 0 )
+                { }
+
+                void setCapacity( int iNewLen )
+                {
+                        //clear();
+                        //iCapacity = iCapacity;
+                        //pData = va.allocate( iCapacity );
+                        if( iNewLen <= iCapacity ) return;
+                        value *pNewData = va.allocate( iNewLen );
+                        if( pData )
+                        {
+                                for( int j = 0; j < iSize; j++ )
+                                {
+                                        va.construct( &pNewData[j], pData[j] );
+                                        va.destroy( &pData[j] );
+                                }
+                                va.deallocate( pData, iCapacity );
+                        }
+                        pData = pNewData;
+                        iCapacity = iNewLen;
+                }
+
+                virtual ~ArrayCore()
+                {
+                        clear();
+                }
+
+                void clear()
+                {
+                        if( pData )
+                        {
+                                for( int j = 0; j < iSize; j++ )
+                                {
+                                        va.destroy( &pData[j] );
+                                }
+                                va.deallocate( pData, iCapacity );
+                                pData = NULL;
+                        }
+                        iSize = 0;
+                        iCapacity = 0;
+                }
+
+                void erase( int iPos )
+                {
+                        for( int j = iPos; j < iSize; j++ )
+                        {
+                                va.destroy( &pData[j] );
+                                if( j == iSize-1 )
+                                {
+                                        iSize--;
+                                        return;
+                                }
+                                va.construct( &pData[j], pData[j+1] );
+                        }
+                }
+
+                void swapErase( int iPos )
+                {
+                        if( iPos == iSize-1 )
+                        {
+                                erase( iPos );
+                                return;
+                        }
+                        va.destroy( &pData[iPos] );
+                        va.construct( &pData[iPos], pData[iSize-1] );
+                        va.destroy( &pData[iSize-1] );
+                        iSize--;
+                }
+
+                valuealloc va;
+                value *pData;
+                long iSize;
+                long iCapacity;
+        };
+
         /**
          * Array type container, just like a normal array only flexible and keeps
          * track of your memory for you.
@@ -25,51 +110,57 @@ namespace Bu
          *@ingroup Containers
          */
         template<typename value, int inc=10, typename valuealloc=std::allocator<value> >
-        class Array
+        class Array : public SharedCore<ArrayCore<value, inc, valuealloc> >
         {
         private:
                 typedef class Array<value, inc, valuealloc> MyType;
+                typedef class ArrayCore<value, inc, valuealloc> Core;
+
+        protected:
+                using SharedCore< Core >::core;
+                using SharedCore< Core >::_hardCopy;
+                using SharedCore< Core >::_allocateCore;
 
         public:
-                Array() :
-                        pData( NULL ),
-                        iSize( 0 ),
-                        iCapacity( 0 )
+                struct const_iterator;
+                struct iterator;
+
+                Array()
                 {
                 }
 
                 Array( const MyType &src ) :
-                        pData( NULL ),
-                        iSize( 0 ),
-                        iCapacity( 0 )
+                        SharedCore< Core >( src )
                 {
-                        *this = src;
                 }
                 
-                Array( long iSetCap ) :
-                        pData( NULL ),
-                        iSize( 0 ),
-                        iCapacity( 0 )
+                Array( long iSetCap )
                 {
                         setCapacity( iSetCap );
                 }
 
                 ~Array()
                 {
-                        clear();
                 }
 
-                MyType &operator=( const MyType &src )
+                bool operator==( const MyType &src ) const
                 {
-                        clear();
-                        setCapacity( src.iCapacity );
+                        if( core == src.core )
+                                return true;
+                        if( core->iSize != src.core->iSize )
+                                return false;
 
-                        long iTop = src.getSize();
-                        for( long i = 0; i < iTop; ++i )
+                        for( int j = 0; j < core->iSize; j++ )
                         {
-                                append( src[i] );
+                                if( core->pData[j] != src.core->pData[j] )
+                                        return false;
                         }
-                        return *this;
+                        return true;
+                }
+
+                bool operator!=( const MyType &src ) const
+                {
+                        return !(*this == src);
                 }
 
                 /**
@@ -77,46 +168,61 @@ namespace Bu
                  */
                 void clear()
                 {
-                        if( pData )
-                        {
-                                for( int j = 0; j < iSize; j++ )
-                                {
-                                        va.destroy( &pData[j] );
-                                }
-                                va.deallocate( pData, iCapacity );
-                                pData = NULL;
-                        }
-                        iSize = 0;
-                        iCapacity = 0;
+                        _hardCopy();
+                        core->clear();
                 }
 
                 void append( const value &rVal )
                 {
-                        if( iSize == iCapacity )
+                        _hardCopy();
+                        if( core->iSize == core->iCapacity )
                         {
-                                setCapacity( iCapacity + inc );
+                                core->setCapacity( core->iCapacity + inc );
                         }
 
-                        va.construct( &pData[iSize++], rVal );
+                        core->va.construct( &core->pData[core->iSize++], rVal );
                 }
 
                 //operator
                 value &operator[]( long iIndex )
                 {
-                        if( iIndex < 0 || iIndex >= iSize )
+                        _hardCopy();
+                        if( iIndex < 0 || iIndex >= core->iSize )
                                 throw ArrayException(
-                                        "Index %d out of range 0:%d", iIndex, iSize );
+                                        "Index %d out of range 0:%d", iIndex, core->iSize );
 
-                        return pData[iIndex];
+                        return core->pData[iIndex];
                 }
                 
                 const value &operator[]( long iIndex ) const
                 {
-                        if( iIndex < 0 || iIndex >= iSize )
+                        if( iIndex < 0 || iIndex >= core->iSize )
                                 throw ArrayException(
-                                        "Index %d out of range 0:%d", iIndex, iSize );
+                                        "Index %d out of range 0:%d", iIndex, core->iSize );
+
+                        return core->pData[iIndex];
+                }
+
+                value &first()
+                {
+                        _hardCopy();
+                        return core->pData[0];
+                }
+
+                const value &first() const
+                {
+                        return core->pData[0];
+                }
+
+                value &last()
+                {
+                        _hardCopy();
+                        return core->pData[core->iSize-1];
+                }
 
-                        return pData[iIndex];
+                const value &last() const
+                {
+                        return core->pData[core->iSize-1];
                 }
 
                 /**
@@ -125,7 +231,7 @@ namespace Bu
                  */
                 long getSize() const
                 {
-                        return iSize;
+                        return core->iSize;
                 }
 
                 /**
@@ -136,7 +242,7 @@ namespace Bu
                  */
                 long getCapacity() const
                 {
-                        return iCapacity;
+                        return core->iCapacity;
                 }
 
                 /**
@@ -149,19 +255,8 @@ namespace Bu
                  */
                 void setCapacity( long iNewLen )
                 {
-                        if( iNewLen <= iCapacity ) return;
-                        value *pNewData = va.allocate( iNewLen );
-                        if( pData )
-                        {
-                                for( int j = 0; j < iSize; j++ )
-                                {
-                                        va.construct( &pNewData[j], pData[j] );
-                                        va.destroy( &pData[j] );
-                                }
-                                va.deallocate( pData, iCapacity );
-                        }
-                        pData = pNewData;
-                        iCapacity = iNewLen;
+                        _hardCopy();
+                        core->setCapacity( iNewLen );
                 }
 
                 typedef struct iterator
@@ -387,16 +482,8 @@ namespace Bu
                  */
                 void erase( iterator i )
                 {
-                        for( int j = i.iPos; j < iSize; j++ )
-                        {
-                                va.destroy( &pData[j] );
-                                if( j == iSize-1 )
-                                {
-                                        iSize--;
-                                        return;
-                                }
-                                va.construct( &pData[j], pData[j+1] );
-                        }
+                        _hardCopy();
+                        core->erase( i.iPos );
                 }
 
                 /**
@@ -408,22 +495,24 @@ namespace Bu
                  */
                 void swapErase( iterator i )
                 {
-                        if( i.iPos == iSize-1 )
-                        {
-                                erase( i );
-                                return;
+                        _hardCopy();
+                        swapErase( i.iPos );
+                }
+
+        protected:
+                virtual Core *_copyCore( Core *src )
+                {
+                        Core *pRet = _allocateCore();
+                        pRet->setCapacity( src->iCapacity );
+                        pRet->iSize = src->iSize;
+                        for( int j = 0; j < src->iSize; j++ )
+                        {       
+                                pRet->va.construct( &pRet->pData[j], src->pData[j] );
                         }
-                        va.destroy( &pData[i.iPos] );
-                        va.construct( &pData[i.iPos], pData[iSize-1] );
-                        va.destroy( &pData[iSize-1] );
-                        iSize--;
+                        return pRet;
                 }
 
         private:
-                valuealloc va;
-                value *pData;
-                long iSize;
-                long iCapacity;
         };
 
         class Formatter;
diff --git a/src/bitstring.cpp b/src/bitstring.cpp
index f05818f..a7881cc 100644
--- a/src/bitstring.cpp
+++ b/src/bitstring.cpp
@@ -443,6 +443,15 @@ long Bu::BitString::getHighestOrderBitPos()
 
         return -1;
 }
+
+Bu::FString Bu::BitString::toString()
+{
+        Bu::FString sRet;
+        for( int j = iBits-1; j >= 0; j-- )
+                sRet.append( getBit( j )?'1':'0' );
+        return sRet;
+}
+
 /*
 bool Bu::BitString::writeToFile( FILE *fh )
 {
diff --git a/src/bitstring.h b/src/bitstring.h
index b5f1ada..7a83f50 100644
--- a/src/bitstring.h
+++ b/src/bitstring.h
@@ -9,6 +9,7 @@
 #define BU_BITSTRING_H
 
 #include "bu/util.h"
+#include "bu/fstring.h"
 
 namespace Bu
 {
@@ -202,6 +203,8 @@ namespace Bu
                  */
                 long toLong( long iStart = 0, long iSize = 32 );
 
+                Bu::FString toString();
+
                 //operators
                 BitString &operator=( const BitString &xSrc );
                 BitString operator~();
diff --git a/src/myriad.cpp b/src/myriad.cpp
new file mode 100644
index 0000000..64c8d20
--- /dev/null
+++ b/src/myriad.cpp
@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2007-2008 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/myriad.h"
+#include "bu/stream.h"
+#include "bu/myriadstream.h"
+#include <stdio.h>
+
+#include "bu/sio.h"
+using Bu::sio;
+using Bu::Fmt;
+
+#define Myriad_MAGIC_CODE               ((unsigned char *)"\xFF\xC3\x99\xBD")
+
+namespace Bu
+{
+        subExceptionDef( MyriadException )
+        template<typename t> t blkDiv( t total, t block ) {
+                return (total/block)+((total%block==0)?(0):(1));
+        }
+}
+
+Bu::Myriad::Myriad( Bu::Stream &sStore ) :
+        sStore( sStore ),
+        iBlockSize( 0 ),
+        iBlocks( 0 ),
+        iUsed( 0 )
+{
+}
+
+Bu::Myriad::~Myriad()
+{
+        updateHeader();
+
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                delete *i;
+        }
+}
+
+void Bu::Myriad::sync()
+{
+        updateHeader();
+
+        // Later, also flush all caches.
+}
+
+void Bu::Myriad::initialize()
+{
+        sStore.setPosEnd( 0 );
+        int iSize = sStore.tell();
+        sStore.setPos( 0 );
+
+        unsigned char buf[4];
+        if( sStore.read( buf, 4 ) < 4 )
+                throw MyriadException("Input stream appears to be empty.");
+        if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
+        {
+                throw MyriadException(
+                        "Stream does not appear to be a valid Myriad format.");
+        }
+        sStore.read( buf, 2 );
+        if( buf[0] != 1 )
+                throw MyriadException(
+                        "We can only handle version 1 for now.");
+        if( buf[1] != 32 )
+                throw MyriadException(
+                        "We can only handle 32-bit words at the moment.");
+        sStore.read( &iBlockSize, 4 );
+        int iStreams;
+        sStore.read( &iStreams, 4 );
+
+        iBlocks = iSize/iBlockSize;
+        sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize
+                << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl;
+        
+        int iHeaderSize = 14 + 8 + 4;
+        int iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+
+        while( iHeaderSize > iHeaderBlocks*iBlockSize )
+        {
+                iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
+                iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+        }
+
+        sio << "Myriad: iHeaderSize=" << iHeaderSize
+                << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
+        
+        bsBlockUsed.setSize( iBlocks, true );
+
+        bool bCanSkip = false; // Can skip around, post initial header stream i/o
+        for( int j = 0; j < iStreams; j++ )
+        {
+                int iHdrBlock = 0;
+                int iCurBlock = 0;
+                aStreams.append( new Stream() );
+                Stream &s = *aStreams[j];
+                sStore.read( &s.iId, 4 );
+                sStore.read( &s.iSize, 4 );
+                int iSBlocks = blkDiv(s.iSize, iBlockSize);
+                sio << "Myriad: - Stream::iId=" << s.iId << ", Stream::iSize=" << s.iSize
+                        << ", Stream::aBlocks=" << iSBlocks
+                        << ", sStore.tell()=" << sStore.tell() << sio.nl;
+                for( int k = 0; k < iSBlocks; k++ )
+                {
+                        int iBId;
+                        sStore.read( &iBId, 4 );
+                        sio << "Myriad:   - iBId=" << iBId
+                                << ", iStartPos=" << iBId*iBlockSize
+                                << ", sStore.tell()=" << sStore.tell() << sio.nl;
+                        s.aBlocks.append( iBId );
+                        bsBlockUsed.setBit( iBId );
+                        if( (j == 0 && k == iHeaderBlocks-1) )
+                        {
+                                sio << "Myriad:   - End of prepartition, unlocking skipping."
+                                        << sio.nl;
+                                bCanSkip = true;
+                                iCurBlock = blkDiv( (int)sStore.tell(), iBlockSize );
+                        }
+                        if( bCanSkip && sStore.tell() >= iCurBlock*iBlockSize+iBlockSize )
+                        {
+                                iHdrBlock++;
+                                iCurBlock = aStreams[0]->aBlocks[iHdrBlock];
+                                sio << "Myriad: Ran out of data in block, finding next header "
+                                        "block:  " << iHdrBlock << " = " << iCurBlock << " ("
+                                        << iCurBlock*iBlockSize << "b)" << sio.nl;
+                                sStore.setPos( iCurBlock*iBlockSize );
+                        }
+                }
+        }
+
+        sio << bsBlockUsed.toString() << sio.nl;
+
+        //printf("%d blocks, %db each, %db block offset\n",
+        //      iBlocks, iBlockSize, iBlockStart );
+
+}
+
+void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
+{
+        int iHeaderSize = 14 + 8 + 4;
+        int iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+        char cBuf = 1;
+        int iBuf = 0;
+        
+        Stream *pStr = new Stream;
+        pStr->iId = 0;
+
+        while( iHeaderSize > iHeaderBlocks*iBlockSize )
+        {
+                iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
+                iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+        }
+
+        iPreAllocate += iHeaderBlocks;
+
+        sio << "Myriad:  iHeaderSize=" << iHeaderSize << ", iBlockSize="
+                << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
+        
+        bsBlockUsed.setSize( iPreAllocate, true );
+
+        char *pBlock = new char[iBlockSize];
+        memset( pBlock, 0, iBlockSize );
+        for( int j = 0; j < iPreAllocate; j++ )
+        {
+                sStore.write( pBlock, iBlockSize );
+                pStr->aBlocks.append( j );
+        }
+        delete[] (char *)pBlock;
+
+        sStore.setPos( 0 );
+
+        // Magic number
+        sStore.write( Myriad_MAGIC_CODE, 4 );
+
+        // Version (0)
+        sStore.write( &cBuf, 1 );
+
+        // Bits per int
+        cBuf = 32;
+        sStore.write( &cBuf, 1 );
+
+        // The size of each block
+        sStore.write( &iBlockSize, 4 );
+
+        iBuf = 1;
+        // The number of streams
+        sStore.write( &iBuf, 4 );
+        
+        // Stream header
+        iBuf = 0;
+        sStore.write( &iBuf, 4 );
+        sStore.write( &iHeaderSize, 4 );
+        for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ )
+        {
+                sStore.write( &iBuf, 4 );
+        }
+
+        this->iBlockSize = iBlockSize;
+        this->iBlocks = iPreAllocate;
+        
+        pStr->iSize = sStore.tell();
+
+        //hStreams.insert( 0, BlockArray( 0 ) );
+}
+
+void Bu::Myriad::updateHeader()
+{
+        if( !sStore.canWrite() )
+                return;
+
+        
+
+        // TODO: Use the stream class to access this really smoothly, I hope :)
+}
+
+int Bu::Myriad::createStream( int iPreAllocate )
+{
+        Stream *pStr = new Stream();
+        pStr->iId = aStreams.last()->iId+1;
+        sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
+                << iPreAllocate << sio.nl;
+        pStr->iSize = 0;
+
+        for( int j = 0; j < iPreAllocate; j++ )
+        {
+                int iFreeBlock = findEmptyBlock();
+                sio << "Myriad: Adding block " << j << sio.nl;
+                pStr->aBlocks.append( j );
+                bsBlockUsed.setBit( j );
+        }
+
+        return 0;
+}
+
+int Bu::Myriad::findEmptyBlock()
+{
+        for( int j = 0; j < bsBlockUsed.getSize(); j++ )
+        {
+                if( bsBlockUsed.getBit( j ) == false )
+                        return j;
+        }
+        sio << "Myriad: findEmptyBlock(): No empty blocks, adding new one." << sio.nl;
+
+        bsBlockUsed.setSize( bsBlockUsed.getSize()+1, false );
+        sStore.setPos( iBlockSize*iBlocks );
+
+        char *pBlock = new char[iBlockSize];
+        memset( pBlock, 0, iBlockSize );
+        sStore.write( pBlock, iBlockSize );
+        delete pBlock;
+
+        return iBlockSize++;
+}
+
+void Bu::Myriad::deleteStream( int /*iID*/ )
+{
+}
+
+Bu::MyriadStream Bu::Myriad::openStream( int iID )
+{
+        return MyriadStream( *this, iID );
+}
+
+int Bu::Myriad::getBlockSize()
+{
+        return iBlockSize;
+}
+
+int Bu::Myriad::getNumBlocks()
+{
+        return iBlocks;
+}
+
+int Bu::Myriad::getNumUsedBlocks()
+{
+        return iUsed;
+}
+
+
diff --git a/src/myriad.h b/src/myriad.h
new file mode 100644
index 0000000..772729f
--- /dev/null
+++ b/src/myriad.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2007-2008 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#ifndef BU_MYRIAD_H
+#define BU_MYRIAD_H
+
+#include <stdint.h>
+#include "bu/bitstring.h"
+#include "bu/exceptionbase.h"
+#include "bu/array.h"
+
+namespace Bu
+{
+        class Stream;
+        class MyriadStream;
+
+        subExceptionDecl( MyriadException )
+
+        /**
+         * Numerically Indexed Data Streams.  This is a working name so I can
+         * actually get some code written instead of agonizing over the name.
+         *
+         * This is a system for creating streams that contain other streams in
+         * a flexible block-allocated system.
+         *
+         * Header format is as follows:
+         *
+         * MMMMvBssssSSSS*
+         * M = Magic number
+         * v = version number
+         * B = Bits per int
+         * s = Blocksize in bytes
+         * S = Number of Streams
+         * 
+         * The * represents the Stream headers, one per stream, as follows:
+         * IIIIssss$
+         * I = Id number of the stream
+         * s = size of stream in bytes
+         *
+         * The $ represents the Block headers, one per used block, as  follows:
+         * IIII
+         * I = Index of the block
+         *
+         * The stream/block data is interleaved in the header, so all blocks stored
+         * with one stream are together.  The block headers are in order, and the
+         * data in them is required to be "solid" you cannot fill partial blocks
+         * mid-way through a stream.
+         *
+         * The initial block starts with the nids header, and is both the zero block
+         * and the zero stream.  For now, the minimum block size is the size needed
+         * to store the base header, the zero stream header, and the first two
+         * blocks of the zero stream, so 30 bytes.  Since it's reccomended to use
+         * a size that will fit evenly into filesystem blocks, then a size of 32 is
+         * probably the smallest reccomended size because all powers of two equal
+         * to or greater than 32 are evenly divisible by 32.
+         *
+         * I have had a thought that if the block size were smaller than 42 bytes
+         * the header would consume the first N blocks where N * block size is
+         * enough space to house the initial header, the first stream header, and
+         * the first N block headers.  This, of course, causes you to hit an
+         * infinite header if the block size is small enough.
+         */
+        class Myriad
+        {
+                friend class MyriadStream;
+        public:
+                Myriad( Bu::Stream &sStore );
+                virtual ~Myriad();
+
+                /**
+                 * Initialize this object based on the data already in the assosiated
+                 * stream.  This will be called automatically for you if you forget,
+                 * but if you want to pre-initialize for some reason, just call this
+                 * once before you actually start doing anything with your Myriad.
+                 */
+                void initialize();
+
+                /**
+                 * Create a new Myriad system in the assosiated stream.  This should be
+                 * used carefully, it will destroy all data already within the stream.
+                 * More options will probably be added soon.
+                 */
+                void initialize( int iBlockSize, int iPreAllocate=1 );
+
+                /**
+                 * Create a new stream within the Myriad system.  The ID of the new stream
+                 * is returned.
+                 */
+                int createStream( int iPreAllocate=1 );
+
+                /**
+                 * Delete a stream that's already within the Myriad.
+                 */
+                void deleteStream( int iID );
+
+                /**
+                 * Return a new Stream object assosiated with the given stream ID.
+                 */
+                MyriadStream openStream( int iID );
+
+                int getBlockSize();
+                int getNumBlocks();
+                int getNumUsedBlocks();
+                int getBlockOverhead();
+
+                /**
+                 * Syncronize the header data, etc. with the storage stream.  It's not
+                 * a bad idea to call this periodically.
+                 */
+                void sync();
+
+        private:
+                enum
+                {
+                        blockUnused     =       0xFFFFFFFFUL
+                };
+
+                void updateHeader();
+                int findEmptyBlock();
+
+        private:
+                Bu::Stream &sStore;
+                int iBlockSize;
+                int iBlocks;
+                int iUsed;
+                Bu::BitString bsBlockUsed;
+                typedef Bu::Array<int> BlockArray;
+                class Stream
+                {
+                public:
+                        int iId;
+                        int iSize;
+                        BlockArray aBlocks;
+                };
+                typedef Bu::Array<Stream *> StreamArray;
+                StreamArray aStreams;
+        };
+};
+
+#endif
diff --git a/src/myriadstream.cpp b/src/myriadstream.cpp
new file mode 100644
index 0000000..c9b3326
--- /dev/null
+++ b/src/myriadstream.cpp
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2007-2008 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/myriadstream.h"
+
+#include <string.h>
+
+Bu::MyriadStream::MyriadStream( Myriad &rMyriad, uint32_t uStream ) :
+        rMyriad( rMyriad ),
+        uStream( uStream ),
+        pCurBlock( NULL ),
+        uCurBlock( uStream ),
+        uSize( 0 ),
+        uBlockSize( rMyriad.iBlockSize ),
+        uPos( 0 )
+{
+        //printf("MyriadStream::allocated\n");
+        //pCurBlock = rMyriad.newBlock();
+        //rMyriad.getBlock( uStream, pCurBlock );
+        //uSize = pCurBlock->uBytesUsed;
+}
+
+Bu::MyriadStream::~MyriadStream()
+{
+        //printf("Destroying stream?\n");
+        //rMyriad.updateStreamSize( uStream, uSize );
+        //rMyriad.deleteBlock( pCurBlock );
+}
+
+void Bu::MyriadStream::close()
+{
+}
+
+size_t Bu::MyriadStream::read( void *pBuf, size_t nBytes )
+{
+        /*
+        if( nBytes == 0 )
+                return 0;
+        if( nBytes + uPos > uSize )
+                nBytes = uSize - uPos;
+        if( (uPos%uBlockSize)+nBytes < uBlockSize )
+        {
+                size_t iRead = nBytes;
+                if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) )
+                        iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
+                memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead );
+                //printf("read buffill:  %ub, %u-%u/%u -> %d-%d/%d (a:%u)",
+                //      iRead, uPos, uPos+iRead-1, uSize, 0, iRead-1, nBytes, uCurBlock );
+                uPos += iRead;
+                //printf(" -- %u\n", uPos%uBlockSize );
+                //printf("ra: block %u = %ub:%u (%ub total)\n",
+                //      uCurBlock, uPos, nBytes, uSize );
+
+                // This can't happen, if we're right on a boundery, it goes to the
+                // other case
+                //if( uPos%uBlockSize == 0 )
+                //      uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock, false );
+                return iRead;
+        }
+        else
+        {
+                size_t nTotal = 0;
+                for(;;)
+                {
+                        uint32_t iRead = nBytes;
+                        if( iRead > uBlockSize-(uPos%uBlockSize) )
+                                iRead = uBlockSize-(uPos%uBlockSize);
+                        if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) )
+                                iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
+                        memcpy( ((char *)pBuf)+nTotal,
+                                pCurBlock->pData+(uPos%uBlockSize), iRead );
+                        //printf(" read buffill:  %ub, %u-%u/%u -> %d-%d/%d (b:%u)\n",
+                        //      iRead, uPos, uPos+iRead-1, uSize,
+                        //      nTotal, nTotal+nBytes-1, nBytes, uCurBlock );
+                        uPos += iRead;
+                        nBytes -= iRead;
+                        nTotal += iRead;
+                        //printf("rb: block %u = %ub:%u (%ub total)\n",
+                        //      uCurBlock, uPos, iRead, uSize );
+                        if( uPos%uBlockSize == 0 )
+                                uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock, false );
+                        if( nBytes == 0 || uPos >= uSize )
+                                return nTotal;
+                }
+        }*/
+        return 0;
+}
+
+size_t Bu::MyriadStream::write( const void *pBuf, size_t nBytes )
+{
+        if( nBytes == 0 )
+                return 0;
+/*      if( pCurBlock->uBytesUsed >= uBlockSize )
+        {
+                // We're at the end of our current block, allocate another before we do
+                // anything.
+                uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock );
+        } */
+        /*
+        if( (uPos%uBlockSize)+nBytes < uBlockSize )
+        {
+                //printf("wa: %u:%u:%u:%u -> ", uPos, uPos%uBlockSize, uSize, pCurBlock->uBytesUsed );
+                memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes );
+                //printf("write buffill:  %ub, %u-%u/%u -> %d-%d/%d (a:%u:%u)\n",
+                //      nBytes, 0, nBytes-1, nBytes,
+                //      uPos, uPos+nBytes-1, uSize, uCurBlock,
+                //      pCurBlock->uBytesUsed );
+                if( (uPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed )
+                        pCurBlock->uBytesUsed = (uPos%uBlockSize)+nBytes;
+                rMyriad.setBlock( uCurBlock, pCurBlock );
+                uPos += nBytes;
+                if( uPos > uSize )
+                        uSize = uPos;
+                //printf("block %u = %ub (%ub total) %d:%u\n",
+                //      uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, uPos );
+                return nBytes;
+        }
+        else
+        {
+                size_t nTotal = 0;
+                for(;;)
+                {
+                        uint32_t uNow = uBlockSize-(uPos%uBlockSize);
+                        //printf("uNow:  %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, uPos, uBlockSize, nBytes );
+                        if( nBytes < uNow )
+                                uNow = nBytes;
+                        memcpy( pCurBlock->pData+(uPos%uBlockSize),
+                                &((char *)pBuf)[nTotal], uNow );
+                        //printf("write buffill:  %ub, %u-%u/%u -> %d-%d/%d (b:%u:%u)\n",
+                        //      uNow, nTotal, nTotal+uNow-1, nBytes,
+                        //      uPos, uPos+uNow-1, uSize, uCurBlock, pCurBlock->uBytesUsed );
+                        if( (uPos%uBlockSize)+uNow > pCurBlock->uBytesUsed )
+                                pCurBlock->uBytesUsed = (uPos%uBlockSize)+uNow;
+                        rMyriad.setBlock( uCurBlock, pCurBlock );
+                        uPos += uNow;
+                        if( uPos > uSize )
+                                uSize = uPos;
+                        nTotal += uNow;
+                        nBytes -= uNow;
+                        //printf("wb: block %u = %ub (%ub total)\n",
+                        //      uCurBlock, pCurBlock->uBytesUsed, uSize );
+                        //if( pCurBlock->uBytesUsed == uBlockSize )
+                        if( uPos%uBlockSize == 0 )
+                                uCurBlock = rMyriad.getNextBlock( uCurBlock, pCurBlock );
+                        if( nBytes == 0 )
+                                return nTotal;
+                }
+        }*/
+        return 0;
+}
+
+long Bu::MyriadStream::tell()
+{
+        return uPos;
+}
+
+void Bu::MyriadStream::seek( long offset )
+{
+        uPos += offset;
+}
+
+void Bu::MyriadStream::setPos( long pos )
+{
+        uPos = pos;
+}
+
+void Bu::MyriadStream::setPosEnd( long pos )
+{
+        uPos = uSize-pos-1;
+}
+
+bool Bu::MyriadStream::isEos()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isOpen()
+{
+        return true;
+}
+
+void Bu::MyriadStream::flush()
+{
+}
+
+bool Bu::MyriadStream::canRead()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::canWrite()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isReadable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isWritable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isSeekable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isBlocking()
+{
+        return true;
+}
+
+void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ )
+{
+}
+
diff --git a/src/myriadstream.h b/src/myriadstream.h
new file mode 100644
index 0000000..755ff53
--- /dev/null
+++ b/src/myriadstream.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2007-2008 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#ifndef BU_MYRIAD_STREAM_H
+#define BU_MYRIAD_STREAM_H
+
+#include "bu/stream.h"
+#include "bu/myriad.h"
+
+namespace Bu
+{
+        class MyriadStream : public Bu::Stream
+        {
+                friend class Myriad;
+        private:
+                /**
+                 * These can only be created by the Myriad class.
+                 */
+                MyriadStream( Myriad &rMyriad, uint32_t uStream );
+
+        public:
+                virtual ~MyriadStream();
+
+                virtual void close();
+                virtual size_t read( void *pBuf, size_t nBytes );
+                virtual size_t write( const void *pBuf, size_t nBytes );
+                using Stream::write;
+                virtual long tell();
+                virtual void seek( long offset );
+                virtual void setPos( long pos );
+                virtual void setPosEnd( long pos );
+                virtual bool isEos();
+                virtual bool isOpen();
+                virtual void flush();
+                virtual bool canRead();
+                virtual bool canWrite();
+                virtual bool isReadable();
+                virtual bool isWritable();
+                virtual bool isSeekable();
+                virtual bool isBlocking();
+                virtual void setBlocking( bool bBlocking=true );
+
+        private:
+                Myriad &rMyriad;
+                uint32_t uStream;
+                char *pCurBlock;
+                uint32_t uCurBlock;
+                uint32_t uSize;
+                uint32_t uBlockSize;
+                uint32_t uPos; 
+        };
+};
+
+#endif
diff --git a/src/tests/nids.cpp b/src/tests/nids.cpp
index bb5993b..41d0432 100644
--- a/src/tests/nids.cpp
+++ b/src/tests/nids.cpp
@@ -23,18 +23,19 @@ int main( int argc, char *argv[] )
                         Bu::File::ReadWrite|Bu::File::Create|Bu::File::Truncate );
                 Bu::Nids n( fOut );
 
-                n.initialize( 120, 1 );
-                Bu::NidsStream s = n.openStream( n.createStream() );
+                n.initialize( 32, 1 );
+/*              Bu::NidsStream s = n.openStream( n.createStream() );
 
                 Bu::FString sBuf( 350 );
                 memset( sBuf.getStr(), 'a', 350 );
-                s.write( sBuf );
+                s.write( sBuf );*/
         }
         else if( argv[1][0] == 'r' )
         {
                 Bu::File fOut( argv[2], Bu::File::Read );
                 Bu::Nids n( fOut );
-
+                n.initialize();
+/*
                 Bu::NidsStream s = n.openStream( 0 );
                 char buf[75];
                 for( int j = 0; j < 3; j++ )
@@ -44,6 +45,7 @@ int main( int argc, char *argv[] )
                         printf("\n(read %d chars)\n", iRead );
                 }
                 printf("Position:  %ld\n", s.tell() );
+                */
         }
         else
         {
diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp
new file mode 100644
index 0000000..e951882
--- /dev/null
+++ b/src/tools/myriad.cpp
@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2007-2008 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/sio.h"
+#include "bu/file.h"
+#include "bu/myriad.h"
+#include "bu/myriadstream.h"
+#include "bu/optparser.h"
+
+#include <stdlib.h>
+
+using namespace Bu;
+
+enum Mode
+{
+        modeCreate,
+        modeInfo,
+
+        modeNone
+};
+
+class Options : public OptParser
+{
+public:
+        Options( int argc, char *argv[] ) :
+                eMode( modeNone ),
+                iBlockSize( 64 ),
+                iPreallocate( 0 )
+        {
+                addHelpBanner("Mode of operation:");
+                addOption( eMode, 'c', "create", "Create a new NIDS file." );
+                addOption( eMode, "info", "Display some info about a NIDS file." );
+                addHelpOption();
+
+                addHelpBanner("\nGeneral options:");
+                addOption( iBlockSize, 'b', "block-size", "Set the block size." );
+                addOption( iPreallocate, 'p', "preallocate",
+                        "Number of blocks to preallocate." );
+                addOption( sOutput, 'o', "output", "Set the output filename." );
+                addOption( sInput, 'i', "input", "Set the input filename." );
+
+                setOverride( "create", "create" );
+                setOverride( "info", "info" );
+
+                parse( argc, argv );
+        }
+
+        Mode eMode;
+        int iBlockSize;
+        int iPreallocate;
+        Bu::FString sOutput;
+        Bu::FString sInput;
+};
+
+Bu::Formatter &operator>>( Bu::Formatter &f, Mode &m )
+{
+        Bu::FString sTok = f.readToken();
+        if( sTok == "create" || sTok == "c" )
+                m = modeCreate;
+        else if( sTok == "info" )
+                m = modeInfo;
+        else
+                m = modeNone;
+        return f;
+}
+
+int main( int argc, char *argv[] )
+{
+        Options opts( argc, argv );
+
+        switch( opts.eMode )
+        {
+                case modeCreate:
+                        if( !opts.sOutput )
+                        {
+                                sio << "Please specify an output file to create a stream for."
+                                        << sio.nl;
+                                return 0;
+                        }
+                        else
+                        {
+                                File fOut( opts.sOutput, File::WriteNew );
+                                Myriad n( fOut );
+                                n.initialize( opts.iBlockSize, opts.iPreallocate );
+                        }
+                        break;
+
+                case modeInfo:
+                        if( !opts.sInput )
+                        {
+                                sio << "Please specify an input file to display info about."
+                                        << sio.nl;
+                                return 0;
+                        }
+                        else
+                        {
+                                File fIn( opts.sInput, File::Read );
+                                Myriad n( fIn );
+                                n.initialize();
+                        }
+                        break;
+
+                case modeNone:
+                        sio << "Please select a mode, for more info, try --help."
+                                << sio.nl << sio.nl;
+                        break;
+        }
+
+        return 0;
+}
+
diff --git a/src/unit/array.unit b/src/unit/array.unit
index b6528eb..daedd5f 100644
--- a/src/unit/array.unit
+++ b/src/unit/array.unit
@@ -76,7 +76,7 @@
 
         StrHashArray a3;
         a3 = a1;
-
+        
         unitTest( a3[0].get("Hi") == "Yo" );
         unitTest( a3[0].get("Bye") == "Later" );
         unitTest( a3[1].get("Test") == "Bloop" );