Myriad is actually fine, I double checked it for cross-platformed-ness. It

doesn't yet normalize the endian-ness, and I guess at this point to maintain
compatibility I'll have to make it a little endian format.  I would still like
to add better thread-safety to it, but that's about it.

4 files changed, 1255 insertions, 0 deletions

diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
new file mode 100644
index 0000000..de44930
--- /dev/null
+++ b/src/stable/myriad.cpp
@@ -0,0 +1,663 @@
+/*
+ * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/myriad.h"
+#include "bu/stream.h"
+#include "bu/myriadstream.h"
+#include <stdio.h>
+
+#include "bu/sio.h"
+using Bu::sio;
+using Bu::Fmt;
+
+#define Myriad_MAGIC_CODE               ((unsigned char *)"\x0a\xd3\xfa\x84")
+
+namespace Bu
+{
+        subExceptionDef( MyriadException )
+        template<typename t> t blkDiv( t total, t block ) {
+                return (total/block)+((total%block==0)?(0):(1));
+        }
+}
+
+Bu::Myriad::Myriad( Bu::Stream &sStore, int iBlockSize, int iPreallocate ) :
+        sStore( sStore ),
+        iBlockSize( iBlockSize ),
+        iBlocks( 0 ),
+        iUsed( 0 ),
+        bHeaderChanged( false )
+{
+        try
+        {
+                initialize();
+        }
+        catch( Bu::MyriadException &e )
+        {
+                if( e.getErrorCode() == MyriadException::emptyStream )
+                {
+                        initialize( iBlockSize, iPreallocate );
+                }
+                else
+                {
+                        throw;
+                }
+        }
+}
+
+Bu::Myriad::~Myriad()
+{
+        if( !hActiveBlocks.isEmpty() )
+        {
+                sio << "Bu::Myriad::~Myriad(): Error: There are "
+                        << hActiveBlocks.getSize() << " unsynced blocks!" << sio.nl;
+        }
+        sync();
+
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                delete *i;
+        }
+}
+
+void Bu::Myriad::sync()
+{
+        updateHeader();
+
+        for( BlockHash::iterator i = hActiveBlocks.begin(); i; i++ )
+        {
+                if( (*i)->bChanged )
+                {
+                        syncBlock( *i );
+                }
+        }
+}
+
+void Bu::Myriad::initialize()
+{
+        sStore.setPosEnd( 0 );
+        int iSize = sStore.tell();
+        sStore.setPos( 0 );
+
+        unsigned char buf[4];
+        if( sStore.read( buf, 4 ) < 4 )
+                throw MyriadException( MyriadException::emptyStream,
+                                "Input stream appears to be empty.");
+        if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
+        {
+                throw MyriadException( MyriadException::invalidFormat,
+                        "Stream does not appear to be a valid Myriad format.");
+        }
+        sStore.read( buf, 2 );
+        if( buf[0] != 1 )
+                throw MyriadException( MyriadException::badVersion,
+                        "We can only handle version 1 for now.");
+        if( buf[1] != 32 )
+                throw MyriadException( MyriadException::invalidWordSize,
+                        "We can only handle 32-bit words at the moment.");
+        sStore.read( &iBlockSize, 4 );
+        int iStreams;
+        sStore.read( &iStreams, 4 );
+
+        iBlocks = iSize/iBlockSize;
+        //sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize
+        //      << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl;
+
+        int iHeaderSize = 14 + 8 + 4;
+        int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
+
+        while( iHeaderSize > iHeaderBlocks*iBlockSize )
+        {
+                iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+                iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
+        }
+
+        //sio << "Myriad: iHeaderSize=" << iHeaderSize
+        //      << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
+
+        Stream *pFakeHdr = new Stream;
+        pFakeHdr->iId = 0;
+        pFakeHdr->iSize = iHeaderSize;
+        for( int j = 0; j < iHeaderBlocks; j++ )
+        {
+                pFakeHdr->aBlocks.append( j );
+        }
+        
+        bsBlockUsed.setSize( iBlocks, true );
+
+//      bool bCanSkip = false; // Can skip around, post initial header stream i/o
+        MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
+        pIn->setPos( sStore.tell() );
+        for( int j = 0; j < iStreams; j++ )
+        {
+                aStreams.append( new Stream() );
+                Stream &s = *aStreams[j];
+                pIn->read( &s.iId, 4 );
+                pIn->read( &s.iSize, 4 );
+                int iSBlocks = blkDiv(s.iSize, iBlockSize);
+        //      sio << "Myriad: - Stream::iId=" << s.iId
+        //              << ", Stream::iSize=" << s.iSize
+        //              << ", Stream::aBlocks=" << iSBlocks
+        //              << ", pIn->tell()=" << pIn->tell() << sio.nl;
+                for( int k = 0; k < iSBlocks; k++ )
+                {
+                        int iBId;
+                        pIn->read( &iBId, 4 );
+        //              sio << "Myriad:   - iBId=" << iBId
+        //                      << ", iStartPos=" << iBId*iBlockSize
+        //                      << ", pIn->tell()=" << pIn->tell() << sio.nl;
+                        s.aBlocks.append( iBId );
+                        bsBlockUsed.setBit( iBId );
+                        iUsed++;
+                        if( (j == 0 && k == iHeaderBlocks-1) )
+                        {
+        //                      sio << "Myriad:   - End of prepartition, unlocking skipping."
+        //                              << sio.nl;
+//                              bCanSkip = true;
+                                MyriadStream *pTmp = new MyriadStream( *this, aStreams[0] );
+        //                      sio << "Myriad    - Position = " << pIn->tell() << sio.nl;
+                                pTmp->setPos( pIn->tell() );
+                                delete pIn;
+                                delete pFakeHdr;
+                                pIn = pTmp;
+                        }
+                }
+        }
+        delete pIn;
+
+        //sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl;
+}
+
+void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
+{
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                delete *i;
+        }
+        aStreams.clear();
+        iUsed = 0;
+
+        int iHeaderSize = 14 + 8 + 4;
+        int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
+        char cBuf = 1;
+        int iBuf = 0;
+        
+        Stream *pStr = new Stream;
+        pStr->iId = 0;
+
+        while( iHeaderSize > iHeaderBlocks*iBlockSize )
+        {
+                iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
+                iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
+        }
+
+        iPreAllocate += iHeaderBlocks;
+
+        //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize="
+        //      << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
+        
+        bsBlockUsed.setSize( iPreAllocate, true );
+        iUsed++;
+
+        char *pBlock = new char[iBlockSize];
+        memset( pBlock, 0, iBlockSize );
+        for( int j = 0; j < iPreAllocate; j++ )
+        {
+                sStore.write( pBlock, iBlockSize );
+        }
+        delete[] (char *)pBlock;
+
+        sStore.setPos( 0 );
+
+        // Magic number
+        sStore.write( Myriad_MAGIC_CODE, 4 );
+
+        // Version (0)
+        sStore.write( &cBuf, 1 );
+
+        // Bits per int
+        cBuf = 32;
+        sStore.write( &cBuf, 1 );
+
+        // The size of each block
+        sStore.write( &iBlockSize, 4 );
+
+        iBuf = 1;
+        // The number of streams
+        sStore.write( &iBuf, 4 );
+        
+        // Stream header
+        iBuf = 0;
+        sStore.write( &iBuf, 4 );
+        sStore.write( &iHeaderSize, 4 );
+        for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ )
+        {
+                sStore.write( &iBuf, 4 );
+        }
+
+        this->iBlockSize = iBlockSize;
+        this->iBlocks = iPreAllocate;
+        
+        pStr->iSize = sStore.tell();
+//      sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl;
+
+        pStr->iSize = iHeaderSize;
+        for( int j = 0; j < iHeaderBlocks; j++ )
+        {
+                pStr->aBlocks.append( j );
+                bsBlockUsed.setBit( j );
+                iUsed++;
+        }
+
+        aStreams.append( pStr );
+
+        //sio << bsBlockUsed.toString() << " - " << pStr->aBlocks << sio.nl;
+
+        bHeaderChanged = true;
+        //hStreams.insert( 0, BlockArray( 0 ) );
+}
+
+void Bu::Myriad::updateHeader()
+{
+        if( bHeaderChanged == false )
+                return;
+        if( !sStore.canWrite() )
+                return;
+
+        char cBuf;
+        int iBuf;
+
+        //for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        //{
+        //      sio << "Myriad: Stream " << Fmt(4) << (*i)->iId << ": " << (*i)->aBlocks << sio.nl;
+        //}
+
+        // Compute the new size of the header.
+        int iHeaderSize = 14 + 8*aStreams.getSize();
+//      sio << "Myriad: updateHeader: aStreams.getSize() = " << aStreams.getSize()
+//              << sio.nl;
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                iHeaderSize += 4*(*i)->aBlocks.getSize();
+//              sio << "Myriad: updateHeader: (*i)->aBlocks.getSize() = "
+//                      << (*i)->aBlocks.getSize() << sio.nl;
+        }
+        int iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
+        while( iNewBlocks > aStreams[0]->aBlocks.getSize() )
+        {
+                int iBlock = findEmptyBlock();
+//              sio << "Myriad: updateHeader: Appending block " << iBlock
+//                      << " to header." << sio.nl;
+                aStreams[0]->aBlocks.append( iBlock );
+                bsBlockUsed.setBit( iBlock );
+                iUsed++;
+                iHeaderSize += 4;
+                iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
+        }
+        aStreams[0]->iSize = iHeaderSize;
+//      sio << "Myriad: updateHeader: iHeaderSize=" << iHeaderSize
+//              << ", iNewBlocks=" << iNewBlocks << ", curBlocks="
+//              << aStreams[0]->aBlocks.getSize() << sio.nl;
+
+        MyriadStream sHdr( *this, aStreams[0] );
+        sHdr.write( Myriad_MAGIC_CODE, 4 );
+
+        // Version (1)
+        cBuf = 1;
+        sHdr.write( &cBuf, 1 );
+
+        // Bits per int
+        cBuf = 32;
+        sHdr.write( &cBuf, 1 );
+
+        // The size of each block
+        sHdr.write( &iBlockSize, 4 );
+
+        iBuf = aStreams.getSize();
+        // The number of streams
+        sHdr.write( &iBuf, 4 );
+        
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                sHdr.write( &(*i)->iId, 4 );
+                sHdr.write( &(*i)->iSize, 4 );
+                int iUsedBlocks = blkDiv( (*i)->iSize, iBlockSize );
+//              for( BlockArray::iterator j = (*i)->aBlocks.begin(); j; j++ )
+                for( int j = 0; j < iUsedBlocks; j++ )
+                {
+                        sHdr.write( &(*i)->aBlocks[j], 4 );
+                }
+        }
+        
+        bHeaderChanged = false;
+}
+
+int Bu::Myriad::createStream( int iPreAllocate )
+{
+        Stream *pStr = new Stream();
+        pStr->iId = aStreams.last()->iId+1;
+        //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
+        //      << iPreAllocate << sio.nl;
+        pStr->iSize = 0;
+        aStreams.append( pStr );
+
+        for( int j = 0; j < iPreAllocate; j++ )
+        {
+                int iFreeBlock = findEmptyBlock();
+//              sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
+                pStr->aBlocks.append( iFreeBlock );
+                bsBlockUsed.setBit( iFreeBlock );
+                iUsed++;
+        }
+
+        bHeaderChanged = true;
+
+        return pStr->iId;
+}
+
+int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate )
+{
+        try
+        {
+                findStream( iId );
+                throw MyriadException( MyriadException::streamExists,
+                        "There is already a stream with the given id.");
+        }
+        catch( MyriadException &e )
+        {
+                Stream *pStr = new Stream();
+                pStr->iId = iId;
+                //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
+                //      << iPreAllocate << sio.nl;
+                pStr->iSize = 0;
+                if( aStreams.last()->iId < iId )
+                {
+                        aStreams.append( pStr );
+                }
+                else
+                {
+                        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+                        {
+                                if( (*i)->iId > iId )
+                                {
+                                        aStreams.insert( i, pStr );
+                                        break;
+                                }
+                        }
+                }
+
+                for( int j = 0; j < iPreAllocate; j++ )
+                {
+                        int iFreeBlock = findEmptyBlock();
+        //              sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
+                        pStr->aBlocks.append( iFreeBlock );
+                        bsBlockUsed.setBit( iFreeBlock );
+                        iUsed++;
+                }
+
+                bHeaderChanged = true;
+
+                return pStr->iId;
+        }
+}
+
+int Bu::Myriad::findEmptyBlock()
+{
+        bHeaderChanged = true;
+
+        for( int j = 0; j < bsBlockUsed.getSize(); j++ )
+        {
+                if( bsBlockUsed.getBit( j ) == false )
+                        return j;
+        }
+//      sio << "Myriad: findEmptyBlock(): No empty blocks, adding new one." << sio.nl;
+
+        bsBlockUsed.setSize( bsBlockUsed.getSize()+1, false );
+        /*
+        sStore.setPos( iBlockSize*iBlocks );
+
+        char *pBlock = new char[iBlockSize];
+        memset( pBlock, 0, iBlockSize );
+        sStore.write( pBlock, iBlockSize );
+        delete[] pBlock;
+        */
+
+        sStore.setSize( (iBlocks+1)*iBlockSize );
+
+        return iBlocks++;
+}
+
+void Bu::Myriad::deleteStream( int iId )
+{
+        if( iId < 0 )
+                throw MyriadException( MyriadException::invalidStreamId,
+                                "Invalid stream id.");
+        if( iId == 0 )
+                throw MyriadException( MyriadException::protectedStream,
+                                "You cannot delete stream zero, it is protected.");
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                if( (*i)->iId == iId )
+                {
+                        Stream *pStream = *i;
+                        for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
+                        {
+                                bsBlockUsed.setBit( *j, false );
+                                iUsed--;
+                        }
+                        aStreams.erase( i );
+                        bHeaderChanged = true;
+                        delete pStream;
+                        return;
+                }
+        }
+}
+
+Bu::Array<int> Bu::Myriad::getStreamIds()
+{
+        Bu::Array<int> aRet( aStreams.getSize() );
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                aRet.append( (*i)->iId );
+        }
+
+        return aRet;
+}
+
+int Bu::Myriad::getStreamSize( int iId )
+{
+        return findStream( iId )->iSize;
+}
+
+bool Bu::Myriad::hasStream( int iId )
+{
+        try
+        {
+                findStream( iId );
+                return true;
+        }catch(...)
+        {
+                return false;
+        }
+}
+
+Bu::MyriadStream Bu::Myriad::openStream( int iId )
+{
+        //sio << "Myriad:  Request to open stream: " << iId << sio.nl;
+        return MyriadStream( *this, findStream( iId ) );
+}
+
+int Bu::Myriad::getNumStreams()
+{
+        return aStreams.getSize();
+}
+
+int Bu::Myriad::getBlockSize()
+{
+        return iBlockSize;
+}
+
+int Bu::Myriad::getNumBlocks()
+{
+        return iBlocks;
+}
+
+int Bu::Myriad::getNumUsedBlocks()
+{
+        return iUsed;
+}
+
+int Bu::Myriad::getTotalUsedBytes()
+{
+        int iTotalSize = 0;
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                iTotalSize += (*i)->iSize;
+        }
+        return iTotalSize;
+}
+
+int Bu::Myriad::getTotalUnusedBytes()
+{
+        int iTotalSize = (iBlocks-iUsed)*iBlockSize;
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                iTotalSize += iBlockSize - ((*i)->iSize%iBlockSize);
+        }
+        return iTotalSize;
+}
+
+int Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize )
+{
+        int iTotalSize = (iBlocks-iUsed)*iFakeBlockSize;
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize);
+        }
+        return iTotalSize;
+}
+
+Bu::Myriad::Stream *Bu::Myriad::findStream( int iId )
+{
+        for( StreamArray::iterator i = aStreams.begin(); i; i++ )
+        {
+                if( (*i)->iId == iId )
+                        return *i;
+        }
+
+        throw MyriadException( MyriadException::noSuchStream,
+                "The requested stream doesn't exist and cannot be opened." );
+
+        return NULL;
+}
+
+Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock )
+{
+//      sio << "Myriad:  Reading block " << iBlock << ", bytes "
+//              << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl;
+        Block *pBlock = new Block;
+        pBlock->pData = new char[iBlockSize];
+        sStore.setPos( iBlockSize * iBlock );
+        sStore.read( pBlock->pData, iBlockSize );
+        pBlock->bChanged = false;
+        pBlock->iBlockIndex = iBlock;
+
+        hActiveBlocks.insert( iBlock, pBlock );
+
+        return pBlock;
+}
+
+void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock )
+{
+        if( pBlock == NULL )
+                return;
+//      sio << "Myriad:  Releasing block " << pBlock->iBlockIndex << sio.nl;
+        syncBlock( pBlock );
+        hActiveBlocks.erase( pBlock->iBlockIndex );
+        delete[] pBlock->pData;
+        delete pBlock;
+}
+
+void Bu::Myriad::syncBlock( Block *pBlock )
+{
+        if( pBlock->bChanged )
+        {
+//              sio << "Myriad:    - Block changed, writing back to stream." << sio.nl;
+                sStore.setPos( iBlockSize * pBlock->iBlockIndex );
+                sStore.write( pBlock->pData, iBlockSize );
+                pBlock->bChanged = false;
+        }
+}
+
+int Bu::Myriad::streamAddBlock( Stream *pStream )
+{
+        int iBlock = findEmptyBlock();
+        pStream->aBlocks.append( iBlock );
+        bsBlockUsed.setBit( iBlock );
+        bHeaderChanged = true;
+        iUsed++;
+        return iBlock;
+}
+
+void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
+{
+        if( pStream->iSize == iSize )
+        {
+                return;
+        }
+        else if( pStream->iSize > iSize )
+        {
+                // Shrink
+                for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
+                         iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize )
+                {
+                        if( bsBlockUsed.getBit( pStream->aBlocks.last() ) )
+                                iUsed--;
+                        bsBlockUsed.setBit( pStream->aBlocks.last(), false );
+                        pStream->aBlocks.eraseLast();
+                }
+                pStream->iSize = iSize;
+                bHeaderChanged = true;
+        }
+        else
+        {
+                // Grow
+                for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
+                         iNewSize < iSize; iNewSize += iBlockSize )
+                {
+                        streamAddBlock( pStream );
+                }
+                pStream->iSize = iSize;
+                bHeaderChanged = true;
+        }
+}
+
+void Bu::Myriad::headerChanged()
+{
+        bHeaderChanged = true;
+}
+
+bool Bu::Myriad::isMyriad( Bu::Stream &sStore )
+{
+        sStore.setPos( 0 );
+
+        unsigned char buf[4];
+        if( sStore.read( buf, 4 ) < 4 )
+                throw MyriadException( MyriadException::emptyStream,
+                                "Input stream appears to be empty.");
+        sStore.setPos( 0 );
+        if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
+        {
+                return false;
+        }
+        return true;
+}
+
+const Bu::BitString &Bu::Myriad::getBlocksUsed() const
+{
+        return bsBlockUsed;
+}
+
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
new file mode 100644
index 0000000..3382ab5
--- /dev/null
+++ b/src/stable/myriad.h
@@ -0,0 +1,222 @@
+/*
+ * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#ifndef BU_MYRIAD_H
+#define BU_MYRIAD_H
+
+#include <stdint.h>
+#include "bu/bitstring.h"
+#include "bu/exceptionbase.h"
+#include "bu/array.h"
+#include "bu/hash.h"
+
+namespace Bu
+{
+        class Stream;
+        class MyriadStream;
+
+        subExceptionDeclBegin( MyriadException )
+                enum
+                {
+                        emptyStream,
+                        invalidFormat,
+                        badVersion,
+                        invalidWordSize,
+                        noSuchStream,
+                        streamExists,
+                        invalidStreamId,
+                        protectedStream
+                };
+        subExceptionDeclEnd();
+
+        /**
+         * Myriad block-allocated stream multiplexing system.  This is a system for
+         * creating streams that contain other streams in a flexible and lightweight
+         * manner.  Basically, you can create a file (or any other stream) that can
+         * store any number of flexible, growing streams.  The streams within the
+         * Myriad stream are automatically numbered, not named.  This works more
+         * or less like a filesystem, but without the extra layer for managing
+         * file and directory links.  This would actually be very easy to add
+         * on top of Myriad, but is not required.
+         *
+         * Header format is as follows:
+         *
+         * MMMMvBssssSSSS*
+         * M = Magic number (0AD3FA84)
+         * v = version number
+         * B = Bits per int
+         * s = Blocksize in bytes
+         * S = Number of Streams
+         * 
+         * The * represents the Stream headers, one per stream, as follows:
+         * IIIIssss$
+         * I = Id number of the stream
+         * s = size of stream in bytes
+         *
+         * The $ represents the Block headers, one per used block, as  follows:
+         * IIII
+         * I = Index of the block
+         *
+         * The stream/block data is interleaved in the header, so all blocks stored
+         * with one stream are together.  The block headers are in order, and the
+         * data in them is required to be "solid" you cannot fill partial blocks
+         * mid-way through a stream.
+         *
+         * The initial block starts with the nids header, and is both the zero block
+         * and the zero stream.  For now, the minimum block size is the size needed
+         * to store the base header, the zero stream header, and the first two
+         * blocks of the zero stream, so 30 bytes.  Since it's reccomended to use
+         * a size that will fit evenly into filesystem blocks, then a size of 32 is
+         * probably the smallest reccomended size because all powers of two equal
+         * to or greater than 32 are evenly divisible by 32.
+         *
+         * I have had a thought that if the block size were smaller than 42 bytes
+         * the header would consume the first N blocks where N * block size is
+         * enough space to house the initial header, the first stream header, and
+         * the first N block headers.  This, of course, causes you to hit an
+         * infinite header if the block size is small enough.
+         */
+        class Myriad
+        {
+                friend class MyriadStream;
+        public:
+                /**
+                 * Create a Myriad object that uses the given stream to store data.
+                 * This stream must be random access.  The block size and preallocate
+                 * values passed in are values that will be used if the given stream
+                 * is empty.  In that case the stream will be "formatted" for myriad
+                 * with the specified block size.  If there is already a viable Myriad
+                 * format present in the stream, then the blocksize and preallocate
+                 * values will be ignored and the values from the stream will be used
+                 * instead.  If the stream doesn't appear to be Myriad formatted an
+                 * exception will be thrown.
+                 */
+                Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 );
+                virtual ~Myriad();
+
+                /**
+                 * Destroy whatever data may be in the base stream and create a new
+                 * Myriad system there with the given blocksize.  Use this with care,
+                 * it will destroy anything that was already in the stream, and
+                 * generally, should not ever have to be used.
+                 */
+                void initialize( int iBlockSize, int iPreAllocate=1 );
+
+                /**
+                 * Create a new stream within the Myriad system.  The ID of the new
+                 * stream is returned.
+                 */
+                int createStream( int iPreAllocate=1 );
+
+                /**
+                 * Create a new stream within the Myriad system with a given id.  The
+                 * id that you provide will be the new id of the stream unless it's
+                 * already used, in which case an error is thrown.  This is primarilly
+                 * useful when copying an old Myriad file into a new one.
+                 */
+                int createStreamWithId( int iId, int iPreAllocate=1 );
+
+                /**
+                 * Delete a stream that's already within the Myriad.
+                 */
+                void deleteStream( int iId );
+
+                /**
+                 * Return a new Stream object assosiated with the given stream ID.
+                 */
+                MyriadStream openStream( int iId );
+
+                Bu::Array<int> getStreamIds();
+                int getStreamSize( int iId );
+                bool hasStream( int iId );
+
+                int getNumStreams();
+                int getBlockSize();
+                int getNumBlocks();
+                int getNumUsedBlocks();
+                int getTotalUsedBytes();
+                int getTotalUnusedBytes();
+                int getTotalUnusedBytes( int iFakeBlockSize );
+
+                /**
+                 * Syncronize the header data, etc. with the storage stream.  It's not
+                 * a bad idea to call this periodically.
+                 */
+                void sync();
+
+                /**
+                 * Read the first few bytes from the given stream and return true/false
+                 * depending on weather or not it's a Myriad stream.  This will throw
+                 * an exception if the stream is empty, or is not random access.
+                 */
+                static bool isMyriad( Bu::Stream &sStore );
+
+                const Bu::BitString &getBlocksUsed() const;
+
+        private:
+                /**
+                 * Initialize this object based on the data already in the assosiated
+                 * stream.  This will be called automatically for you if you forget,
+                 * but if you want to pre-initialize for some reason, just call this
+                 * once before you actually start doing anything with your Myriad.
+                 */
+                void initialize();
+
+                enum
+                {
+                        blockUnused     =       0xFFFFFFFFUL
+                };
+                
+                typedef Bu::Array<int> BlockArray;
+                class Stream
+                {
+                public:
+                        int iId;
+                        int iSize;
+                        BlockArray aBlocks;
+                };
+                typedef Bu::Array<Stream *> StreamArray;
+
+                class Block
+                {
+                public:
+                        char *pData;
+                        bool bChanged;
+                        int iBlockIndex;
+                };
+
+                void updateHeader();
+                int findEmptyBlock();
+
+                /**
+                 *@todo Change this to use a binary search, it's nicer.
+                 */
+                Stream *findStream( int iId );
+
+                Block *getBlock( int iBlock );
+                void releaseBlock( Block *pBlock );
+                void syncBlock( Block *pBlock );
+
+                int streamAddBlock( Stream *pStream );
+                void setStreamSize( Stream *pStream, long iSize );
+
+                void headerChanged();
+
+        private:
+                Bu::Stream &sStore;
+                int iBlockSize;
+                int iBlocks;
+                int iUsed;
+                Bu::BitString bsBlockUsed;
+                StreamArray aStreams;
+                typedef Bu::Hash<int, Block *> BlockHash;
+                BlockHash hActiveBlocks;
+                bool bHeaderChanged;
+        };
+};
+
+#endif
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
new file mode 100644
index 0000000..e6e94bc
--- /dev/null
+++ b/src/stable/myriadstream.cpp
@@ -0,0 +1,309 @@
+/*
+ * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#include "bu/myriadstream.h"
+
+#include <string.h>
+
+// #define MYRIAD_STREAM_DEBUG  1
+
+#ifdef MYRIAD_STREAM_DEBUG
+#include "bu/sio.h"
+
+using Bu::sio;
+using Bu::Fmt;
+#endif
+
+Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad,
+                Bu::Myriad::Stream *pStream ) :
+        rMyriad( rMyriad ),
+        pStream( pStream ),
+        pCurBlock( NULL ),
+        iPos( 0 )
+{
+#ifdef MYRIAD_STREAM_DEBUG
+        sio << "MyriadStream: " << __LINE__ << ": Created, iId=" << pStream->iId << ", iSize="
+                << pStream->iSize << sio.nl;
+#endif
+        //pCurBlock = rMyriad.newBlock();
+        //rMyriad.getBlock( uStream, pCurBlock );
+        //uSize = pCurBlock->uBytesUsed;
+}
+
+Bu::MyriadStream::~MyriadStream()
+{
+        if( pCurBlock )
+                rMyriad.releaseBlock( pCurBlock );
+        //rMyriad.updateStreamSize( uStream, uSize );
+        //rMyriad.deleteBlock( pCurBlock );
+}
+
+void Bu::MyriadStream::close()
+{
+}
+
+Bu::size Bu::MyriadStream::read( void *pBuf, Bu::size nBytes )
+{
+#ifdef MYRIAD_STREAM_DEBUG
+        sio << "MyriadStream: read: " << __LINE__ << ": Started, asked to read " << nBytes << "b."
+                << sio.nl;
+#endif
+        if( nBytes > (Bu::size)pStream->iSize-iPos )
+                nBytes = pStream->iSize-iPos;
+        if( nBytes <= 0 )
+                return 0;
+        int iLeft = nBytes;
+#ifdef MYRIAD_STREAM_DEBUG
+        sio << "MyriadStream: read: " << __LINE__ << ": Started, going to read " << nBytes << "b."
+                << sio.nl;
+#endif
+        if( pCurBlock == NULL )
+        {
+#ifdef MYRIAD_STREAM_DEBUG
+                sio << "MyriadStream: read: " << __LINE__ << ": No block loaded, loading initial block."
+                        << sio.nl;
+#endif
+                pCurBlock = rMyriad.getBlock(
+                        pStream->aBlocks[iPos/rMyriad.iBlockSize]
+                        );
+        }
+        while( iLeft > 0 )
+        {
+                int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
+                if( pCurBlock->iBlockIndex != iCurBlock )
+                {
+#ifdef MYRIAD_STREAM_DEBUG
+                        sio << "MyriadStream: read: " << __LINE__ << ": Loading new block " << iCurBlock << "."
+                                << sio.nl;
+#endif
+                        rMyriad.releaseBlock( pCurBlock );
+                        pCurBlock = rMyriad.getBlock( iCurBlock );
+                }
+
+                int iAmnt = Bu::min(
+                        Bu::min(
+                                rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
+                                iLeft
+                                ),
+                        pStream->iSize-iPos
+                        );
+#ifdef MYRIAD_STREAM_DEBUG
+                sio << "MyriadStream: read: " << __LINE__ << ": Copying out bytes: "
+                        << iPos << "(" << (iPos%rMyriad.iBlockSize) << ")+"
+                        << iAmnt
+                        << ", " << iLeft << "b left." << sio.nl;
+#endif
+                memcpy(
+                        pBuf,
+                        pCurBlock->pData+(iPos%rMyriad.iBlockSize),
+                        iAmnt
+                        );
+                iPos += iAmnt;
+                pBuf = &((char *)pBuf)[iAmnt];
+                iLeft -= iAmnt;
+        }
+        return nBytes;
+}
+
+Bu::size Bu::MyriadStream::write( const void *pBuf, Bu::size nBytes )
+{
+        if( nBytes <= 0 )
+                return 0;
+
+#ifdef MYRIAD_STREAM_DEBUG
+        sio << "MyriadStream: write: " << __LINE__ << ": Started, asked to write " << nBytes << "b."
+                << sio.nl;
+#endif
+        if( nBytes <= 0 )
+                return 0;
+        int iLeft = nBytes;
+        /*
+        if( pCurBlock == NULL )
+        {
+#ifdef MYRIAD_STREAM_DEBUG
+                sio << "MyriadStream: write: No block loaded, loading initial block."
+                        << sio.nl;
+#endif
+                pCurBlock = rMyriad.getBlock(
+                        pStream->aBlocks[iPos/rMyriad.iBlockSize]
+                        );
+        }*/
+        
+        while( iLeft > 0 )
+        {
+                int iCurBlock;
+                if( iPos/rMyriad.iBlockSize < pStream->aBlocks.getSize() )
+                {
+                        iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
+                }
+                else
+                {
+                        iCurBlock = rMyriad.streamAddBlock( pStream );
+#ifdef MYRIAD_STREAM_DEBUG
+                        sio << "MyriadStream: write: " << __LINE__ << ": New block allocated and appended: "
+                                << iCurBlock << "." << sio.nl;
+
+#endif
+                }
+                if( !pCurBlock || pCurBlock->iBlockIndex != iCurBlock )
+                {
+#ifdef MYRIAD_STREAM_DEBUG
+                        sio << "MyriadStream: write: " << __LINE__ << ": Loading new block " << iCurBlock << "."
+                                << sio.nl;
+#endif
+                        rMyriad.releaseBlock( pCurBlock );
+                        pCurBlock = rMyriad.getBlock( iCurBlock );
+                }
+                pCurBlock->bChanged = true;
+                
+                // There are two main writing modes when it comes down to it.
+                // Overwrite mode and append mode.  Append is what pretty much always
+                // happens when creating a new stream.
+                if( iPos < pStream->iSize )
+                {
+                        int iAmnt = Bu::min(
+                                Bu::min(
+                                        rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
+                                        iLeft
+                                        ),
+                                pStream->iSize-iPos
+                                );
+#ifdef MYRIAD_STREAM_DEBUG
+                        sio << "MyriadStream: write (ovr): " << __LINE__ << ": Copying in bytes: "
+                                << (iPos%rMyriad.iBlockSize) << "+"
+                                << iAmnt
+                                << ", " << iLeft << "b left." << sio.nl;
+#endif
+                        memcpy(
+                                pCurBlock->pData+(iPos%rMyriad.iBlockSize),
+                                pBuf,
+                                iAmnt
+                                );
+                        iPos += iAmnt;
+                        pBuf = &((char *)pBuf)[iAmnt];
+                        iLeft -= iAmnt;
+                }
+                else
+                {
+                        int iAmnt = Bu::min(
+                                rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
+                                iLeft
+                                );
+#ifdef MYRIAD_STREAM_DEBUG
+                        sio << "MyriadStream: write (app): " << __LINE__ << ": Copying in bytes: "
+                                << (iPos%rMyriad.iBlockSize) << "+"
+                                << iAmnt
+                                << ", " << iLeft << "b left." << sio.nl;
+#endif
+                        memcpy(
+                                pCurBlock->pData+(iPos%rMyriad.iBlockSize),
+                                pBuf,
+                                iAmnt
+                                );
+                        iPos += iAmnt;
+                        pStream->iSize += iAmnt;
+                        rMyriad.headerChanged();
+                        pBuf = &((char *)pBuf)[iAmnt];
+                        iLeft -= iAmnt;
+                }
+        }
+
+        return nBytes;
+}
+
+Bu::size Bu::MyriadStream::tell()
+{
+        return iPos;
+}
+
+void Bu::MyriadStream::seek( Bu::size offset )
+{
+        iPos += offset;
+}
+
+void Bu::MyriadStream::setPos( Bu::size pos )
+{
+        iPos = pos;
+}
+
+void Bu::MyriadStream::setPosEnd( Bu::size pos )
+{
+        iPos = pStream->iSize-pos;
+}
+
+bool Bu::MyriadStream::isEos()
+{
+        return iPos >= pStream->iSize;
+}
+
+bool Bu::MyriadStream::isOpen()
+{
+        return true;
+}
+
+void Bu::MyriadStream::flush()
+{
+}
+
+bool Bu::MyriadStream::canRead()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::canWrite()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isReadable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isWritable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isSeekable()
+{
+        return true;
+}
+
+bool Bu::MyriadStream::isBlocking()
+{
+        return true;
+}
+
+void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ )
+{
+}
+
+void Bu::MyriadStream::setSize( Bu::size iSize )
+{
+        if( iSize < 0 )
+                iSize = 0;
+        rMyriad.setStreamSize( pStream, iSize );
+        if( iPos > iSize )
+                iPos = iSize;
+}
+
+Bu::size Bu::MyriadStream::getSize() const
+{
+        return pStream->iSize;
+}
+
+Bu::size Bu::MyriadStream::getBlockSize() const
+{
+        return rMyriad.getBlockSize();
+}
+
+Bu::String Bu::MyriadStream::getLocation() const
+{
+        return Bu::String("%1").arg( pStream->iId );
+}
+
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
new file mode 100644
index 0000000..fdad669
--- /dev/null
+++ b/src/stable/myriadstream.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#ifndef BU_MYRIAD_STREAM_H
+#define BU_MYRIAD_STREAM_H
+
+#include "bu/stream.h"
+#include "bu/myriad.h"
+
+namespace Bu
+{
+        class MyriadStream : public Bu::Stream
+        {
+                friend class Myriad;
+        private:
+                /**
+                 * These can only be created by the Myriad class.
+                 */
+                MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream );
+
+        public:
+                virtual ~MyriadStream();
+
+                virtual void close();
+                virtual Bu::size read( void *pBuf, Bu::size nBytes );
+                virtual Bu::size write( const void *pBuf, Bu::size nBytes );
+                using Stream::write;
+                virtual Bu::size tell();
+                virtual void seek( Bu::size offset );
+                virtual void setPos( Bu::size pos );
+                virtual void setPosEnd( Bu::size pos );
+                virtual bool isEos();
+                virtual bool isOpen();
+                virtual void flush();
+                virtual bool canRead();
+                virtual bool canWrite();
+                virtual bool isReadable();
+                virtual bool isWritable();
+                virtual bool isSeekable();
+                virtual bool isBlocking();
+                virtual void setBlocking( bool bBlocking=true );
+                virtual void setSize( Bu::size iSize );
+
+                virtual size getSize() const;
+                virtual size getBlockSize() const;
+                virtual Bu::String getLocation() const;
+
+        private:
+                Myriad &rMyriad;
+                Myriad::Stream *pStream;
+                Myriad::Block *pCurBlock;
+                int iBlockSize;
+                int iPos; 
+        };
+};
+
+#endif