1 files changed, 181 insertions, 166 deletions

diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 14467a4..5accd1e 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -1,24 +1,16 @@
-/*
- * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
- *
- * This file is part of the libbu++ library and is released under the
- * terms of the license contained in the file LICENSE.
- */
-
 #ifndef BU_MYRIAD_H
 #define BU_MYRIAD_H
 
-#include <stdint.h>
-#include "bu/bitstring.h"
+#include "bu/stream.h"
 #include "bu/exceptionbase.h"
+#include "bu/mutex.h"
 #include "bu/array.h"
 #include "bu/hash.h"
-#include "bu/mutex.h"
-#include "bu/extratypes.h"
+
+#include "bu/bitstring.h"
 
 namespace Bu
 {
-    class Stream;
     class MyriadStream;
 
     subExceptionDeclBegin( MyriadException )
@@ -31,206 +23,229 @@ namespace Bu
             noSuchStream,
             streamExists,
             invalidStreamId,
-            protectedStream
+            protectedStream,
+            invalidParameter,
+            invalidBackingStream,
+            badMode,
+            streamOpen,
         };
     subExceptionDeclEnd();
 
     /**
-     * Myriad block-allocated stream multiplexing system.  This is a system for
-     * creating streams that contain other streams in a flexible and lightweight
-     * manner.  Basically, you can create a file (or any other stream) that can
-     * store any number of flexible, growing streams.  The streams within the
-     * Myriad stream are automatically numbered, not named.  This works more
-     * or less like a filesystem, but without the extra layer for managing
-     * file and directory links.  This would actually be very easy to add
-     * on top of Myriad, but is not required.
-     *
-     * Header format is as follows:
-     *
-     * MMMMvBssssSSSS*
-     * M = Magic number (0AD3FA84)
-     * v = version number
-     * B = Bits per int
-     * s = Blocksize in bytes
-     * S = Number of Streams
-     * 
-     * The * represents the Stream headers, one per stream, as follows:
-     * IIIIssss$
-     * I = Id number of the stream
-     * s = size of stream in bytes
-     *
-     * The $ represents the Block headers, one per used block, as  follows:
-     * IIII
-     * I = Index of the block
-     *
-     * The stream/block data is interleaved in the header, so all blocks stored
-     * with one stream are together.  The block headers are in order, and the
-     * data in them is required to be "solid" you cannot fill partial blocks
-     * mid-way through a stream.
-     *
-     * The initial block starts with the nids header, and is both the zero block
-     * and the zero stream.  For now, the minimum block size is the size needed
-     * to store the base header, the zero stream header, and the first two
-     * blocks of the zero stream, so 30 bytes.  Since it's reccomended to use
-     * a size that will fit evenly into filesystem blocks, then a size of 32 is
-     * probably the smallest reccomended size because all powers of two equal
-     * to or greater than 32 are evenly divisible by 32.
-     *
-     * I have had a thought that if the block size were smaller than 42 bytes
-     * the header would consume the first N blocks where N * block size is
-     * enough space to house the initial header, the first stream header, and
-     * the first N block headers.  This, of course, causes you to hit an
-     * infinite header if the block size is small enough.
+     * Myriad Stream Multiplexer. This is a system that allows you to store
+     * many streams within a single backing stream. This is great for databases,
+     * caching, etc. It's fairly lightweight, and allows all streams to grow
+     * dynamically using a block-allocation scheme.  This is used extensively
+     * by the caching system and MyriadFs as well as other systems within
+     * libbu++.
      */
     class Myriad
     {
-        friend class MyriadStream;
+    public:
+        typedef int32_t StreamId;
+        typedef Bu::Array<StreamId> StreamIdArray;
+        typedef Bu::List<StreamId> StreamIdList;
+        enum Mode : int32_t {
+            None        = 0x00,
+
+            // Flags
+            Read        = 0x01, ///< Open file for reading
+            Write       = 0x02, ///< Open file for writing
+            Create      = 0x04, ///< Create file if it doesn't exist
+            Truncate    = 0x08, ///< Truncate file if it does exist
+            Append      = 0x10, ///< Start writing at end of file
+            //NonBlock    = 0x20, ///< Open file in non-blocking mode
+            Exclusive   = 0x40, ///< Create file, if it exists then fail
+
+            // Helpful mixes
+            ReadWrite   = 0x03, ///< Open for reading and writing
+            WriteNew    = 0x0E  ///< Create a file (or truncate) for writing.
+                                /// Same as Write|Create|Truncate
+        };
+
     public:
         /**
-         * Create a Myriad object that uses the given stream to store data.
-         * This stream must be random access.  The block size and preallocate
-         * values passed in are values that will be used if the given stream
-         * is empty.  In that case the stream will be "formatted" for myriad
-         * with the specified block size.  If there is already a viable Myriad
-         * format present in the stream, then the blocksize and preallocate
-         * values will be ignored and the values from the stream will be used
-         * instead.  If the stream doesn't appear to be Myriad formatted an
-         * exception will be thrown.
+         * Open existing Myriad container, or initialize a new one if the
+         * backing stream is empty. If other data is already in the provided
+         * backing stream an error is thrown.
+         *
+         * Myriad format V0
+         *   0 -   3:  Myriad_MAGIC_CODE (0ad3fa84)
+         *   4 -   4:  Version Id (1)
+         *   5 -   5:  Bits per integer (32)
+         *   6 -   9:  Block size in bytes.
+         *  10 -  13:  Number of streams.
+         *  14 - ...:  Stream Data
+         *
+         * Stream Data:
+         *   0 -   3:  Stream Id
+         *   4 -   7:  Size of stream in bytes
+         *   8 - ...:  List of blocks in stream (4 bytes per block
          */
-        Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 );
+        Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1,
+                int32_t iPreallocateBlocks=-1 );
         virtual ~Myriad();
 
         /**
-         * Destroy whatever data may be in the base stream and create a new
-         * Myriad system there with the given blocksize.  Use this with care,
-         * it will destroy anything that was already in the stream, and
-         * generally, should not ever have to be used.
+         * Creates a new stream open in the specified eMode and, optionally,
+         * preallocates the specificed amount of space. The stream is zero
+         * bytes even if space is preallocated. The open stream is returned,
+         * ready for use. Use this if you don't care what the id is of the
+         * newly created stream.
          */
-        void initialize( int iBlockSize, int iPreAllocate=1 );
+        MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
 
         /**
-         * Create a new stream within the Myriad system.  The ID of the new
-         * stream is returned.
+         * Open an existing stream or create a new stream with the specified
+         * id (iStream) with the specified eMode. This respects the normal file
+         * modes, see Bu::Myriad::Mode for details.
          */
-        int createStream( int iPreAllocate=1 );
+        MyriadStream open( StreamId iStream, Mode eMode );
 
         /**
-         * Create a new stream within the Myriad system with a given id.  The
-         * id that you provide will be the new id of the stream unless it's
-         * already used, in which case an error is thrown.  This is primarilly
-         * useful when copying an old Myriad file into a new one.
+         * Allocate a new stream but do not open it, just ensure it exists and
+         * return the id of the newly allocated stream.
          */
-        int createStreamWithId( int iId, int iPreAllocate=1 );
+        StreamId allocate();
 
         /**
-         * Delete a stream that's already within the Myriad.
+         * Erase the stream specified by iStream. This only can work when the
+         * stream is not open at the moment.
          */
-        void deleteStream( int iId );
-
+        void erase( StreamId iStream );
+        void setSize( StreamId iStream, int32_t iNewSize );
+        int32_t getSize( StreamId iStream ) const;
+        bool exists( StreamId iStream ) const;
+        Bu::String getLocation() const;
+        int32_t getBlockSize() const;
+        int32_t getTotalBlocks() const;
+        int32_t getUsedBlocks() const;
+        int32_t getFreeBlocks() const;
+        int32_t getTotalStreams() const;
+        int32_t getTotalUsedBytes() const;
+        int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
+        Bu::BitString buildBlockUseMap() const;
+        StreamIdArray buildBlockMap() const;
+        
         /**
-         * Return a new Stream object assosiated with the given stream ID.
+         * Lists all stream ids that you are allowed to open. Technically there
+         * is always a zero stream, but it is used by Myriad for stream/block
+         * accounting. It works like a normal stream but you should not open
+         * it.
          */
-        MyriadStream openStream( int iId );
-
-        Bu::Array<int> getStreamIds();
-        int getStreamSize( int iId );
-        bool hasStream( int iId );
-
-        int getNumStreams();
-        int getBlockSize();
-        int getNumBlocks();
-        int getNumUsedBlocks();
-        Bu::size getTotalUsedBytes();
-        Bu::size getTotalUnusedBytes();
-        Bu::size getTotalUnusedBytes( int iFakeBlockSize );
+        StreamIdList getStreamList() const;
 
         /**
-         * Syncronize the header data, etc. with the storage stream.  It's not
-         * a bad idea to call this periodically.
+         * Flush all caches to the backing stream, write all structural and
+         * header changes.
          */
         void sync();
 
+    private:
+        bool loadMyriad();
+        void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks );
+        void writeHeader();
+        int32_t __calcHeaderSize();
+        int32_t allocateBlock();
+        int32_t __allocateBlock();
+        void releaseBlock( int32_t iBlockId, bool bBlank=true );
+        void __releaseBlock( int32_t iBlockId, bool bBlank=true );
+        void blankBlock( int32_t iBlockId );
+
+        void openStream( StreamId id );
+        void closeStream( StreamId id );
         /**
-         * Read the first few bytes from the given stream and return true/false
-         * depending on weather or not it's a Myriad stream.  This will throw
-         * an exception if the stream is empty, or is not random access.
+         * Block restricted read, it will not read past the end of the block
+         * that iStart places it in.
          */
-        static bool isMyriad( Bu::Stream &sStore, uint8_t &uVer );
-        
+        int32_t blockRead( int32_t iBlock, int32_t iStart,
+                void *pTarget, int32_t iSize );
+
         /**
-         * Read the first few bytes from the given stream and return true/false
-         * depending on weather or not it's a Myriad stream.  This will throw
-         * an exception if the stream is empty, or is not random access.
+         * Block restricted write, it will not write past the end of the block
+         * that iStart places it in. If this returns a non-zero number it's an
+         * indication that you need to allocate a new block.
          */
-        static bool isMyriad( Bu::Stream &sStore );
+        int32_t blockWrite( int32_t iBlock, int32_t iStart,
+                const void *pTarget, int32_t iSize );
 
-        const Bu::BitString getBlocksUsed() const;
-
-    private:
+    public:
         /**
-         * Initialize this object based on the data already in the assosiated
-         * stream.  This will be called automatically for you if you forget,
-         * but if you want to pre-initialize for some reason, just call this
-         * once before you actually start doing anything with your Myriad.
+         * Bridge/communication/tracking class for individual Myriad streams.
+         * Not for general use, this is used by Myriad and MyriadStream to
+         * control access.
          */
-        void initialize();
-
-        enum
-        {
-            blockUnused =   0xFFFFFFFFUL
-        };
-        
-        typedef Bu::Array<int> BlockArray;
         class Stream
         {
-        public:
-            int iId;
-            int iSize;
-            BlockArray aBlocks;
-        };
-        typedef Bu::Array<Stream *> StreamArray;
+        friend Bu::Myriad;
+        private:
+            Stream( Myriad &rParent, StreamId iStream, int32_t iSize );
+            virtual ~Stream();
 
-        class Block
-        {
         public:
-            char *pData;
-            bool bChanged;
-            int iBlockIndex;
+            int32_t getSize() const;
+            int32_t getBlockSize() const;
+            StreamId getStreamId() const;
+            int32_t getOpenCount() const;
+
+            void setSize( int32_t iNewSize );
+            int32_t read( int32_t iStart, void *pTarget, int32_t iSize );
+            int32_t write( int32_t iStart, const void *pTarget, int32_t iSize );
+            Bu::String getLocation() const;
+            Bu::Array<int32_t> getBlockList() const;
+
+            /**
+             * Doesn't actually open, just increments the open counter.
+             * If the open counter is non-zero then at least one stream has
+             * a lock on this stream.
+             */
+            void open();
+
+            /**
+             * Doesn't actually close, just decrements the open counter.
+             *@returns true if there are still handles open, false if no
+             * streams have a lock.
+             */
+            bool close();
+
+        private:
+            mutable Bu::Mutex mAccess;
+            Myriad &rParent;
+            StreamId iStream;
+            int32_t iSize;
+            Bu::Array<int32_t> aBlocks;
+            int32_t iOpenCount;
         };
 
-        void updateHeader();
-        int findEmptyBlock();
-
-        /**
-         *@todo Change this to use a binary search, it's nicer.
-         */
-        Stream *findStream( int iId );
-
-        Block *getBlock( int iBlock );
-        void releaseBlock( Block *pBlock );
-        void syncBlock( Block *pBlock );
-
-        int streamAddBlock( Stream *pStream );
-        void setStreamSize( Stream *pStream, long iSize );
-
-        void headerChanged();
-
     private:
-        Bu::Stream &sStore;
-        int iBlockSize;
-        int iBlocks;
-        int iUsed;
-        typedef Bu::List<int> IndexList;
+        typedef Bu::Hash<StreamId, Stream *> StreamHash;
+        typedef Bu::List<int32_t> IndexList;
+        mutable Bu::Mutex mAccess;
+        mutable Bu::Mutex mBacking;
+        Bu::Stream &rBacking;
+        int32_t iBlockSize;
+        int32_t iBlockCount;
+        bool bIsNewStream;
+        bool bStructureChanged;
+        mutable Bu::Mutex mhStream;
+        StreamHash hStream;
         IndexList lFreeBlocks;
-//      Bu::BitString bsBlockUsed;
-        StreamArray aStreams;
-        typedef Bu::Hash<int, Block *> BlockHash;
-        BlockHash hActiveBlocks;
-        bool bHeaderChanged;
-
-        Bu::Mutex mHeader;
-        Bu::Mutex mActiveBlocks;
+        StreamId iLastUsedIndex;
     };
+    constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b )
+    {
+        return static_cast<Myriad::Mode>(
+            static_cast<std::underlying_type<Myriad::Mode>::type>(a) &
+            static_cast<std::underlying_type<Myriad::Mode>::type>(b)
+            );
+    }
+    constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b )
+    {
+        return static_cast<Myriad::Mode>(
+            static_cast<std::underlying_type<Myriad::Mode>::type>(a) |
+            static_cast<std::underlying_type<Myriad::Mode>::type>(b)
+            );
+    }
 };
 
 #endif