9 files changed, 288 insertions, 33 deletions

diff --git a/src/experimental/debugmutex.cpp b/src/experimental/debugmutex.cpp
new file mode 100644
index 0000000..2b61ae2
--- /dev/null
+++ b/src/experimental/debugmutex.cpp
@@ -0,0 +1,78 @@
+#include "bu/debugmutex.h"
+
+#include "bu/exceptionbase.h"
+
+Bu::DebugMutex::DebugMutex()
+{
+}
+
+Bu::DebugMutex::~DebugMutex()
+{
+}
+
+int Bu::DebugMutex::lock()
+{
+    pthread_t self = pthread_self();
+    mState.lock();
+    bool bFound = false;
+    for( ThreadList::iterator i = lThreads.begin(); i; i++ )
+    {
+        if( (*i) == self )
+        {
+            bFound = true;
+            if( (*i).bLocked == true )
+            {
+                throw Bu::ExceptionBase( Bu::String("Double lock in thread: %1").arg( (*i).sName ).end().getStr() );
+
+            }
+            else
+            {
+                (*i).bLocked = true;
+            }
+            break;
+        }
+    }
+    if( bFound == false )
+    {
+        lThreads.append( ThreadInfo( true ) );
+    }
+    mState.unlock();
+    return Bu::Mutex::lock();
+}
+
+int Bu::DebugMutex::unlock()
+{
+    pthread_t self = pthread_self();
+    mState.lock();
+    bool bFound = false;
+    for( ThreadList::iterator i = lThreads.begin(); i; i++ )
+    {
+        if( (*i) == self )
+        {
+            bFound = true;
+            if( (*i).bLocked == false )
+            {
+                throw Bu::ExceptionBase( Bu::String("Unlock in thread that did not lock: %1").arg( (*i).sName ).end().getStr() );
+
+            }
+            else
+            {
+                (*i).bLocked = false;
+            }
+            break;
+        }
+    }
+    if( bFound == false )
+    {
+        ThreadInfo info( false );
+        throw Bu::ExceptionBase( Bu::String("Unlock in thread that never locked mutex: %1").arg( info.sName ).end().getStr() );
+    }
+    mState.unlock();
+    return Bu::Mutex::unlock();
+}
+
+int Bu::DebugMutex::trylock()
+{
+    return Bu::Mutex::trylock();
+}
+
diff --git a/src/experimental/debugmutex.h b/src/experimental/debugmutex.h
new file mode 100644
index 0000000..ca8ef9f
--- /dev/null
+++ b/src/experimental/debugmutex.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
+ *
+ * This file is part of the libbu++ library and is released under the
+ * terms of the license contained in the file LICENSE.
+ */
+
+#ifndef BU_DEBUG_MUTEX_H
+#define BU_DEBUG_MUTEX_H
+
+#include "bu/mutex.h"
+#include "bu/list.h"
+#include "bu/string.h"
+
+namespace Bu
+{
+    /**
+     * Simple mutex wrapper.  Currently this doesn't do anything extra for you
+     * except keep all of the functionality together in an OO sorta' way and
+     * keep you from having to worry about cleaning up your mutexes properly,
+     * or initing them.
+     *@ingroup Threading
+     */
+    class DebugMutex : public Mutex
+    {
+    public:
+        /**
+         * Create an unlocked mutex.
+         */
+        DebugMutex();
+        
+        /**
+         * Destroy a mutex.  This can only be done when a mutex is unlocked.
+         * Failure to unlock before destroying a mutex object could cause it to
+         * wait for the mutex to unlock, the odds of which are usually farily
+         * low at deconstruction time.
+         */
+        virtual ~DebugMutex();
+
+        /**
+         * Lock the mutex.  This causes all future calls to lock on this
+         * instance of mutex to block until the first thread that called mutex
+         * unlocks it.  At that point the next thread that called lock will get
+         * a chance to go to work.  Because of the nature of a mutex lock it is
+         * a very bad idea to do any kind of serious or rather time consuming
+         * computation within a locked section.  This can cause thread-deadlock
+         * and your program may hang.
+         */
+        virtual int lock();
+
+        /**
+         * Unlock the mutex.  This allows the next thread that asked for a lock
+         * to lock the mutex and continue with execution.
+         */
+        virtual int unlock();
+
+        /**
+         * Try to lock the mutex.  This is the option to go with if you cannot
+         * avoid putting lengthy operations within a locked section.  trylock
+         * will attempt to lock the mutex, if the mutex is already locked this
+         * function returns immediately with an error code.
+         */
+        virtual int trylock();
+
+    private:
+        Bu::Mutex mState;
+
+        class ThreadInfo
+        {
+        public:
+            ThreadInfo( bool bLocked=false ) :
+                idThread( pthread_self() ),
+                bLocked( bLocked )
+            {
+                char buf[64];
+                if( pthread_getname_np( idThread, buf, 64 ) == 0 )
+                    sName = buf;
+            }
+            ~ThreadInfo() {}
+
+            bool operator==( const ThreadInfo &rhs )
+            {
+                return pthread_equal( idThread, rhs.idThread );
+            }
+
+            bool operator==( const pthread_t &rhs )
+            {
+                return pthread_equal( idThread, rhs );
+            }
+
+            pthread_t idThread;
+            Bu::String sName;
+            bool bLocked;
+        };
+        typedef Bu::List<ThreadInfo> ThreadList;
+        ThreadList lThreads;
+    };
+}
+
+#endif
+
+
diff --git a/src/stable/mutex.h b/src/stable/mutex.h
index d9e8910..8034974 100644
--- a/src/stable/mutex.h
+++ b/src/stable/mutex.h
@@ -33,7 +33,7 @@ namespace Bu
          * wait for the mutex to unlock, the odds of which are usually farily
          * low at deconstruction time.
          */
-        ~Mutex();
+        virtual ~Mutex();
 
         /**
          * Lock the mutex.  This causes all future calls to lock on this
@@ -44,13 +44,13 @@ namespace Bu
          * computation within a locked section.  This can cause thread-deadlock
          * and your program may hang.
          */
-        int lock();
+        virtual int lock();
 
         /**
          * Unlock the mutex.  This allows the next thread that asked for a lock
          * to lock the mutex and continue with execution.
          */
-        int unlock();
+        virtual int unlock();
 
         /**
          * Try to lock the mutex.  This is the option to go with if you cannot
@@ -58,7 +58,7 @@ namespace Bu
          * will attempt to lock the mutex, if the mutex is already locked this
          * function returns immediately with an error code.
          */
-        int trylock();
+        virtual int trylock();
 
     protected:
         pthread_mutex_t mutex;  /**< The internal mutex reference. */
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 03cffa9..492676e 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -129,6 +129,19 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
     return Bu::MyriadStream( *this, pStream, eMode );
 }
 
+Bu::Myriad::StreamId Bu::Myriad::allocate()
+{
+    Bu::MutexLocker l( mAccess );
+
+    Stream *pStream = new Stream( *this, ++iLastUsedIndex, 0 );
+    mhStream.lock();
+    hStream.insert( pStream->iStream, pStream );
+    mhStream.unlock();
+    bStructureChanged = true;
+
+    return pStream->iStream;
+}
+
 void Bu::Myriad::erase( Bu::Myriad::StreamId iStream )
 {
     // For now, let's prevent you from erasing a stream if it's open.
@@ -252,6 +265,19 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
     return iTotal;
 }
 
+Bu::Myriad::StreamIdList Bu::Myriad::getStreamList() const
+{
+    mhStream.lock();
+    StreamIdList lIds = hStream.getKeys();
+    mhStream.unlock();
+    lIds.sort();
+    if( lIds.first() == 0 )
+    {
+        lIds.eraseFirst();
+    }
+    return lIds;
+}
+
 Bu::BitString Bu::Myriad::buildBlockUseMap() const
 {
     Bu::MutexLocker l( mAccess );
@@ -264,10 +290,10 @@ Bu::BitString Bu::Myriad::buildBlockUseMap() const
     return bsMap;
 }
 
-Bu::Array<int32_t> Bu::Myriad::buildBlockMap() const
+Bu::Myriad::StreamIdArray Bu::Myriad::buildBlockMap() const
 {
     Bu::MutexLocker l( mAccess );
-    Bu::Array<int32_t> bm( iBlockCount );
+    StreamIdArray bm( iBlockCount );
     for( int j = 0; j < iBlockCount; j++ )
     {
         bm.append( -1 );
@@ -768,9 +794,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
 
     if( iStart+iSize >= this->iSize )
     {
-        int32_t iDiff = (iStart+iSize)-this->iSize;
-        iSize -= iDiff;
-        iStart += iDiff;
+        iSize = this->iSize-iStart;
     }
 
     while( iSize > 0 )
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 6168aa2..5accd1e 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -31,10 +31,20 @@ namespace Bu
         };
     subExceptionDeclEnd();
 
+    /**
+     * Myriad Stream Multiplexer. This is a system that allows you to store
+     * many streams within a single backing stream. This is great for databases,
+     * caching, etc. It's fairly lightweight, and allows all streams to grow
+     * dynamically using a block-allocation scheme.  This is used extensively
+     * by the caching system and MyriadFs as well as other systems within
+     * libbu++.
+     */
     class Myriad
     {
     public:
         typedef int32_t StreamId;
+        typedef Bu::Array<StreamId> StreamIdArray;
+        typedef Bu::List<StreamId> StreamIdList;
         enum Mode : int32_t {
             None        = 0x00,
 
@@ -55,8 +65,9 @@ namespace Bu
 
     public:
         /**
-         * Open existing Myriad stream, or initialize a new one if it doesn't
-         * exist.
+         * Open existing Myriad container, or initialize a new one if the
+         * backing stream is empty. If other data is already in the provided
+         * backing stream an error is thrown.
          *
          * Myriad format V0
          *   0 -   3:  Myriad_MAGIC_CODE (0ad3fa84)
@@ -75,8 +86,32 @@ namespace Bu
                 int32_t iPreallocateBlocks=-1 );
         virtual ~Myriad();
 
+        /**
+         * Creates a new stream open in the specified eMode and, optionally,
+         * preallocates the specificed amount of space. The stream is zero
+         * bytes even if space is preallocated. The open stream is returned,
+         * ready for use. Use this if you don't care what the id is of the
+         * newly created stream.
+         */
         MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
+
+        /**
+         * Open an existing stream or create a new stream with the specified
+         * id (iStream) with the specified eMode. This respects the normal file
+         * modes, see Bu::Myriad::Mode for details.
+         */
         MyriadStream open( StreamId iStream, Mode eMode );
+
+        /**
+         * Allocate a new stream but do not open it, just ensure it exists and
+         * return the id of the newly allocated stream.
+         */
+        StreamId allocate();
+
+        /**
+         * Erase the stream specified by iStream. This only can work when the
+         * stream is not open at the moment.
+         */
         void erase( StreamId iStream );
         void setSize( StreamId iStream, int32_t iNewSize );
         int32_t getSize( StreamId iStream ) const;
@@ -90,7 +125,15 @@ namespace Bu
         int32_t getTotalUsedBytes() const;
         int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
         Bu::BitString buildBlockUseMap() const;
-        Bu::Array<int32_t> buildBlockMap() const;
+        StreamIdArray buildBlockMap() const;
+        
+        /**
+         * Lists all stream ids that you are allowed to open. Technically there
+         * is always a zero stream, but it is used by Myriad for stream/block
+         * accounting. It works like a normal stream but you should not open
+         * it.
+         */
+        StreamIdList getStreamList() const;
 
         /**
          * Flush all caches to the backing stream, write all structural and
diff --git a/src/tests/myriadfs.cpp b/src/tests/myriadfs.cpp
index 1266e4b..29ac3d9 100644
--- a/src/tests/myriadfs.cpp
+++ b/src/tests/myriadfs.cpp
@@ -9,7 +9,7 @@ using namespace Bu;
 int main()
 {
 //  Bu::MemBuf mb;
-    Bu::File mb("store.myr", File::Read|File::Write|File::Create );
+    Bu::File mb("store.mfs", File::Read|File::Write|File::Create );
     Bu::MyriadFs mfs( mb, 512 );
 
     sio << "Creating dirs..." << sio.nl;
diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h
index d6842a5..f71f9b5 100644
--- a/src/unstable/myriadcache.h
+++ b/src/unstable/myriadcache.h
@@ -86,8 +86,8 @@ namespace Bu
         {
             Bu::ReadWriteMutex::WriteLocker wl( rwStore );
             {
-                Bu::MyriadStream ms = mStore.create( Bu::Myriad::Create );
-                hIndex.insert( o->getKey(), ms.getId() );
+                Bu::Myriad::StreamId id = mStore.allocate();
+                hIndex.insert( o->getKey(), id );
             }
             _save( o );
 
diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp
index ab9ca74..f748a53 100644
--- a/src/unstable/myriadfs.cpp
+++ b/src/unstable/myriadfs.cpp
@@ -8,6 +8,7 @@
 #include "bu/config.h"
 #include "bu/myriadfs.h"
 #include "bu/myriadstream.h"
+#include "bu/mutexlocker.h"
 
 #include <string.h>
 #include <unistd.h>
@@ -107,6 +108,7 @@ Bu::MyriadFs::~MyriadFs()
 
 void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent;
     int32_t iNode = lookupInode( sPath, iParent );
     Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read );
@@ -116,6 +118,7 @@ void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf )
 Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode,
         uint16_t uPerms )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
     try
@@ -164,6 +167,7 @@ void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
 void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
         uint32_t uSpecial )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
 //    int32_t iNode;
     try
@@ -200,6 +204,7 @@ void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms )
 void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
         const Bu::String &sPath )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
     try
@@ -232,6 +237,7 @@ void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
 void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
         const Bu::String &sPath )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
     
@@ -267,6 +273,7 @@ void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
 
 Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
     iNode = lookupInode( sPath, iParent );
@@ -279,6 +286,7 @@ Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
 
 Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode = lookupInode( sPath, iParent );
     return readDir( iNode );
@@ -287,6 +295,7 @@ Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
 void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
         int64_t iMTime )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
 
@@ -297,6 +306,7 @@ void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
 
 void Bu::MyriadFs::unlink( const Bu::String &sPath )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
 //  int32_t iNode;
 
@@ -352,6 +362,7 @@ void Bu::MyriadFs::unlink( const Bu::String &sPath )
 
 void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
 {
+    Bu::MutexLocker lLock( mAccess );
     int32_t iParent = -1;
     int32_t iNode;
     iNode = lookupInode( sPath, iParent );
@@ -361,6 +372,7 @@ void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
 
 void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo )
 {
+    Bu::MutexLocker lLock( mAccess );
     mkHardLink( sFrom, sTo );
     unlink( sFrom );
 }
@@ -572,27 +584,17 @@ int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial )
             {
                 case typeRegFile:
                 case typeSymLink:
-                    {
-                        Bu::MyriadStream ms = mStore.create(
-                            Bu::Myriad::Create
-                            );
-                        rs.uStreamIndex = ms.getId();
-                    }
+                    rs.uStreamIndex = mStore.allocate();
                     break;
                 
                 case typeDir:
-                    {
-                        Bu::MyriadStream ms = mStore.create(
-                            Bu::Myriad::Create
-                            );
-                        rs.uStreamIndex = ms.getId();
-                    }
 //                    sio << "Creating directory node, storage: "
 //                        << rs.uStreamIndex << sio.nl;
                     {
-                        Bu::MyriadStream msDir = mStore.open(
-                            rs.uStreamIndex, Bu::Myriad::Write
+                        Bu::MyriadStream msDir = mStore.create(
+                            Bu::Myriad::Write
                             );
+                        rs.uStreamIndex = msDir.getId();
                         uint32_t uSize = 0;
                         msDir.write( &uSize, 4 );
                     }
diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h
index ff14292..e3008bc 100644
--- a/src/unstable/myriadfs.h
+++ b/src/unstable/myriadfs.h
@@ -11,7 +11,7 @@
 #include <sys/types.h>
 
 #include "bu/myriad.h"
-#include "bu/readwritemutex.h"
+#include "bu/debugmutex.h"
 
 namespace Bu
 {
@@ -108,7 +108,7 @@ namespace Bu
             Truncate    = 0x08, ///< Truncate file if it does exist
             Append      = 0x10, ///< Always append on every write
             NonBlock    = 0x20, ///< Open file in non-blocking mode
-            Exclusive   = 0x44, ///< Create file, if it exists then fail
+            Exclusive   = 0x40, ///< Create file, if it exists then fail
 
             // Helpful mixes
             ReadWrite   = 0x03, ///< Open for reading and writing
@@ -172,7 +172,13 @@ namespace Bu
         typedef Bu::Hash<int32_t, int32_t> NodeIndex;
 
     private:
+        /**
+         * Lookup inode.
+         */
         int32_t lookupInode( const Bu::String &sPath, int32_t &iParent );
+        /**
+         * Lookup inode.
+         */
         int32_t lookupInode( Bu::String::const_iterator iStart,
                 int32_t iNode, int32_t &iParent );
         void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs );
@@ -190,12 +196,12 @@ namespace Bu
         void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime );
         void destroyNode( int32_t iNode );
 
-        Bu::String filePart( const Bu::String &sPath );
+        static Bu::String filePart( const Bu::String &sPath );
 
     private:
         Bu::Stream &rStore;
         Bu::Myriad mStore;
-        Bu::ReadWriteMutex mNodeIndex;
+        Bu::DebugMutex mAccess;
         NodeIndex hNodeIndex;
         int32_t iUser;
         int32_t iGroup;