The new cache system is tested and ready, it just needs to be put in it's place.

The old cache system will be going away now, so if you need it for anything,
grab a version before this commit.

Oh, also, happy 1,000th commit!

1 files changed, 330 insertions, 31 deletions

diff --git a/src/tests/cache.cpp b/src/tests/cache.cpp
index aa71c39..112d916 100644
--- a/src/tests/cache.cpp
+++ b/src/tests/cache.cpp
@@ -6,6 +6,7 @@
 #include <bu/file.h>
 #include <bu/streamstack.h>
 #include <bu/readwritemutex.h>
+#include <bu/mutexlocker.h>
 
 #include <bu/sio.h>
 #include <bu/string.h>
@@ -15,6 +16,7 @@
 //
 
 template<typename keytype, typename obtype> class CacheBase;
+template<typename keytype, typename obtype> class CacheEntry;
 
 template<typename keytype, typename obtype>
 class CacheObject
@@ -36,6 +38,27 @@ public:
     virtual keytype getKey() const=0;
     virtual int getPersistenceScore() const { return 0; }
 
+    void lock()
+    {
+        pEntry->lock();
+    }
+
+    void unlock()
+    {
+        pEntry->unlock();
+    }
+
+    int getRefCount() const
+    {
+        return pEntry->getRefCount();
+    }
+
+    bool hasChanged() const
+    {
+        return bChanged;
+    }
+
+protected:
     void changed( bool bChanged=true )
     {
         if( this->bChanged == false && bChanged == true )
@@ -45,13 +68,16 @@ public:
     }
 
 private:
-    void setCache( CacheType *pCache )
+    typedef CacheEntry<keytype, obtype> Entry;
+    void setCache( CacheType *pCache, Entry *pEntry )
     {
         this->pCache = pCache;
+        this->pEntry = pEntry;
     }
 
 private:
     CacheType *pCache;
+    Entry *pEntry;
     bool bChanged;
 };
 
@@ -93,6 +119,11 @@ public:
         mObject.unlock();
     }
 
+    Bu::Mutex &getMutex()
+    {
+        return mObject;
+    }
+
     void incRef()
     {
         mEntry.lock();
@@ -263,7 +294,13 @@ public:
 
         return *this;
     }
-
+    
+    template<typename castto>
+    CachePtr<keytype, castto, basetype> cast()
+    {
+        return pCache->cast<obtype, castto>( *this );
+    }
+    
     bool operator==( const MyType &rhs ) const
     {
         return pCache == rhs.pCache &&
@@ -339,6 +376,7 @@ template<typename keytype, typename obtype>
 class CacheBase
 {
 friend class CachePtrInterface<keytype, obtype>;
+friend class CacheObject<keytype, obtype>;
 public:
     CacheBase()
     {
@@ -346,13 +384,14 @@ public:
 
     virtual ~CacheBase()
     {
+        Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry );
+        syncChanges();
     }
 
     typedef CacheEntry<keytype, obtype> Entry;
     typedef Bu::List<keytype> KeyList;
 
-    
-    CachePtr<keytype, obtype> insert( const obtype *pObject )
+    CachePtr<keytype, obtype> insert( obtype *pObject )
     {
         Entry *pEnt = addEntry( pObject );
 
@@ -362,9 +401,9 @@ public:
     }
 
     template<typename supertype>
-    CachePtr<keytype, supertype, obtype> insert( const supertype *pObject )
+    CachePtr<keytype, supertype, obtype> insert( supertype *pObject )
     {
-        const obtype *pCast = dynamic_cast<const obtype *>( pObject );
+        obtype *pCast = dynamic_cast<obtype *>( pObject );
         if( pCast == NULL )
             throw std::bad_cast();
 
@@ -399,9 +438,45 @@ public:
         return CachePtr<keytype, supertype, obtype>( this, key );
     }
 
+    template<typename supertype, typename castto>
+    CachePtr<keytype, castto, obtype> cast( CachePtr<keytype, supertype, obtype> &ptr )
+    {
+        if( ptr.pEnt )
+            return CachePtr<keytype, castto, obtype>( this, ptr.pEnt, ptr.kId );
+        else
+            return CachePtr<keytype, castto, obtype>( this, ptr.kId );
+    }
+
+    void erase( const keytype &key )
+    {
+        Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry );
+        if( hCacheEntry.has( key ) )
+        {
+            Entry *pEnt = hCacheEntry.get( key );
+            pEnt->mEntry.lock();
+            if( pEnt->iRefCount > 0 )
+            {
+                int iCount = pEnt->iRefCount;
+                pEnt->mEntry.unlock();
+                throw Bu::ExceptionBase( Bu::String("Cache entry %1 cannot be erased, there are %2 active references.").arg( key ).arg( iCount ).end().getStr() );
+            }
+            delete pEnt->pObject;
+            delete pEnt;
+            hCacheEntry.erase( key );
+        }
+        _erase( key );
+    }
+
     virtual KeyList getKeys() const=0;
+    virtual int getSize() const=0;
 
-    virtual void sync()=0;
+    void sync()
+    {
+        Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry );
+        Bu::println("Syncing storage layer...");
+        _sync();
+        syncChanges();
+    }
 
     void _debug()
     {
@@ -428,9 +503,10 @@ protected:
 
     void objectChanged( const keytype &k )
     {
+        Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry );
+        hChanged.insert( k, true );
     }
 
-
 protected:
     virtual void _create( const obtype *o )=0;
     virtual void _erase( const keytype &k )=0;
@@ -438,14 +514,17 @@ protected:
     virtual obtype *_load( const keytype &k )=0;
     virtual void _save( const obtype *o )=0;
 
+    virtual void _sync()=0;
+
 private:
-    Entry *addEntry( const obtype *pObject )
+    Entry *addEntry( obtype *pObject )
     {
-        Entry *pEnt = new Entry( const_cast<obtype *>(pObject) );
+        Entry *pEnt = new Entry( pObject );
         mCacheEntry.lockWrite();
         hCacheEntry.insert( pObject->getKey(), pEnt );
         mCacheEntry.unlockWrite();
         _create( pObject );
+        pObject->setCache( this, pEnt );
         
         return pEnt;
     }
@@ -463,25 +542,60 @@ private:
             // try to load the object from the backing store
             obtype *pObject = _load( k );
             pEnt = new Entry( pObject );
+            pObject->setCache( this, pEnt );
             Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry );
             hCacheEntry.insert( k, pEnt );
         }
         return pEnt;
     }
 
+    void syncChanges()
+    {
+        Bu::println("Syncing: %1 entries changed.").arg( hChanged.getSize() );
+        if( !hChanged.isEmpty() )
+        {
+            for( typename CacheKeySet::iterator i = hChanged.begin(); i; i++ )
+            {
+                Entry *pEnt = hCacheEntry.get( i.getKey() );
+                Bu::MutexLocker ml( pEnt->getMutex() );
+                _save( pEnt->getPtr() );
+            }
+            hChanged.clear();
+        }
+    }
+
 private:
     typedef Bu::Hash<keytype, Entry *> CacheEntryHash;
+    typedef Bu::Hash<keytype, bool> CacheKeySet;
     CacheEntryHash hCacheEntry;
+    CacheKeySet hChanged;
     Bu::ReadWriteMutex mCacheEntry;
 };
 
+template<typename obtype>
+void _cacheObjectSave( Bu::Stream &s, obtype *pObject )
+{
+    Bu::Archive ar( s, Bu::Archive::save );
+    ar << *pObject;
+}
+
+template<typename obtype>
+obtype *_cacheObjectLoad( Bu::Stream &s )
+{
+    Bu::Archive ar( s, Bu::Archive::load );
+    obtype *ret = new obtype();
+    ar >> *ret;
+    return ret;
+}
+
 template<typename keytype, typename obtype>
 class MyriadCache : public CacheBase<keytype, obtype>
 {
 public:
     MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) :
         sStore( sStore ),
-        mStore( sStore, iBlockSize, iPreallocate )
+        mStore( sStore, iBlockSize, iPreallocate ),
+        bStructureChanged( false )
     {
         try
         {
@@ -502,13 +616,14 @@ public:
             if( mStore.createStreamWithId( 1 ) != 1 )
                 throw Bu::ExceptionBase("Error creating index stream.");
 
-            sync();
+            _sync();
         }
     }
 
     virtual ~MyriadCache()
     {
-        sync();
+        Bu::println("MyriadCache destroying.");
+        _sync();
     }
    
     using typename CacheBase<keytype,obtype>::KeyList;
@@ -518,47 +633,60 @@ public:
         Bu::ReadWriteMutex::ReadLocker rl( rwStore );
         return hIndex.getKeys();
     }
-
-    virtual void sync()
+    
+    virtual int getSize() const
     {
-        Bu::ReadWriteMutex::ReadLocker wl( rwStore );
-        Bu::MyriadStream ms = mStore.openStream( 1 );
-        Bu::Archive ar( ms, Bu::Archive::save );
-        ar << (uint8_t)0 << hIndex;
-        ar.close();
-        ms.setSize( ms.tell() );
+        Bu::ReadWriteMutex::ReadLocker rl( rwStore );
+        return hIndex.getSize();
     }
 
 protected:
     virtual void _create( const obtype *o )
     {
+        Bu::ReadWriteMutex::WriteLocker wl( rwStore );
         hIndex.insert( o->getKey(), mStore.createStream() );
         _save( o );
+
+        bStructureChanged = true;
     }
 
     virtual void _erase( const keytype &k )
     {
+        Bu::ReadWriteMutex::WriteLocker wl( rwStore );
         mStore.deleteStream( hIndex.get( k ) );
         hIndex.erase( k );
+
+        bStructureChanged = true;
     }
 
     virtual obtype *_load( const keytype &k )
     {
         Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) );
-        Bu::Archive ar( ms, Bu::Archive::load );
-        obtype *ret = new obtype();
-        ar >> *ret;
-        return ret;
+        return _cacheObjectLoad<obtype>( ms );
     }
 
     virtual void _save( const obtype *o )
     {
         Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) );
-        {
-            Bu::Archive ar( ms, Bu::Archive::save );
-            ar << *o;
-        }
+        _cacheObjectSave( ms, o );
+        ms.setSize( ms.tell() );
+    }
+
+    virtual void _sync()
+    {
+        Bu::println("Syncing myriad header.");
+        Bu::ReadWriteMutex::ReadLocker wl( rwStore );
+        if( !bStructureChanged )
+            return;
+
+        Bu::println(" --> Header changed, write required.");
+        Bu::MyriadStream ms = mStore.openStream( 1 );
+        Bu::Archive ar( ms, Bu::Archive::save );
+        ar << (uint8_t)0 << hIndex;
+        ar.close();
         ms.setSize( ms.tell() );
+
+        bStructureChanged = false;
     }
 
 private:
@@ -566,6 +694,7 @@ private:
     Bu::Myriad mStore;
     Bu::Hash<keytype, int> hIndex;
     mutable Bu::ReadWriteMutex rwStore;
+    bool bStructureChanged;
 };
 
 //
@@ -596,16 +725,72 @@ public:
         return uId;
     }
 
-    Bu::String getName()
+    Bu::String getName() const
     {
         return sName;
     }
 
+    void setName( const Bu::String &sNewName )
+    {
+        sName = sNewName;
+        changed();
+    }
+
+    virtual Bu::String toString() const=0;
+
 private:
     Bu::Uuid uId;
     Bu::String sName;
 };
 
+class SubSomethingA : public Something
+{
+friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s );
+friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s );
+public:
+    SubSomethingA()
+    {
+    }
+
+    SubSomethingA( const Bu::String &sName, int iNumber ) :
+        Something( sName ),
+        iNumber( iNumber )
+    {
+    }
+    
+    virtual Bu::String toString() const
+    {
+        return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber );
+    }
+
+private:
+    int iNumber;
+};
+
+class SubSomethingB : public Something
+{
+friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s );
+friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s );
+public:
+    SubSomethingB()
+    {
+    }
+
+    SubSomethingB( const Bu::String &sName, const Bu::String &sString ) :
+        Something( sName ),
+        sString( sString )
+    {
+    }
+    
+    virtual Bu::String toString() const
+    {
+        return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString );
+    }
+
+private:
+    Bu::String sString;
+};
+
 Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s )
 {
     return ar >> s.uId >> s.sName;
@@ -616,7 +801,77 @@ Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s )
     return ar << s.uId << s.sName;
 }
 
+Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s )
+{
+    return ar >> (Something &)s >> s.iNumber;
+}
+
+Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s )
+{
+    return ar << (Something &)s << s.iNumber;
+}
+
+Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s )
+{
+    return ar >> (Something &)s >> s.sString;
+}
+
+Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s )
+{
+    return ar << (Something &)s << s.sString;
+}
+    
+template<>
+void _cacheObjectSave<const Something>( Bu::Stream &s, const Something *pObject )
+{
+    Bu::Archive ar( s, Bu::Archive::save );
+    if( typeid(*pObject) == typeid(SubSomethingA) )
+    {
+        ar << (uint8_t)1 << (SubSomethingA &)*pObject;
+    }
+    else if( typeid(*pObject) == typeid(SubSomethingB) )
+    {
+        ar << (uint8_t)2 << (SubSomethingB &)*pObject;
+    }
+    else
+    {
+        Bu::println("Not a recognized type!");
+        throw Bu::ExceptionBase("Not recognized type!");
+    }
+}
+
+template<>
+Something *_cacheObjectLoad<Something>( Bu::Stream &s )
+{
+    Bu::Archive ar( s, Bu::Archive::load );
+    uint8_t uType;
+    ar >> uType;
+    switch( uType )
+    {
+        case 1:
+            {
+                SubSomethingA *ret = new SubSomethingA();
+                ar >> *ret;
+                return ret;
+            }
+            break;
+
+        case 2:
+            {
+                SubSomethingB *ret = new SubSomethingB();
+                ar >> *ret;
+                return ret;
+            }
+            break;
+
+        default:
+            throw Bu::ExceptionBase("Flagrant error! Invalid type!");
+    }
+}
+
 typedef CachePtr<Bu::Uuid, Something> SomethingPtr;
+typedef CachePtr<Bu::Uuid, SubSomethingA, Something> SomethingAPtr;
+typedef CachePtr<Bu::Uuid, SubSomethingB, Something> SomethingBPtr;
 typedef MyriadCache<Bu::Uuid, Something> SomethingCache;
 
 int main( int argc, char *argv[] )
@@ -624,6 +879,39 @@ int main( int argc, char *argv[] )
     Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite);
     SomethingCache c( fStore );
 
+    SomethingPtr ptr;
+    if( time(NULL)%2 )
+        ptr = c.insert( new SubSomethingA("Hello", 55) ).cast<Something>();
+    else
+        ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast<Something>();
+
+    Bu::println("Something[%1]: %2").
+        arg( ptr.getKey() ).
+        arg( ptr->getName() );
+    
+    SomethingCache::KeyList lKeys = c.getKeys();
+    Bu::println("Count: %1").arg( lKeys.getSize() );
+    int j = 0;
+    for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ )
+    {
+        Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->toString() );
+    }
+    Bu::println("Count: %1").arg( c.getSize() );
+
+    SomethingPtr p2 = c.get( ptr.getKey() );
+    Bu::println("%1 == %2").arg( p2->getName() ).arg( ptr->getName() );
+    SomethingPtr p3 = ptr.cast<Something>();
+    Bu::println("%1: %2").arg( p3.getKey() ).arg( p3->getName() );
+
+    return 0;
+}
+
+/*
+int main( int argc, char *argv[] )
+{
+    Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite);
+    SomethingCache c( fStore );
+
     for( int j = 1; j < argc; j++ )
     {
         SomethingPtr ptr = c.insert( new Something(argv[j]) );
@@ -634,10 +922,15 @@ int main( int argc, char *argv[] )
     }
     
     SomethingCache::KeyList lKeys = c.getKeys();
+    Bu::println("Count: %1").arg( lKeys.getSize() );
+    int j = 0;
     for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ )
     {
         Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() );
+        if( ((j++)%2) )
+            c.erase( *i );
     }
+    Bu::println("Count: %1").arg( c.getSize() );
 
     c._debug();
 
@@ -673,12 +966,18 @@ int main( int argc, char *argv[] )
 
     c._debug();
 
+    {
     SomethingPtr::Locker l( p3 );
 
     Bu::println("Name again: %1").arg( p3->getName() );
 
+    p3->setName( p3->getName() + " - again" );
+    }
+
+    c.sync();
+
     c._debug();
 
     return 0;
 }
-
+*/