From a6d249cad214dc0baff0e80e56ffdec91d8a1cf0 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Sun, 17 Mar 2013 23:22:35 +0000 Subject: The new cache system is tested and ready, it just needs to be put in it's place. The old cache system will be going away now, so if you need it for anything, grab a version before this commit. Oh, also, happy 1,000th commit! --- src/tests/cache.cpp | 361 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 330 insertions(+), 31 deletions(-) diff --git a/src/tests/cache.cpp b/src/tests/cache.cpp index aa71c39..112d916 100644 --- a/src/tests/cache.cpp +++ b/src/tests/cache.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ // template class CacheBase; +template class CacheEntry; template class CacheObject @@ -36,6 +38,27 @@ public: virtual keytype getKey() const=0; virtual int getPersistenceScore() const { return 0; } + void lock() + { + pEntry->lock(); + } + + void unlock() + { + pEntry->unlock(); + } + + int getRefCount() const + { + return pEntry->getRefCount(); + } + + bool hasChanged() const + { + return bChanged; + } + +protected: void changed( bool bChanged=true ) { if( this->bChanged == false && bChanged == true ) @@ -45,13 +68,16 @@ public: } private: - void setCache( CacheType *pCache ) + typedef CacheEntry Entry; + void setCache( CacheType *pCache, Entry *pEntry ) { this->pCache = pCache; + this->pEntry = pEntry; } private: CacheType *pCache; + Entry *pEntry; bool bChanged; }; @@ -93,6 +119,11 @@ public: mObject.unlock(); } + Bu::Mutex &getMutex() + { + return mObject; + } + void incRef() { mEntry.lock(); @@ -263,7 +294,13 @@ public: return *this; } - + + template + CachePtr cast() + { + return pCache->cast( *this ); + } + bool operator==( const MyType &rhs ) const { return pCache == rhs.pCache && @@ -339,6 +376,7 @@ template class CacheBase { friend class CachePtrInterface; +friend class CacheObject; public: CacheBase() { @@ -346,13 +384,14 @@ public: virtual ~CacheBase() { + Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry ); + syncChanges(); } typedef CacheEntry Entry; typedef Bu::List KeyList; - - CachePtr insert( const obtype *pObject ) + CachePtr insert( obtype *pObject ) { Entry *pEnt = addEntry( pObject ); @@ -362,9 +401,9 @@ public: } template - CachePtr insert( const supertype *pObject ) + CachePtr insert( supertype *pObject ) { - const obtype *pCast = dynamic_cast( pObject ); + obtype *pCast = dynamic_cast( pObject ); if( pCast == NULL ) throw std::bad_cast(); @@ -399,9 +438,45 @@ public: return CachePtr( this, key ); } + template + CachePtr cast( CachePtr &ptr ) + { + if( ptr.pEnt ) + return CachePtr( this, ptr.pEnt, ptr.kId ); + else + return CachePtr( this, ptr.kId ); + } + + void erase( const keytype &key ) + { + Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry ); + if( hCacheEntry.has( key ) ) + { + Entry *pEnt = hCacheEntry.get( key ); + pEnt->mEntry.lock(); + if( pEnt->iRefCount > 0 ) + { + int iCount = pEnt->iRefCount; + pEnt->mEntry.unlock(); + throw Bu::ExceptionBase( Bu::String("Cache entry %1 cannot be erased, there are %2 active references.").arg( key ).arg( iCount ).end().getStr() ); + } + delete pEnt->pObject; + delete pEnt; + hCacheEntry.erase( key ); + } + _erase( key ); + } + virtual KeyList getKeys() const=0; + virtual int getSize() const=0; - virtual void sync()=0; + void sync() + { + Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry ); + Bu::println("Syncing storage layer..."); + _sync(); + syncChanges(); + } void _debug() { @@ -428,9 +503,10 @@ protected: void objectChanged( const keytype &k ) { + Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry ); + hChanged.insert( k, true ); } - protected: virtual void _create( const obtype *o )=0; virtual void _erase( const keytype &k )=0; @@ -438,14 +514,17 @@ protected: virtual obtype *_load( const keytype &k )=0; virtual void _save( const obtype *o )=0; + virtual void _sync()=0; + private: - Entry *addEntry( const obtype *pObject ) + Entry *addEntry( obtype *pObject ) { - Entry *pEnt = new Entry( const_cast(pObject) ); + Entry *pEnt = new Entry( pObject ); mCacheEntry.lockWrite(); hCacheEntry.insert( pObject->getKey(), pEnt ); mCacheEntry.unlockWrite(); _create( pObject ); + pObject->setCache( this, pEnt ); return pEnt; } @@ -463,25 +542,60 @@ private: // try to load the object from the backing store obtype *pObject = _load( k ); pEnt = new Entry( pObject ); + pObject->setCache( this, pEnt ); Bu::ReadWriteMutex::WriteLocker wl( mCacheEntry ); hCacheEntry.insert( k, pEnt ); } return pEnt; } + void syncChanges() + { + Bu::println("Syncing: %1 entries changed.").arg( hChanged.getSize() ); + if( !hChanged.isEmpty() ) + { + for( typename CacheKeySet::iterator i = hChanged.begin(); i; i++ ) + { + Entry *pEnt = hCacheEntry.get( i.getKey() ); + Bu::MutexLocker ml( pEnt->getMutex() ); + _save( pEnt->getPtr() ); + } + hChanged.clear(); + } + } + private: typedef Bu::Hash CacheEntryHash; + typedef Bu::Hash CacheKeySet; CacheEntryHash hCacheEntry; + CacheKeySet hChanged; Bu::ReadWriteMutex mCacheEntry; }; +template +void _cacheObjectSave( Bu::Stream &s, obtype *pObject ) +{ + Bu::Archive ar( s, Bu::Archive::save ); + ar << *pObject; +} + +template +obtype *_cacheObjectLoad( Bu::Stream &s ) +{ + Bu::Archive ar( s, Bu::Archive::load ); + obtype *ret = new obtype(); + ar >> *ret; + return ret; +} + template class MyriadCache : public CacheBase { public: MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) : sStore( sStore ), - mStore( sStore, iBlockSize, iPreallocate ) + mStore( sStore, iBlockSize, iPreallocate ), + bStructureChanged( false ) { try { @@ -502,13 +616,14 @@ public: if( mStore.createStreamWithId( 1 ) != 1 ) throw Bu::ExceptionBase("Error creating index stream."); - sync(); + _sync(); } } virtual ~MyriadCache() { - sync(); + Bu::println("MyriadCache destroying."); + _sync(); } using typename CacheBase::KeyList; @@ -518,47 +633,60 @@ public: Bu::ReadWriteMutex::ReadLocker rl( rwStore ); return hIndex.getKeys(); } - - virtual void sync() + + virtual int getSize() const { - Bu::ReadWriteMutex::ReadLocker wl( rwStore ); - Bu::MyriadStream ms = mStore.openStream( 1 ); - Bu::Archive ar( ms, Bu::Archive::save ); - ar << (uint8_t)0 << hIndex; - ar.close(); - ms.setSize( ms.tell() ); + Bu::ReadWriteMutex::ReadLocker rl( rwStore ); + return hIndex.getSize(); } protected: virtual void _create( const obtype *o ) { + Bu::ReadWriteMutex::WriteLocker wl( rwStore ); hIndex.insert( o->getKey(), mStore.createStream() ); _save( o ); + + bStructureChanged = true; } virtual void _erase( const keytype &k ) { + Bu::ReadWriteMutex::WriteLocker wl( rwStore ); mStore.deleteStream( hIndex.get( k ) ); hIndex.erase( k ); + + bStructureChanged = true; } virtual obtype *_load( const keytype &k ) { Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) ); - Bu::Archive ar( ms, Bu::Archive::load ); - obtype *ret = new obtype(); - ar >> *ret; - return ret; + return _cacheObjectLoad( ms ); } virtual void _save( const obtype *o ) { Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) ); - { - Bu::Archive ar( ms, Bu::Archive::save ); - ar << *o; - } + _cacheObjectSave( ms, o ); + ms.setSize( ms.tell() ); + } + + virtual void _sync() + { + Bu::println("Syncing myriad header."); + Bu::ReadWriteMutex::ReadLocker wl( rwStore ); + if( !bStructureChanged ) + return; + + Bu::println(" --> Header changed, write required."); + Bu::MyriadStream ms = mStore.openStream( 1 ); + Bu::Archive ar( ms, Bu::Archive::save ); + ar << (uint8_t)0 << hIndex; + ar.close(); ms.setSize( ms.tell() ); + + bStructureChanged = false; } private: @@ -566,6 +694,7 @@ private: Bu::Myriad mStore; Bu::Hash hIndex; mutable Bu::ReadWriteMutex rwStore; + bool bStructureChanged; }; // @@ -596,16 +725,72 @@ public: return uId; } - Bu::String getName() + Bu::String getName() const { return sName; } + void setName( const Bu::String &sNewName ) + { + sName = sNewName; + changed(); + } + + virtual Bu::String toString() const=0; + private: Bu::Uuid uId; Bu::String sName; }; +class SubSomethingA : public Something +{ +friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ); +friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ); +public: + SubSomethingA() + { + } + + SubSomethingA( const Bu::String &sName, int iNumber ) : + Something( sName ), + iNumber( iNumber ) + { + } + + virtual Bu::String toString() const + { + return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber ); + } + +private: + int iNumber; +}; + +class SubSomethingB : public Something +{ +friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ); +friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ); +public: + SubSomethingB() + { + } + + SubSomethingB( const Bu::String &sName, const Bu::String &sString ) : + Something( sName ), + sString( sString ) + { + } + + virtual Bu::String toString() const + { + return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString ); + } + +private: + Bu::String sString; +}; + Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s ) { return ar >> s.uId >> s.sName; @@ -616,9 +801,112 @@ Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s ) return ar << s.uId << s.sName; } +Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ) +{ + return ar >> (Something &)s >> s.iNumber; +} + +Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ) +{ + return ar << (Something &)s << s.iNumber; +} + +Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ) +{ + return ar >> (Something &)s >> s.sString; +} + +Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ) +{ + return ar << (Something &)s << s.sString; +} + +template<> +void _cacheObjectSave( Bu::Stream &s, const Something *pObject ) +{ + Bu::Archive ar( s, Bu::Archive::save ); + if( typeid(*pObject) == typeid(SubSomethingA) ) + { + ar << (uint8_t)1 << (SubSomethingA &)*pObject; + } + else if( typeid(*pObject) == typeid(SubSomethingB) ) + { + ar << (uint8_t)2 << (SubSomethingB &)*pObject; + } + else + { + Bu::println("Not a recognized type!"); + throw Bu::ExceptionBase("Not recognized type!"); + } +} + +template<> +Something *_cacheObjectLoad( Bu::Stream &s ) +{ + Bu::Archive ar( s, Bu::Archive::load ); + uint8_t uType; + ar >> uType; + switch( uType ) + { + case 1: + { + SubSomethingA *ret = new SubSomethingA(); + ar >> *ret; + return ret; + } + break; + + case 2: + { + SubSomethingB *ret = new SubSomethingB(); + ar >> *ret; + return ret; + } + break; + + default: + throw Bu::ExceptionBase("Flagrant error! Invalid type!"); + } +} + typedef CachePtr SomethingPtr; +typedef CachePtr SomethingAPtr; +typedef CachePtr SomethingBPtr; typedef MyriadCache SomethingCache; +int main( int argc, char *argv[] ) +{ + Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite); + SomethingCache c( fStore ); + + SomethingPtr ptr; + if( time(NULL)%2 ) + ptr = c.insert( new SubSomethingA("Hello", 55) ).cast(); + else + ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast(); + + Bu::println("Something[%1]: %2"). + arg( ptr.getKey() ). + arg( ptr->getName() ); + + SomethingCache::KeyList lKeys = c.getKeys(); + Bu::println("Count: %1").arg( lKeys.getSize() ); + int j = 0; + for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ ) + { + Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->toString() ); + } + Bu::println("Count: %1").arg( c.getSize() ); + + SomethingPtr p2 = c.get( ptr.getKey() ); + Bu::println("%1 == %2").arg( p2->getName() ).arg( ptr->getName() ); + SomethingPtr p3 = ptr.cast(); + Bu::println("%1: %2").arg( p3.getKey() ).arg( p3->getName() ); + + return 0; +} + +/* int main( int argc, char *argv[] ) { Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite); @@ -634,10 +922,15 @@ int main( int argc, char *argv[] ) } SomethingCache::KeyList lKeys = c.getKeys(); + Bu::println("Count: %1").arg( lKeys.getSize() ); + int j = 0; for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ ) { Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() ); + if( ((j++)%2) ) + c.erase( *i ); } + Bu::println("Count: %1").arg( c.getSize() ); c._debug(); @@ -673,12 +966,18 @@ int main( int argc, char *argv[] ) c._debug(); + { SomethingPtr::Locker l( p3 ); Bu::println("Name again: %1").arg( p3->getName() ); + p3->setName( p3->getName() + " - again" ); + } + + c.sync(); + c._debug(); return 0; } - +*/ -- cgit v1.2.3