From fb5176bbd5355b02b7d0e65da3ef3f0105824cd0 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Sun, 17 Mar 2013 23:45:21 +0000 Subject: The new cache system has been broken out into it's individual headers, and is now ready for actual use. --- src/experimental/cache.cpp | 8 - src/experimental/cache.h | 437 ---------------------------------- src/experimental/cachecalc.cpp | 8 - src/experimental/cachecalc.h | 63 ----- src/experimental/cachestore.cpp | 9 - src/experimental/cachestore.h | 46 ---- src/experimental/cachestorefiles.cpp | 9 - src/experimental/cachestorefiles.h | 211 ---------------- src/experimental/cachestoremyriad.cpp | 9 - src/experimental/cachestoremyriad.h | 158 ------------ 10 files changed, 958 deletions(-) delete mode 100644 src/experimental/cache.cpp delete mode 100644 src/experimental/cache.h delete mode 100644 src/experimental/cachecalc.cpp delete mode 100644 src/experimental/cachecalc.h delete mode 100644 src/experimental/cachestore.cpp delete mode 100644 src/experimental/cachestore.h delete mode 100644 src/experimental/cachestorefiles.cpp delete mode 100644 src/experimental/cachestorefiles.h delete mode 100644 src/experimental/cachestoremyriad.cpp delete mode 100644 src/experimental/cachestoremyriad.h (limited to 'src/experimental') diff --git a/src/experimental/cache.cpp b/src/experimental/cache.cpp deleted file mode 100644 index 840682a..0000000 --- a/src/experimental/cache.cpp +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/cache.h" diff --git a/src/experimental/cache.h b/src/experimental/cache.h deleted file mode 100644 index b9d1b7a..0000000 --- a/src/experimental/cache.h +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_CACHE_H -#define BU_CACHE_H - -// #include "bu/cptr.h" -#include "bu/hash.h" -#include "bu/list.h" -#include "bu/cachestore.h" -#include "bu/cachecalc.h" - -#include "bu/trace.h" - -namespace Bu -{ -// template -// keytype __cacheGetKey( obtype *&pObj ); - template - keytype __cacheGetKey( const obtype *pObj ) - { - return pObj->getKey(); - } - - template - class Cache - { - public: - /** - * Cache Pointer - Provides access to data that is held within the - * cache. This provides safe, refcounting access to data stored in - * the cache, with support for lazy loading. - */ - class Ptr - { - friend class Bu::Cache; - private: - Ptr( Cache *pCache, obtype *pData, - const keytype &kId ) : - pCache( pCache ), - pData( pData ), - kId( kId ) - { - if( pCache ) - pCache->incRef( kId ); - } - - Ptr( Cache *pCache, const keytype &kId ) : - pCache( pCache ), - pData( NULL ), - kId( kId ) - { - } - - public: - Ptr( const Ptr &rSrc ) : - pCache( rSrc.pCache ), - pData( rSrc.pData ), - kId( rSrc.kId ) - { - if( pCache && pData ) - pCache->incRef( kId ); - } - - Ptr() : - pCache( 0 ), - pData( 0 ) - { - } - - virtual ~Ptr() - { - if( pCache && pData ) - pCache->decRef( kId ); - } - - obtype &operator*() - { - checkPtr(); - return *pData; - } - - const obtype &operator*() const - { - checkPtr(); - return *pData; - } - - obtype *operator->() - { - checkPtr(); - return pData; - } - - const obtype *operator->() const - { - checkPtr(); - return pData; - } - - bool isValid() const - { - return pCache != NULL; - } - - bool isBound() const - { - return pData != NULL; - } - - bool isSet() const - { - return pCache != NULL; - } - - const keytype &getKey() const - { - return kId; - } - - void unbind() - { - if( pCache && pData ) - pCache->decRef( kId ); - pData = NULL; - } - - void clear() - { - unbind(); - pCache = NULL; - } - - void unset() - { - clear(); - } - - Ptr &operator=( const Ptr &rRhs ) - { - if( pCache && pData ) - pCache->decRef( kId ); - pCache = rRhs.pCache; - pData = rRhs.pData; - kId = rRhs.kId; - if( pCache && pData ) - pCache->incRef( kId ); - return *this; - } - - bool operator==( const Ptr &rRhs ) const - { - return pCache == rRhs.pCache && kId == rRhs.kId; - } - - bool operator!=( const Ptr &rRhs ) const - { - return pCache != rRhs.pCache || kId != rRhs.kId; - } - - private: - void checkPtr() const - { - if( pCache && !pData ) - { - pData = pCache->getRaw( kId ); - pCache->incRef( kId ); - } - } - - private: - Bu::Cache *pCache; - mutable obtype *pData; - mutable keytype kId; - }; - - private: - typedef Bu::CacheStore Store; - typedef Bu::List StoreList; - typedef Bu::CacheCalc Calc; - - typedef struct CacheEntry - { - obtype *pData; - int iRefs; - time_t tLastSync; - } CacheEntry; - - typedef Bu::Hash CidHash; - - public: - typedef keytype Key; - Cache( Calc *pCalc, Store *pStore ) : - pCalc( pCalc ), - pStore( pStore ) - { - TRACE(); - pCalc->setCache( this ); - } - - virtual ~Cache() - { - TRACE(); - - // Better safe than sorry, better try a sync before anything - // else happens. - sync(); - - // Cycle through and unload all objects from the system. - for( typename CidHash::iterator i = hEnt.begin(); - i != hEnt.end(); i++ ) - { - if( i.getValue().iRefs > 0 ) - { - // TODO: Throw an error in this case? iRefs != 0 for an - // object when the Cache is destroyed. - throw Bu::ExceptionBase("iRefs not zero."); - } - pStore->unload( - i.getValue().pData, - i.getKey() - ); - } - delete pCalc; - delete pStore; - } - - Ptr insert( obtype *pData ) - { - TRACE( pData ); - if( pStore->has( __cacheGetKey( pData ) ) ) - throw Bu::ExceptionBase("Key already exists in cache."); - CacheEntry e = {pData, 0, 0}; - keytype k = pStore->create( pData ); - hEnt.insert( k, e ); - - pCalc->onLoad( pData, k ); - - pStore->sync(); - - return Ptr( this, pData, k ); - } - - bool has( const keytype &cId ) - { - return hEnt.has( cId ) || pStore->has( cId ); - } - - /** - * Retrieve an object from the cache and return a pointer to it. - * The object returned may be loaded from backend storage if needed, - * or the currently live object will be returned. - *@param cId The id of the object to load. - *@returns A pointer to the object. - */ - Ptr get( const keytype &cId ) - { - TRACE( cId ); - try { - return Ptr( this, hEnt.get( cId ).pData, cId ); - } - catch( Bu::HashException &e ) { - CacheEntry e = {pStore->load( cId ), 0, time( NULL )}; - pCalc->onLoad( e.pData, cId ); - hEnt.insert( cId, e ); - return Ptr( this, e.pData, cId ); - } - } - - /** - * Retrieve a handle to an object without loading it now. This function - * will return a pointer that has not yet been "realized" but can be - * used normally. Upon initial use in any way the object will be - * loaded from the cache, either linking against the already loaded - * object or loading it fresh from the backend storage. The advantage - * of this is that you recieve a usable handle to the data, but it - * does not count as a reference yet, meaning that the data is loaded - * when you need it, not before. - */ - Ptr getLazy( const keytype &cId ) - { - TRACE( cId ); - return Ptr( this, cId ); - } - - int getRefCount( const keytype &cId ) - { - TRACE( cId ); - return hEnt.get( cId ).iRefs; - } - - void unload( const keytype &cId ) - { - TRACE( cId ); - try { - if( hEnt.get( cId ).iRefs > 0 ) - { - printf("Shouldn't unload, references still exist!\n"); - return; - } - } - catch( Bu::HashException &e ) { - // It's not here? Eh, return. - return; - } - obtype *pObj = hEnt.get( cId ).pData; - pCalc->onUnload( pObj, cId ); - hEnt.erase( cId ); - - // The unload has to happen last just in case cId is a reference - // to data that is about to be deleted from memory by the unload. - pStore->unload( pObj, cId ); - } - - void erase( const keytype &cId ) - { - TRACE( cId ); - try { - if( hEnt.get( cId ).iRefs > 0 ) - { - printf("Shouldn't erase, references still exist!\n"); - return; - } - - obtype *pObj = hEnt.get( cId ).pData; - pCalc->onDestroy( pObj, cId ); - hEnt.erase( cId ); - - pStore->destroy( pObj, cId ); - pStore->sync(); - } - catch( Bu::HashException &e ) { - pCalc->onDestroy( cId ); - - if( hEnt.has( cId ) ) - { - // The object was loaded by onDestroy - erase( cId ); - } - else - { - pStore->destroy( cId ); - pStore->sync(); - } - } - } - - typedef Bu::List KeyList; - KeyList getKeys() - { - return pStore->getKeys(); - } - - KeyList getActiveKeys() - { - return hEnt.getKeys(); - } - - int getSize() - { - return pStore->getSize(); - } - - /** - * Make sure all currently loaded but not-in-use objects are synced to - * the store. - */ - void sync() - { - TRACE(); - int iSynced = 0; - for( typename CidHash::iterator i = hEnt.begin(); - i != hEnt.end(); i++ ) - { - if( i.getValue().iRefs == 0 ) - { - if( pCalc->shouldSync( - i.getValue().pData, - i.getKey(), - i.getValue().tLastSync - ) ) - { - pStore->sync( - i.getValue().pData, - i.getKey() - ); - iSynced++; - i.getValue().tLastSync = time( NULL ); - } - } - } - if( iSynced > 0 ) - { - pStore->sync(); - } - } - - private: - void incRef( const keytype &cId ) - { - TRACE( cId ); - hEnt.get( cId ).iRefs++; - } - - void decRef( const keytype &cId ) - { - TRACE( cId ); - CacheEntry &e = hEnt.get( cId ); - e.iRefs--; - } - - obtype *getRaw( const keytype &cId ) - { - TRACE( cId ); - try { - return hEnt.get( cId ).pData; - } - catch( Bu::HashException &e ) { - CacheEntry e = {pStore->load( cId ), 0, time( NULL )}; - pCalc->onLoad( e.pData, cId ); - hEnt.insert( cId, e ); - return e.pData; - } - } - - private: - CidHash hEnt; - Calc *pCalc; - Store *pStore; - }; -}; - -#endif diff --git a/src/experimental/cachecalc.cpp b/src/experimental/cachecalc.cpp deleted file mode 100644 index 7e2940c..0000000 --- a/src/experimental/cachecalc.cpp +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/cachecalc.h" diff --git a/src/experimental/cachecalc.h b/src/experimental/cachecalc.h deleted file mode 100644 index c6cf33a..0000000 --- a/src/experimental/cachecalc.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_CACHE_CALC_H -#define BU_CACHE_CALC_H - -#include "bu/trace.h" - -#include - -namespace Bu -{ - template class Cache; - - template - class CacheCalc - { - friend class Cache; - private: - typedef Cache MyCache; - public: - CacheCalc() : - pCache( (MyCache *)0 ) - { - TRACE(); - } - - virtual ~CacheCalc() - { - TRACE(); - } - - virtual void onLoad( obtype *pSrc, const keytype &key )=0; - virtual void onUnload( obtype *pSrc, const keytype &key )=0; - virtual void onDestroy( obtype *pSrc, const keytype &key )=0; - virtual void onDestroy( const keytype &key )=0; - virtual bool shouldSync( obtype *pSrc, const keytype &key, - time_t tLastSync )=0; - virtual void onTick() { }; - - protected: - MyCache *getCache() - { - TRACE(); - return pCache; - } - - private: - void setCache( MyCache *pCache ) - { - TRACE(); - this->pCache = pCache; - } - - MyCache *pCache; - }; -}; - -#endif diff --git a/src/experimental/cachestore.cpp b/src/experimental/cachestore.cpp deleted file mode 100644 index 2082a50..0000000 --- a/src/experimental/cachestore.cpp +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/cachestore.h" - diff --git a/src/experimental/cachestore.h b/src/experimental/cachestore.h deleted file mode 100644 index 48a84ad..0000000 --- a/src/experimental/cachestore.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_CACHE_STORE_H -#define BU_CACHE_STORE_H - -#include "bu/list.h" - -namespace Bu -{ - /** - * Handles I/O for data in the cache. This also assigns ID's to the newly - * created objects that are requested through this system. - */ - template - class CacheStore - { - public: - CacheStore() - { - } - - virtual ~CacheStore() - { - } - - virtual obtype *load( const keytype &key )=0; - virtual void unload( obtype *pObj, const keytype &key )=0; - virtual keytype create( obtype *pSrc )=0; - virtual void sync()=0; - virtual void sync( obtype *pObj, const keytype &key )=0; - virtual void destroy( obtype *pObj, const keytype &key )=0; - virtual void destroy( const keytype &key )=0; - virtual bool has( const keytype &key )=0; - virtual Bu::List getKeys() { return Bu::List(); } - virtual int getSize() { return -1; } - - private: - }; -}; - -#endif diff --git a/src/experimental/cachestorefiles.cpp b/src/experimental/cachestorefiles.cpp deleted file mode 100644 index bf65660..0000000 --- a/src/experimental/cachestorefiles.cpp +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/cachestorefiles.h" - diff --git a/src/experimental/cachestorefiles.h b/src/experimental/cachestorefiles.h deleted file mode 100644 index 90b0a00..0000000 --- a/src/experimental/cachestorefiles.h +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_CACHE_STORE_FILES_H -#define BU_CACHE_STORE_FILES_H - -#include "bu/string.h" -#include "bu/file.h" -#include "bu/cachestore.h" -#include "bu/archive.h" -#include "bu/membuf.h" -#include "bu/formatter.h" -#include "bu/sio.h" - -#include -#include -#include -#include - -namespace Bu -{ - template - keytype __cacheGetKey( const obtype *pObj ); - - template - obtype *__cacheStoreFilesAlloc( const keytype &key ) - { - return new obtype(); - } - - template - void __cacheStoreFilesStore( Bu::Stream &s, obtype &rObj, - const keytype & ) - { - Bu::Archive ar( s, Bu::Archive::save ); - ar << rObj; - } - - template - obtype *__cacheStoreFilesLoad( Bu::Stream &s, const keytype &key ) - { - obtype *pObj = __cacheStoreFilesAlloc( key ); - Bu::Archive ar( s, Bu::Archive::load ); - ar >> (*pObj); - return pObj; - } - - template - class CacheStoreFiles : public CacheStore - { - public: - CacheStoreFiles( const Bu::String &sPrefix ) : - sPrefix( sPrefix ) - { - if( access( sPrefix.getStr(), W_OK|R_OK|X_OK ) ) - { -#ifdef WIN32 - mkdir( sPrefix.getStr() ); -#else - mkdir( sPrefix.getStr(), 0755 ); -#endif - } - } - - virtual ~CacheStoreFiles() - { - } - - virtual obtype *load( const keytype &key ) - { -// try -// { - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/" << key; - Bu::File fIn( mb.getString(), Bu::File::Read ); - obtype *pOb = __cacheStoreFilesLoad( fIn, key ); - return pOb; -// } -// catch( std::exception &e ) -// { -// throw Bu::HashException( e.what() ); -// } - } - - virtual void unload( obtype *pObj, const keytype & ) - { - delete pObj; - } - - virtual keytype create( obtype *pSrc ) - { - keytype key = __cacheGetKey( pSrc ); - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/" << key; - - Bu::File fTouch( mb.getString(), Bu::File::WriteNew ); - - return key; - } - - virtual void sync() - { - } - - virtual void sync( obtype *pSrc, const keytype &key ) - { - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/" << key; - - Bu::File fOut( mb.getString(), Bu::File::WriteNew ); - __cacheStoreFilesStore( fOut, *pSrc, key ); - } - - virtual void destroy( obtype *pObj, const keytype &key ) - { - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/" << key; - - unlink( mb.getString().getStr() ); - delete pObj; - } - - virtual void destroy( const keytype &key ) - { - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/" << key; - - unlink( mb.getString().getStr() ); - } - - virtual bool has( const keytype &key ) - { - Bu::MemBuf mb; - Bu::Formatter f( mb ); - f << sPrefix << "/"; - Bu::String sBase = mb.getString(); - f << key; - - if( sBase == mb.getString() ) - return false; - - return access( mb.getString().getStr(), F_OK ) == 0; - } - - virtual Bu::List getKeys() - { - DIR *dir = opendir( sPrefix.getStr() ); - struct dirent *de; - Bu::List lKeys; - - while( (de = readdir( dir ) ) ) - { - if( de->d_type != DT_REG ) - continue; - - keytype tmp; - Bu::MemBuf mb( de->d_name ); - Bu::Formatter f( mb ); - try - { - Fmt fm; - fm.tokenize( false ); - f << fm; - f >> tmp; - } - catch( Bu::ExceptionBase &e ) - { - Bu::sio << "Parse error in dir-scan: " << e.what() - << Bu::sio.nl; - } - lKeys.append( tmp ); - } - closedir( dir ); - - return lKeys; - } - - virtual int getSize() - { - DIR *dir = opendir( sPrefix.getStr() ); - struct dirent *de; - int iCount = 0; - - while( (de = readdir( dir ) ) ) - { - if( de->d_type != DT_REG ) - continue; - - iCount++; - } - closedir( dir ); - - return iCount; - } - - private: - Bu::String sPrefix; - }; - -}; - -#endif diff --git a/src/experimental/cachestoremyriad.cpp b/src/experimental/cachestoremyriad.cpp deleted file mode 100644 index f6edc7a..0000000 --- a/src/experimental/cachestoremyriad.cpp +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/cachestoremyriad.h" - diff --git a/src/experimental/cachestoremyriad.h b/src/experimental/cachestoremyriad.h deleted file mode 100644 index 798d205..0000000 --- a/src/experimental/cachestoremyriad.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (C) 2007-2013 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_CACHE_STORE_MYRIAD_H -#define BU_CACHE_STORE_MYRIAD_H - -#include "bu/string.h" -#include "bu/stream.h" -#include "bu/myriad.h" -#include "bu/cachestore.h" -#include "bu/myriadstream.h" - -#include "bu/archive.h" - -namespace Bu -{ - template - keytype __cacheGetKey( const obtype *pObj ); - - template - obtype *__cacheStoreMyriadAlloc( const keytype &key ) - { - return new obtype(); - } - - template - void __cacheStoreMyriadStore( Bu::Stream &s, obtype &rObj, - const keytype & ) - { - Bu::Archive ar( s, Bu::Archive::save ); - ar << rObj; - } - - template - obtype *__cacheStoreMyriadLoad( Bu::Stream &s, const keytype &key ) - { - obtype *pObj = __cacheStoreMyriadAlloc( key ); - Bu::Archive ar( s, Bu::Archive::load ); - ar >> (*pObj); - return pObj; - } - - template - class CacheStoreMyriad : public CacheStore - { - public: - CacheStoreMyriad( Bu::Stream &sArch, - int iBlockSize=512, int iPreAllocate=8 ) : - mStore( sArch, iBlockSize, iPreAllocate ) - { - try - { - MyriadStream ns = mStore.openStream( 1 ); - Bu::Archive ar( ns, Bu::Archive::load ); - ar >> hId; - } - catch( Bu::MyriadException &e ) - { - int iStream = mStore.createStream(); - if( iStream != 1 ) - throw Bu::ExceptionBase("That's...horrible...id = %d.\n\n", - iStream ); - MyriadStream ns = mStore.openStream( 1 ); - Bu::Archive ar( ns, Bu::Archive::save ); - ar << hId; - } - } - - virtual ~CacheStoreMyriad() - { - MyriadStream ns = mStore.openStream( 1 ); - Bu::Archive ar( ns, Bu::Archive::save ); - ar << hId; - } - - virtual obtype *load( const keytype &key ) - { - int iStream = hId.get( key ); - MyriadStream ns = mStore.openStream( iStream ); - obtype *pOb = __cacheStoreMyriadLoad( ns, key ); - return pOb; - } - - virtual void unload( obtype *pObj, const keytype & ) - { - delete pObj; - } - - virtual keytype create( obtype *pSrc ) - { - keytype key = __cacheGetKey( pSrc ); - int iStream = mStore.createStream(); - hId.insert( key, iStream ); - MyriadStream ns = mStore.openStream( iStream ); - __cacheStoreMyriadStore( ns, *pSrc, key ); - ns.setSize( ns.tell() ); - return key; - } - - virtual void sync() - { - MyriadStream ns = mStore.openStream( 1 ); - Bu::Archive ar( ns, Bu::Archive::save ); - ar << hId; - ns.setSize( ns.tell() ); - mStore.sync(); - } - - virtual void sync( obtype *pSrc, const keytype &key ) - { - int iStream = hId.get( key ); - MyriadStream ns = mStore.openStream( iStream ); - __cacheStoreMyriadStore( ns, *pSrc, key ); - ns.setSize( ns.tell() ); - } - - virtual void destroy( obtype *pObj, const keytype &key ) - { - int iStream = hId.get( key ); - mStore.deleteStream( iStream ); - hId.erase( key ); - delete pObj; - } - - virtual void destroy( const keytype &key ) - { - int iStream = hId.get( key ); - mStore.deleteStream( iStream ); - hId.erase( key ); - } - - virtual bool has( const keytype &key ) - { - return hId.has( key ); - } - - virtual Bu::List getKeys() - { - return hId.getKeys(); - } - - virtual int getSize() - { - return hId.getSize(); - } - - private: - Myriad mStore; - typedef Bu::Hash StreamHash; - StreamHash hId; - }; -}; - -#endif -- cgit v1.2.3