From f1e3f25d9b7a12cdedb99e4cb0bfa66157a1a972 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 27 Aug 2024 13:37:36 -0700 Subject: Making progress. --- src/stable/myriad.cpp | 930 +++++++++++-------------------------------- src/stable/myriad.h | 287 +++++-------- src/stable/myriadstream.cpp | 257 ++---------- src/stable/myriadstream.h | 49 +-- src/tests/bigmyriad.cpp | 8 +- src/tests/cache.cpp | 289 -------------- src/tests/cachedel.cpp | 288 -------------- src/tests/mfstest.cpp | 88 ---- src/tests/myriadfs.cpp | 62 --- src/tools/myriad.cpp | 280 ------------- src/tools/myriadfs.cpp | 269 ------------- src/unit/myriad.unit | 385 ------------------ src/unstable/myriadcache.cpp | 8 - src/unstable/myriadcache.h | 140 ------- src/unstable/myriadfs.cpp | 722 --------------------------------- src/unstable/myriadfs.h | 205 ---------- 16 files changed, 376 insertions(+), 3891 deletions(-) delete mode 100644 src/tests/cache.cpp delete mode 100644 src/tests/cachedel.cpp delete mode 100644 src/tests/mfstest.cpp delete mode 100644 src/tests/myriadfs.cpp delete mode 100644 src/tools/myriad.cpp delete mode 100644 src/tools/myriadfs.cpp delete mode 100644 src/unit/myriad.unit delete mode 100644 src/unstable/myriadcache.cpp delete mode 100644 src/unstable/myriadcache.h delete mode 100644 src/unstable/myriadfs.cpp delete mode 100644 src/unstable/myriadfs.h diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp index c606369..5278ac5 100644 --- a/src/stable/myriad.cpp +++ b/src/stable/myriad.cpp @@ -1,25 +1,18 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/config.h" #include "bu/myriad.h" -#include "bu/stream.h" #include "bu/myriadstream.h" + #include "bu/mutexlocker.h" -#include +#include "bu/util.h" #include "bu/sio.h" -using Bu::sio; -using Bu::Fmt; #define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84") -// #define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(sStore.getLocation()).arg(x) -#define TRACE( x ) (void)0 +#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \ +{ \ + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ + "Insufficent data reading myriad data from backing stream."); \ +} (void)0 namespace Bu { @@ -29,803 +22,324 @@ namespace Bu } } -Bu::Myriad::Myriad( Bu::Stream &sStore, int iBlockSize, int iPreallocate ) : - sStore( sStore ), +Bu::Myriad::Myriad( Bu::Stream &rBacking, int iBlockSize, + int iPreallocateBlocks ) : + rBacking( rBacking ), iBlockSize( iBlockSize ), - iBlocks( 0 ), - iUsed( 0 ), - bHeaderChanged( false ) + iBlockCount( 0 ), + bIsNewStream( true ) { - try + if( !rBacking.isSeekable() ) { - initialize(); + throw Bu::MyriadException( Bu::MyriadException::invalidBackingStream, + "Myriad backing stream must be random access (seekable)."); } - catch( Bu::MyriadException &e ) + if( !loadMyriad() ) { - if( e.getErrorCode() == MyriadException::emptyStream ) - { - initialize( iBlockSize, iPreallocate ); - } - else - { - throw; - } + createMyriad( iBlockSize, iPreallocateBlocks ); } } Bu::Myriad::~Myriad() { - mActiveBlocks.lock(); - TRACE("mActiveBlocks locked."); - if( !hActiveBlocks.isEmpty() ) - { - sio << "Bu::Myriad::~Myriad(): Error: There are " - << hActiveBlocks.getSize() << " unsynced blocks!" << sio.nl; - } - TRACE("mActiveBlocks unlocking..."); - mActiveBlocks.unlock(); - sync(); - - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - delete *i; - } } -void Bu::Myriad::sync() +Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode /*eMode*/, + int32_t /*iPreallocateBytes*/ ) { - updateHeader(); - - mActiveBlocks.lock(); - TRACE("mActiveBlocks locked."); - for( BlockHash::iterator i = hActiveBlocks.begin(); i; i++ ) - { - if( (*i)->bChanged ) - { - syncBlock( *i ); - } - } - TRACE("mActiveBlocks unlocked..."); - mActiveBlocks.unlock(); + return Bu::MyriadStream( *this, NULL, (Mode)0 ); } -void Bu::Myriad::initialize() +Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, + Bu::Myriad::Mode eMode ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - lFreeBlocks.clear(); - sStore.setPosEnd( 0 ); - Bu::size iSize = sStore.tell(); - sStore.setPos( 0 ); - - unsigned char buf[4]; - if( sStore.read( buf, 4 ) < 4 ) - { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::emptyStream, - "Input stream appears to be empty."); - } - if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) ) - { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::invalidFormat, - "Stream does not appear to be a valid Myriad format."); - } - sStore.read( buf, 2 ); - if( buf[0] != 1 ) - { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::badVersion, - "We can only handle version 1 for now."); - } - if( buf[1] != 32 ) - { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::invalidWordSize, - "We can only handle 32-bit words at the moment."); - } - sStore.read( &iBlockSize, 4 ); - int iStreams; - sStore.read( &iStreams, 4 ); - - iBlocks = iSize/iBlockSize; - //sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize - // << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl; - - int iHeaderSize = 14 + 8 + 4; - int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize ); - - while( iHeaderSize > iHeaderBlocks*iBlockSize ) - { - iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); - iHeaderSize = 14 + 8 + 4*iHeaderBlocks; - } - - //sio << "Myriad: iHeaderSize=" << iHeaderSize - // << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; - - Stream *pFakeHdr = new Stream; - pFakeHdr->iId = 0; - pFakeHdr->setSize( iHeaderSize ); - for( int j = 0; j < iHeaderBlocks; j++ ) - { - pFakeHdr->aBlocks.append( j ); - } - -// sio << "Blocks: " << iBlocks << " (size = " << iSize << "/" << iBlockSize -// << ")" << sio.nl; - Bu::BitString bsBlockUsed( iBlocks, false ); - bsBlockUsed.clear(); - -// bool bCanSkip = false; // Can skip around, post initial header stream i/o - MyriadStream *pIn = new MyriadStream( *this, pFakeHdr ); - pIn->setPos( sStore.tell() ); - for( int j = 0; j < iStreams; j++ ) + Bu::MutexLocker l( mAccess ); + if( !hStream.has( iStream ) ) { - int iSizeTmp; - aStreams.append( new Stream() ); - Stream &s = *aStreams[j]; - pIn->read( &s.iId, 4 ); - pIn->read( &iSizeTmp, 4 ); - s.setSize( iSizeTmp ); - int iSBlocks = blkDiv(s.getSize(), iBlockSize); - // sio << "Myriad: - Stream::iId=" << s.iId - // << ", Stream::iSize=" << s.iSize - // << ", Stream::aBlocks=" << iSBlocks - // << ", pIn->tell()=" << pIn->tell() << sio.nl; - for( int k = 0; k < iSBlocks; k++ ) - { - int iBId; - pIn->read( &iBId, 4 ); - // sio << "Myriad: - iBId=" << iBId - // << ", iStartPos=" << iBId*iBlockSize - // << ", pIn->tell()=" << pIn->tell() << sio.nl; - s.aBlocks.append( iBId ); - bsBlockUsed.setBit( iBId ); - iUsed++; - if( (j == 0 && k == iHeaderBlocks-1) ) - { - // sio << "Myriad: - End of prepartition, unlocking skipping." - // << sio.nl; -// bCanSkip = true; - MyriadStream *pTmp = new MyriadStream( *this, aStreams[0] ); - // sio << "Myriad - Position = " << pIn->tell() << sio.nl; - pTmp->setPos( pIn->tell() ); - delete pIn; - delete pFakeHdr; - pIn = pTmp; - } - } + throw Bu::MyriadException( MyriadException::noSuchStream, + "No such stream."); } - delete pIn; - - for( int j = 0; j < iBlocks; j++ ) { - if( bsBlockUsed.getBit( j ) == false ) + Bu::MutexLocker l2( mBacking ); + if( (eMode&Write) && rBacking.isWritable() ) { -// sio << "Preinitialized block " << j << " is free." << sio.nl; - lFreeBlocks.append( j ); + throw Bu::MyriadException( MyriadException::badMode, + "Backing stream does not support writing."); } } -// sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; - TRACE("mHeader unlocked..."); + return Bu::MyriadStream( *this, hStream.get( iStream ), eMode ); } -void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) +bool Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - lFreeBlocks.clear(); - - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - delete *i; - } - aStreams.clear(); - iUsed = 0; - - int iHeaderSize = 14 + 8 + 4; - int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize ); - char cBuf = 1; - int iBuf = 0; - - Stream *pStr = new Stream; - pStr->iId = 0; - - while( iHeaderSize > iHeaderBlocks*iBlockSize ) - { - iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); - iHeaderSize = 14 + 8 + 4*iHeaderBlocks; - } - - iPreAllocate += iHeaderBlocks; - - //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" - // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; - -// bsBlockUsed.setSize( iPreAllocate, true ); - iUsed++; - - char *pBlock = new char[iBlockSize]; - memset( pBlock, 0, iBlockSize ); - for( int j = 0; j < iPreAllocate; j++ ) - { - sStore.write( pBlock, iBlockSize ); - } - delete[] (char *)pBlock; - - sStore.setPos( 0 ); - - // Magic number - sStore.write( Myriad_MAGIC_CODE, 4 ); - - // Version (0) - sStore.write( &cBuf, 1 ); - - // Bits per int - cBuf = 32; - sStore.write( &cBuf, 1 ); - - // The size of each block - sStore.write( &iBlockSize, 4 ); - - iBuf = 1; - // The number of streams - sStore.write( &iBuf, 4 ); - - // Stream header - iBuf = 0; - sStore.write( &iBuf, 4 ); - sStore.write( &iHeaderSize, 4 ); - for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ ) - { - sStore.write( &iBuf, 4 ); - } - - this->iBlockSize = iBlockSize; - this->iBlocks = iPreAllocate; - - pStr->setSize( sStore.tell() ); -// sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl; - - pStr->setSize( iHeaderSize ); - for( int j = 0; j < iHeaderBlocks; j++ ) - { -// sio << "Started block " << j << " is header." << sio.nl; - pStr->aBlocks.append( j ); -// bsBlockUsed.setBit( j ); - iUsed++; - } - for( int j = iHeaderBlocks; j < this->iBlocks; j++ ) - { -// sio << "Started block " << j << " is free." << sio.nl; - lFreeBlocks.append( j ); - } - - aStreams.append( pStr ); - - //sio << bsBlockUsed.toString() << " - " << pStr->aBlocks << sio.nl; - - bHeaderChanged = true; - //hStreams.insert( 0, BlockArray( 0 ) ); - TRACE("mHeader unlocked..."); -} - -void Bu::Myriad::Stream::setSize( int iNewSize ) -{ - MutexLocker l( mStream ); - iSize = iNewSize; -} - -void Bu::Myriad::Stream::growTo( int iNewSize ) -{ - MutexLocker l( mStream ); - if( iNewSize < iSize ) - return; - iSize = iNewSize; + return false; } -int Bu::Myriad::Stream::getSize() const +bool Bu::Myriad::setSize( Bu::Myriad::StreamId /*iStream*/, + int32_t /*iNewSize*/ ) { - MutexLocker l( mStream ); - return iSize; + return false; } -void Bu::Myriad::updateHeader() +bool Bu::Myriad::loadMyriad() { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - if( bHeaderChanged == false ) + Bu::println("Load myriad!"); + char sMagicCode[4]; + rBacking.setPos( 0 ); + MyriadRead( sMagicCode, 4 ); + if( memcmp( sMagicCode, Myriad_MAGIC_CODE, 4 ) ) { - TRACE("mHeader unlocked..."); - return; + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "Backing stream does not seem to be a Myriad structure."); } - if( !sStore.canWrite() ) + uint8_t uVer; + uint8_t uBitsPerInt; + MyriadRead( &uVer, 1 ); + if( uVer != 1 ) { - TRACE("mHeader unlocked..."); - return; + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "Only version 1 myriad structures are supported."); } - - char cBuf; - int iBuf; - - //for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - //{ - // sio << "Myriad: Stream " << Fmt(4) << (*i)->iId << ": " << (*i)->aBlocks << sio.nl; - //} - - // Compute the new size of the header. - int iHeaderSize = 14 + 8*aStreams.getSize(); -// sio << "Myriad: updateHeader: aStreams.getSize() = " << aStreams.getSize() -// << sio.nl; - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) + MyriadRead( &uBitsPerInt, 1 ); + if( uBitsPerInt != 32 ) { - iHeaderSize += 4*(*i)->aBlocks.getSize(); -// sio << "Myriad: updateHeader: (*i)->aBlocks.getSize() = " -// << (*i)->aBlocks.getSize() << sio.nl; + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "Only 32 bits per int are supported at this time."); } - int iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); - while( iNewBlocks > aStreams[0]->aBlocks.getSize() ) + MyriadRead( &iBlockSize, 4 ); + int iStreamCount; + MyriadRead( &iStreamCount, 4 ); + + // + // Read stream data -- Bootstrap the zero stream + // + StreamId iStream; + MyriadRead( &iStream, 4 ); + if( iStream != 0 ) { - int iBlock = findEmptyBlock(); -// sio << "Myriad: updateHeader: Appending block " << iBlock -// << " to header." << sio.nl; - aStreams[0]->aBlocks.append( iBlock ); -// bsBlockUsed.setBit( iBlock ); - iUsed++; - iHeaderSize += 4; - iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "The first stream defined must be the header/zero stream."); } - aStreams[0]->setSize( iHeaderSize ); -// sio << "Myriad: updateHeader: iHeaderSize=" << iHeaderSize -// << ", iNewBlocks=" << iNewBlocks << ", curBlocks=" -// << aStreams[0]->aBlocks.getSize() << sio.nl; + int32_t iHeaderStreamBytes; + MyriadRead( &iHeaderStreamBytes, 4 ); - MyriadStream sHdr( *this, aStreams[0] ); - sHdr.write( Myriad_MAGIC_CODE, 4 ); + Stream *pHeaderStream = new Stream( *this, iStream, iHeaderStreamBytes ); + int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize ); - // Version (1) - cBuf = 1; - sHdr.write( &cBuf, 1 ); - - // Bits per int - cBuf = 32; - sHdr.write( &cBuf, 1 ); - - // The size of each block - sHdr.write( &iBlockSize, 4 ); - - iBuf = aStreams.getSize(); - // The number of streams - sHdr.write( &iBuf, 4 ); - - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) + while( iHeaderStreamBytes+(iHeaderStreamBlocks*4) + > iHeaderStreamBlocks*iBlockSize ) { - int iSizeTmp; - sHdr.write( &(*i)->iId, 4 ); - sHdr.write( &iSizeTmp, 4 ); - (*i)->setSize( iSizeTmp ); - int iUsedBlocks = blkDiv( iSizeTmp, iBlockSize ); -// for( BlockArray::iterator j = (*i)->aBlocks.begin(); j; j++ ) - for( int j = 0; j < iUsedBlocks; j++ ) - { - sHdr.write( &(*i)->aBlocks[j], 4 ); - } + iHeaderStreamBlocks = blkDiv( + (iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize + ); } - - bHeaderChanged = false; - TRACE("mHeader unlocked..."); -} - -int Bu::Myriad::createStream( int iPreAllocate ) -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - Stream *pStr = new Stream(); - pStr->iId = aStreams.last()->iId+1; - //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate=" - // << iPreAllocate << sio.nl; - pStr->setSize( 0 ); - aStreams.append( pStr ); - for( int j = 0; j < iPreAllocate; j++ ) + for( int32_t j = 0; j < iHeaderStreamBlocks; j++ ) { - int iFreeBlock = findEmptyBlock(); -// sio << "Myriad: Adding block " << iFreeBlock << sio.nl; - pStr->aBlocks.append( iFreeBlock ); -// bsBlockUsed.setBit( iFreeBlock ); - iUsed++; + int32_t iBlockIndex; + MyriadRead( &iBlockIndex, 4 ); + pHeaderStream->aBlocks.append( iBlockIndex ); } - bHeaderChanged = true; + // Bootstrap now using the header stream to read the rest of the data. - TRACE("mHeader unlocked..."); - return pStr->iId; + return true; } -int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate ) +void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - try + if( iBlockSize < 8 ) { - findStream( iId ); - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::streamExists, - "There is already a stream with the given id."); + throw Bu::MyriadException( Bu::MyriadException::invalidParameter, + "iBlockSize cannot be below 8"); } - catch( MyriadException &e ) + if( rBacking.getSize() ) { - Stream *pStr = new Stream(); - pStr->iId = iId; - //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate=" - // << iPreAllocate << sio.nl; - pStr->setSize( 0 ); - if( aStreams.last()->iId < iId ) - { - aStreams.append( pStr ); - } - else - { - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - if( (*i)->iId > iId ) - { - aStreams.insert( i, pStr ); - break; - } - } - } - - for( int j = 0; j < iPreAllocate; j++ ) - { - int iFreeBlock = findEmptyBlock(); - // sio << "Myriad: Adding block " << iFreeBlock << sio.nl; - pStr->aBlocks.append( iFreeBlock ); -// bsBlockUsed.setBit( iFreeBlock ); - iUsed++; - } - - bHeaderChanged = true; - - TRACE("mHeader unlocked..."); - return pStr->iId; - } - TRACE("mHeader unlocked..."); + throw Bu::MyriadException( Bu::MyriadException::invalidFormat, + "Backing stream contains data, but not a myriad structure."); } - -int Bu::Myriad::findEmptyBlock() -{ - bHeaderChanged = true; - - if( lFreeBlocks.isEmpty() ) - { - sStore.setSize( (iBlocks+1)*(Bu::size)iBlockSize ); - return iBlocks++; +/* + struct { + char sMagicCode[4]; + uint8_t uVer; + uint8_t uBitsPerInt; + uint32_t uBlockSize; + uint32_t uStreamCount; + } sHeader; + + struct { + uint32_t uStreamId; + uint32_t uStreamSize; + } sStreamHeader; + + Bu::println("sHeader = %1, sStreamHeader = %2").arg( sizeof(sHeader) ).arg( sizeof(sStreamHeader) ); +*/ + + // Start with the bytes for the file header and initial stream header + int iHeaderStreamBytes + = 14 // Base header + + 8; // Stream header + + // Pick the block count that matches our current estimate for the header + // plus one block index. + int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize ); + + Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") + .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ) + .arg( iHeaderStreamBlocks ) + .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ) + .arg( iHeaderStreamBlocks*iBlockSize ); + while( iHeaderStreamBytes+(iHeaderStreamBlocks*4) + > iHeaderStreamBlocks*iBlockSize ) + { + iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize); + if( iHeaderStreamBlocks > 100 ) + break; + Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") + .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ) + .arg( iHeaderStreamBlocks ) + .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ) + .arg( iHeaderStreamBlocks*iBlockSize ); + } + + if( iPreallocateBlocks > iHeaderStreamBlocks ) + { + rBacking.setSize( iBlockSize*iPreallocateBlocks ); } else { - return lFreeBlocks.dequeue(); + rBacking.setSize( iBlockSize*iHeaderStreamBlocks ); } -} -void Bu::Myriad::deleteStream( int iId ) -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); + // + // Write Myriad header + // + uint8_t uVer = 1; + uint8_t uBpi = 32; + int32_t iStreamCount = 1; + rBacking.setPos( 0 ); + rBacking.write( Myriad_MAGIC_CODE, 4 ); + rBacking.write( &uVer, 1 ); + rBacking.write( &uBpi, 1 ); + rBacking.write( &iBlockSize, 4 ); + rBacking.write( &iStreamCount, 4 ); - if( iId < 0 ) + Stream *pHeadStream = new Stream( *this, 0, Bu::Myriad::ReadWrite ); + // + // Write stream header + // + uint32_t uStreamId = 0; + uint32_t uStreamSize = iHeaderStreamBytes+iHeaderStreamBlocks*4; + rBacking.write( &uStreamId, 4 ); + rBacking.write( &uStreamSize, 4 ); + for( int iBlockIndex = 0; iBlockIndex < iHeaderStreamBlocks; iBlockIndex++ ) { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::invalidStreamId, - "Invalid stream id."); + rBacking.write( &iBlockIndex, 4 ); + pHeadStream->aBlocks.append( iBlockIndex ); } - if( iId == 0 ) - { - TRACE("mHeader unlocked..."); - throw MyriadException( MyriadException::protectedStream, - "You cannot delete stream zero, it is protected."); - } - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - if( (*i)->iId == iId ) - { - Stream *pStream = *i; - for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ ) - { - lFreeBlocks.append( *j ); -// bsBlockUsed.setBit( *j, false ); - iUsed--; - } - aStreams.erase( i ); - bHeaderChanged = true; - delete pStream; - TRACE("mHeader unlocked..."); - return; - } - } - TRACE("mHeader unlocked..."); -} + rBacking.flush(); -Bu::Array Bu::Myriad::getStreamIds() -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); + hStream.insert( pHeadStream->iStream, pHeadStream ); - Bu::Array aRet( aStreams.getSize() ); - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) + for( int32_t j = iHeaderStreamBlocks; j < iPreallocateBlocks; j++ ) { - aRet.append( (*i)->iId ); + lFreeBlocks.append( j ); } - - TRACE("mHeader unlocked..."); - return aRet; -} - -int Bu::Myriad::getStreamSize( int iId ) -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - TRACE("mHeader unlocked..."); - return findStream( iId )->getSize(); } -bool Bu::Myriad::hasStream( int iId ) +int32_t Bu::Myriad::allocateBlock() { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - try - { - findStream( iId ); - TRACE("mHeader unlocked..."); - return true; - }catch(...) + Bu::MutexLocker l( mAccess ); + if( lFreeBlocks.isEmpty() ) { - TRACE("mHeader unlocked..."); - return false; + // Increase the size of the backing stream + int32_t iIndex = iBlockCount++; + rBacking.setSize( iBlockCount*iBlockSize ); + return iIndex; } -} - -Bu::MyriadStream Bu::Myriad::openStream( int iId ) -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - TRACE("mHeader unlocked..."); - //sio << "Myriad: Request to open stream: " << iId << sio.nl; - return MyriadStream( *this, findStream( iId ) ); -} - -int Bu::Myriad::getNumStreams() -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - TRACE("mHeader unlocked..."); - return aStreams.getSize(); -} - -int Bu::Myriad::getBlockSize() -{ - return iBlockSize; -} - -int Bu::Myriad::getNumBlocks() -{ - return iBlocks; -} - -int Bu::Myriad::getNumUsedBlocks() -{ - return iUsed; -} - -Bu::size Bu::Myriad::getTotalUsedBytes() -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - Bu::size iTotalSize = 0; - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) + else { - iTotalSize += (*i)->getSize(); + // Provide an existing free block. + return lFreeBlocks.peekPop(); } - TRACE("mHeader unlocked..."); - return iTotalSize; } -Bu::size Bu::Myriad::getTotalUnusedBytes() +void Bu::Myriad::openStream( StreamId id ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - Bu::size iTotalSize = (iBlocks-iUsed)*iBlockSize; - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - iTotalSize += iBlockSize - ((Bu::size)(*i)->getSize()%iBlockSize); - } - TRACE("mHeader unlocked..."); - return iTotalSize; + Bu::MutexLocker l( mAccess ); + hStream.get( id )->open(); } -Bu::size Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize ) +void Bu::Myriad::closeStream( StreamId id ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - Bu::size iTotalSize = (iBlocks-iUsed)*iFakeBlockSize; - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - iTotalSize += iFakeBlockSize - ((*i)->getSize()%iFakeBlockSize); - } - TRACE("mHeader unlocked..."); - return iTotalSize; + Bu::MutexLocker l( mAccess ); + hStream.get( id )->close(); } -Bu::Myriad::Stream *Bu::Myriad::findStream( int iId ) +int32_t Bu::Myriad::blockRead( int32_t iStart, void *pTarget, int32_t iSize ) { - for( StreamArray::iterator i = aStreams.begin(); i; i++ ) - { - if( (*i)->iId == iId ) - return *i; - } + int32_t iUpperSize = iBlockSize - (iStart%iBlockSize); + Bu::println("Max size within block: %1 vs %2 (start=%3, blocksize=%4)") + .arg( iUpperSize ).arg( iSize ) + .arg( iStart ).arg( iBlockSize ); - throw MyriadException( MyriadException::noSuchStream, - "The requested stream doesn't exist and cannot be opened." ); + int32_t iAmnt = std::min( iSize, iUpperSize ); + Bu::MutexLocker l( mBacking ); + rBacking.setPos( iStart ); - return NULL; + return rBacking.read( pTarget, iAmnt ); } -Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock ) -{ -// sio << "Myriad: Reading block " << iBlock << ", bytes " -// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl; - Block *pBlock = new Block; - pBlock->pData = new char[iBlockSize]; - sStore.setPos( iBlockSize * (Bu::size)iBlock ); - sStore.read( pBlock->pData, iBlockSize ); - pBlock->bChanged = false; - pBlock->iBlockIndex = iBlock; - - mActiveBlocks.lock(); - TRACE("mHeader locked."); - hActiveBlocks.insert( iBlock, pBlock ); - TRACE("mHeader unlocked..."); - mActiveBlocks.unlock(); - - return pBlock; -} +///////// +// Bu::Myriad::Stream +// -void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock ) +Bu::Myriad::Stream::Stream( Bu::Myriad &rParent, Bu::Myriad::StreamId iStream, + int32_t iSize ) : + rParent( rParent ), + iStream( iStream ), + iSize( iSize ), + iOpenCount( 0 ), + bStructureChanged( false ) { - if( pBlock == NULL ) - return; -// sio << "Myriad: Releasing block " << pBlock->iBlockIndex << sio.nl; - syncBlock( pBlock ); - mActiveBlocks.lock(); - TRACE("mHeader locked."); - hActiveBlocks.erase( pBlock->iBlockIndex ); - TRACE("mHeader unlocked..."); - mActiveBlocks.unlock(); - - delete[] pBlock->pData; - delete pBlock; } -void Bu::Myriad::syncBlock( Block *pBlock ) +Bu::Myriad::Stream::~Stream() { - if( pBlock->bChanged ) - { -// sio << "Myriad: - Block changed, writing back to stream." << sio.nl; - sStore.setPos( iBlockSize * (Bu::size)pBlock->iBlockIndex ); - sStore.write( pBlock->pData, iBlockSize ); - pBlock->bChanged = false; - } -} - -int Bu::Myriad::streamAddBlock( Stream *pStream ) -{ - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - int iBlock = findEmptyBlock(); - pStream->aBlocks.append( iBlock ); -// bsBlockUsed.setBit( iBlock ); -// bHeaderChanged = true; - iUsed++; - TRACE("mHeader unlocked..."); - return iBlock; } -void Bu::Myriad::setStreamSize( Stream *pStream, long iSize ) +int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget, + int32_t iSize ) { - MutexLocker mLock( mHeader ); - TRACE("mHeader locked."); - - if( pStream->getSize() == iSize ) - { - TRACE("mHeader unlocked..."); - return; - } - else if( pStream->getSize() > iSize ) + int32_t iPos = iStart; + int32_t iRead = 0; + Bu::MutexLocker l( mAccess ); + while( iStart > 0 ) { - // Shrink - TRACE(Bu::String("Shrink stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->getSize()).arg(iSize).end() ); - for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; - iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize ) - { -// if( bsBlockUsed.getBit( pStream->aBlocks.last() ) ) - iUsed--; -// else -// sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl; - lFreeBlocks.enqueue( pStream->aBlocks.last() ); -// bsBlockUsed.setBit( pStream->aBlocks.last(), false ); - pStream->aBlocks.eraseLast(); - } - pStream->setSize( iSize ); - bHeaderChanged = true; + int32_t iBlock = aBlocks[iStart/rParent.iBlockSize]; + int32_t iOffset = iPos % rParent.iBlockSize; + int32_t iChunkRead = rParent.blockRead( + iBlock*rParent.iBlockSize+iOffset, pTarget, iSize + ); + if( iChunkRead == 0 ) + break; + iRead += iChunkRead; + reinterpret_cast(pTarget) += iChunkRead; + iSize -= iChunkRead; } - else - { - // Grow - TRACE(Bu::String("Grow stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->getSize()).arg(iSize).end() ); - for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; - iNewSize < iSize; iNewSize += iBlockSize ) - { - //streamAddBlock( pStream ); - int iBlock = findEmptyBlock(); - pStream->aBlocks.append( iBlock ); -// bsBlockUsed.setBit( iBlock ); -// bHeaderChanged = true; - iUsed++; - } - pStream->setSize( iSize ); - bHeaderChanged = true; - } - TRACE("mHeader unlocked..."); -} -void Bu::Myriad::headerChanged() -{ - bHeaderChanged = true; + return iRead; } -bool Bu::Myriad::isMyriad( Bu::Stream &sStore ) +void Bu::Myriad::Stream::open() { - uint8_t uTmp; - - return isMyriad( sStore, uTmp ); -} - -bool Bu::Myriad::isMyriad( Bu::Stream &sStore, uint8_t &uTmp ) -{ - sStore.setPos( 0 ); - - unsigned char buf[4]; - if( sStore.read( buf, 4 ) < 4 ) - throw MyriadException( MyriadException::emptyStream, - "Input stream appears to be empty."); - sStore.read( &uTmp, 1 ); - sStore.setPos( 0 ); - if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) ) - { - return false; - } - return true; + Bu::MutexLocker l( mAccess ); + iOpenCount++; } -const Bu::BitString Bu::Myriad::getBlocksUsed() const +bool Bu::Myriad::Stream::close() { - Bu::BitString bs( iBlocks, false ); - for( int j = 0; j < iBlocks; j++ ) - bs.setBit( j ); - for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ ) - bs.setBit( *i, false ); - return bs; + Bu::MutexLocker l( mAccess ); + return (bool)(--iOpenCount); } diff --git a/src/stable/myriad.h b/src/stable/myriad.h index e63c29f..07b4a1d 100644 --- a/src/stable/myriad.h +++ b/src/stable/myriad.h @@ -1,24 +1,14 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - #ifndef BU_MYRIAD_H #define BU_MYRIAD_H -#include -#include "bu/bitstring.h" +#include "bu/stream.h" #include "bu/exceptionbase.h" +#include "bu/mutex.h" #include "bu/array.h" #include "bu/hash.h" -#include "bu/mutex.h" -#include "bu/extratypes.h" namespace Bu { - class Stream; class MyriadStream; subExceptionDeclBegin( MyriadException ) @@ -31,212 +21,127 @@ namespace Bu noSuchStream, streamExists, invalidStreamId, - protectedStream + protectedStream, + invalidParameter, + invalidBackingStream, + badMode, }; subExceptionDeclEnd(); - /** - * Myriad block-allocated stream multiplexing system. This is a system for - * creating streams that contain other streams in a flexible and lightweight - * manner. Basically, you can create a file (or any other stream) that can - * store any number of flexible, growing streams. The streams within the - * Myriad stream are automatically numbered, not named. This works more - * or less like a filesystem, but without the extra layer for managing - * file and directory links. This would actually be very easy to add - * on top of Myriad, but is not required. - * - * Header format is as follows: - * - * MMMMvBssssSSSS* - * M = Magic number (0AD3FA84) - * v = version number - * B = Bits per int - * s = Blocksize in bytes - * S = Number of Streams - * - * The * represents the Stream headers, one per stream, as follows: - * IIIIssss$ - * I = Id number of the stream - * s = size of stream in bytes - * - * The $ represents the Block headers, one per used block, as follows: - * IIII - * I = Index of the block - * - * The stream/block data is interleaved in the header, so all blocks stored - * with one stream are together. The block headers are in order, and the - * data in them is required to be "solid" you cannot fill partial blocks - * mid-way through a stream. - * - * The initial block starts with the nids header, and is both the zero block - * and the zero stream. For now, the minimum block size is the size needed - * to store the base header, the zero stream header, and the first two - * blocks of the zero stream, so 30 bytes. Since it's reccomended to use - * a size that will fit evenly into filesystem blocks, then a size of 32 is - * probably the smallest reccomended size because all powers of two equal - * to or greater than 32 are evenly divisible by 32. - * - * I have had a thought that if the block size were smaller than 42 bytes - * the header would consume the first N blocks where N * block size is - * enough space to house the initial header, the first stream header, and - * the first N block headers. This, of course, causes you to hit an - * infinite header if the block size is small enough. - */ class Myriad { - friend class MyriadStream; public: - /** - * Create a Myriad object that uses the given stream to store data. - * This stream must be random access. The block size and preallocate - * values passed in are values that will be used if the given stream - * is empty. In that case the stream will be "formatted" for myriad - * with the specified block size. If there is already a viable Myriad - * format present in the stream, then the blocksize and preallocate - * values will be ignored and the values from the stream will be used - * instead. If the stream doesn't appear to be Myriad formatted an - * exception will be thrown. - */ - Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ); - virtual ~Myriad(); - - /** - * Destroy whatever data may be in the base stream and create a new - * Myriad system there with the given blocksize. Use this with care, - * it will destroy anything that was already in the stream, and - * generally, should not ever have to be used. - */ - void initialize( int iBlockSize, int iPreAllocate=1 ); - - /** - * Create a new stream within the Myriad system. The ID of the new - * stream is returned. - */ - int createStream( int iPreAllocate=1 ); - - /** - * Create a new stream within the Myriad system with a given id. The - * id that you provide will be the new id of the stream unless it's - * already used, in which case an error is thrown. This is primarilly - * useful when copying an old Myriad file into a new one. - */ - int createStreamWithId( int iId, int iPreAllocate=1 ); - - /** - * Delete a stream that's already within the Myriad. - */ - void deleteStream( int iId ); + typedef int32_t StreamId; + enum Mode { + None = 0x00, + + // Flags + Read = 0x01, ///< Open file for reading + Write = 0x02, ///< Open file for writing + Create = 0x04, ///< Create file if it doesn't exist + Truncate = 0x08, ///< Truncate file if it does exist + Append = 0x10, ///< Always append on every write + NonBlock = 0x20, ///< Open file in non-blocking mode + Exclusive = 0x44, ///< Create file, if it exists then fail + + // Helpful mixes + ReadWrite = 0x03, ///< Open for reading and writing + WriteNew = 0x0E ///< Create a file (or truncate) for writing. + /// Same as Write|Create|Truncate + }; + public: /** - * Return a new Stream object assosiated with the given stream ID. + * Open existing Myriad stream, or initialize a new one if it doesn't + * exist. + * + * Myriad format V0 + * 0 - 3: Myriad_MAGIC_CODE (0ad3fa84) + * 4 - 4: Version Id (1) + * 5 - 5: Bits per integer (32) + * 6 - 9: Block size in bytes. + * 10 - 13: Number of streams. + * 14 - ...: Stream Data + * + * Stream Data: + * 0 - 3: Stream Id + * 4 - 7: Size of stream in bytes + * 8 - ...: List of blocks in stream (4 bytes per block */ - MyriadStream openStream( int iId ); - - Bu::Array getStreamIds(); - int getStreamSize( int iId ); - bool hasStream( int iId ); + Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1, int32_t iPreallocateBlocks=-1 ); + virtual ~Myriad(); - int getNumStreams(); - int getBlockSize(); - int getNumBlocks(); - int getNumUsedBlocks(); - Bu::size getTotalUsedBytes(); - Bu::size getTotalUnusedBytes(); - Bu::size getTotalUnusedBytes( int iFakeBlockSize ); + MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 ); + MyriadStream open( StreamId iStream, Mode eMode ); + bool erase( StreamId iStream ); + bool setSize( StreamId iStream, int32_t iNewSize ); - /** - * Syncronize the header data, etc. with the storage stream. It's not - * a bad idea to call this periodically. - */ - void sync(); + private: + bool loadMyriad(); + void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ); + int32_t allocateBlock(); + void openStream( StreamId id ); + void closeStream( StreamId id ); /** - * Read the first few bytes from the given stream and return true/false - * depending on weather or not it's a Myriad stream. This will throw - * an exception if the stream is empty, or is not random access. - */ - static bool isMyriad( Bu::Stream &sStore, uint8_t &uVer ); - - /** - * Read the first few bytes from the given stream and return true/false - * depending on weather or not it's a Myriad stream. This will throw - * an exception if the stream is empty, or is not random access. + * Block restricted read, it will not read past the end of the block + * that iStart places it in. */ - static bool isMyriad( Bu::Stream &sStore ); + int32_t blockRead( int32_t iStart, void *pTarget, int32_t iSize ); - const Bu::BitString getBlocksUsed() const; - - private: + public: /** - * Initialize this object based on the data already in the assosiated - * stream. This will be called automatically for you if you forget, - * but if you want to pre-initialize for some reason, just call this - * once before you actually start doing anything with your Myriad. + * Bridge/communication/tracking class for individual Myriad streams. + * Not for general use, this is used by Myriad and MyriadStream to + * control access. */ - void initialize(); - - enum - { - blockUnused = 0xFFFFFFFFUL - }; - - typedef Bu::Array BlockArray; class Stream { - public: - void setSize( int iNewSize ); - void growTo( int iNewSize ); - int getSize() const; - - int iId; - BlockArray aBlocks; - + friend Bu::Myriad; private: - int iSize; - mutable Bu::Mutex mStream; - }; - typedef Bu::Array StreamArray; + Stream( Myriad &rParent, StreamId iStream, int32_t iSize ); + virtual ~Stream(); - class Block - { public: - char *pData; - bool bChanged; - int iBlockIndex; - }; - - void updateHeader(); - int findEmptyBlock(); - - /** - *@todo Change this to use a binary search, it's nicer. - */ - Stream *findStream( int iId ); + int32_t read( int32_t iStart, void *pTarget, int32_t iSize ); + + /** + * Doesn't actually open, just increments the open counter. + * If the open counter is non-zero then at least one stream has + * a lock on this stream. + */ + void open(); + + /** + * Doesn't actually close, just decrements the open counter. + *@returns true if there are still handles open, false if no + * streams have a lock. + */ + bool close(); - Block *getBlock( int iBlock ); - void releaseBlock( Block *pBlock ); - void syncBlock( Block *pBlock ); + private: + mutable Bu::Mutex mAccess; + Myriad &rParent; + StreamId iStream; + int32_t iSize; + Bu::Array aBlocks; + int32_t iOpenCount; + bool bStructureChanged; + }; - int streamAddBlock( Stream *pStream ); - void setStreamSize( Stream *pStream, long iSize ); + private: - void headerChanged(); + typedef Bu::Hash StreamHash; - private: - Bu::Stream &sStore; - int iBlockSize; - int iBlocks; - int iUsed; - typedef Bu::List IndexList; + typedef Bu::List IndexList; + Bu::Mutex mAccess; + mutable Bu::Mutex mBacking; + Bu::Stream &rBacking; + int32_t iBlockSize; + int32_t iBlockCount; + bool bIsNewStream; + StreamHash hStream; IndexList lFreeBlocks; -// Bu::BitString bsBlockUsed; - StreamArray aStreams; - typedef Bu::Hash BlockHash; - BlockHash hActiveBlocks; - bool bHeaderChanged; - - Bu::Mutex mHeader; - Bu::Mutex mActiveBlocks; }; }; diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp index 3c78bb0..cbbd4fe 100644 --- a/src/stable/myriadstream.cpp +++ b/src/stable/myriadstream.cpp @@ -1,254 +1,79 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - #include "bu/myriadstream.h" -#include - -// #define MYRIAD_STREAM_DEBUG 1 - -#ifdef MYRIAD_STREAM_DEBUG -#include "bu/sio.h" - -using Bu::sio; -using Bu::Fmt; -#endif -#include "bu/sio.h" - -// #define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(rMyriad.sStore.getLocation()).arg(x) -#define TRACE( x ) (void)0 +#include "bu/mutexlocker.h" Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad, - Bu::Myriad::Stream *pStream ) : + Bu::Myriad::Stream *pStream, Bu::Myriad::Mode eMode ) : rMyriad( rMyriad ), pStream( pStream ), - pCurBlock( NULL ), - iPos( 0 ) + eMode( eMode ) { -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: " << __LINE__ << ": Created, iId=" << pStream->iId << ", iSize=" - << pStream->iSize << sio.nl; -#endif - //pCurBlock = rMyriad.newBlock(); - //rMyriad.getBlock( uStream, pCurBlock ); - //uSize = pCurBlock->uBytesUsed; + if( (eMode&Bu::Myriad::ReadWrite) == 0 ) + { + throw Bu::MyriadException( Bu::MyriadException::invalidParameter, + "MyriadStream must be opened Read or Write or both."); + } + Bu::MutexLocker l( mAccess ); + pStream->open(); } Bu::MyriadStream::~MyriadStream() { - if( pCurBlock ) - rMyriad.releaseBlock( pCurBlock ); - //rMyriad.updateStreamSize( uStream, uSize ); - //rMyriad.deleteBlock( pCurBlock ); + close(); } void Bu::MyriadStream::close() { -} - -Bu::size Bu::MyriadStream::read( void *pBuf, Bu::size nBytes ) -{ -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: read: " << __LINE__ << ": Started, asked to read " << nBytes << "b." - << sio.nl; -#endif - if( nBytes > (Bu::size)pStream->getSize()-iPos ) - nBytes = pStream->getSize()-iPos; - if( nBytes <= 0 ) - return 0; - int iLeft = nBytes; -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: read: " << __LINE__ << ": Started, going to read " << nBytes << "b." - << sio.nl; -#endif - if( pCurBlock == NULL ) + Bu::MutexLocker l( mAccess ); + if( eMode ) { -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: read: " << __LINE__ << ": No block loaded, loading initial block." - << sio.nl; -#endif - pCurBlock = rMyriad.getBlock( - pStream->aBlocks[iPos/rMyriad.iBlockSize] - ); + pStream->close(); + eMode = Bu::Myriad::None; } - while( iLeft > 0 ) - { - int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize]; - if( pCurBlock->iBlockIndex != iCurBlock ) - { -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: read: " << __LINE__ << ": Loading new block " << iCurBlock << "." - << sio.nl; -#endif - rMyriad.releaseBlock( pCurBlock ); - pCurBlock = rMyriad.getBlock( iCurBlock ); - } +} - int iAmnt = Bu::buMin( - Bu::buMin( - rMyriad.iBlockSize - iPos%rMyriad.iBlockSize, - iLeft - ), - pStream->getSize()-iPos - ); -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: read: " << __LINE__ << ": Copying out bytes: " - << iPos << "(" << (iPos%rMyriad.iBlockSize) << ")+" - << iAmnt - << ", " << iLeft << "b left." << sio.nl; -#endif - memcpy( - pBuf, - pCurBlock->pData+(iPos%rMyriad.iBlockSize), - iAmnt - ); - iPos += iAmnt; - pBuf = &((char *)pBuf)[iAmnt]; - iLeft -= iAmnt; - } - return nBytes; +Bu::size Bu::MyriadStream::read( void *pBuf, size iBytes ) +{ } -Bu::size Bu::MyriadStream::write( const void *pBuf, Bu::size nBytes ) +Bu::String Bu::MyriadStream::readLine() { - if( nBytes <= 0 ) - return 0; +} -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write: " << __LINE__ << ": Started, asked to write " << nBytes << "b." - << sio.nl; -#endif - if( nBytes <= 0 ) - return 0; - int iLeft = nBytes; - /* - if( pCurBlock == NULL ) - { -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write: No block loaded, loading initial block." - << sio.nl; -#endif - pCurBlock = rMyriad.getBlock( - pStream->aBlocks[iPos/rMyriad.iBlockSize] - ); - }*/ - - while( iLeft > 0 ) - { - int iCurBlock; - if( iPos/rMyriad.iBlockSize < pStream->aBlocks.getSize() ) - { - iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize]; - } - else - { - iCurBlock = rMyriad.streamAddBlock( pStream ); -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write: " << __LINE__ << ": New block allocated and appended: " - << iCurBlock << "." << sio.nl; +Bu::String Bu::MyriadStream::readAll() +{ +} -#endif - } - if( !pCurBlock || pCurBlock->iBlockIndex != iCurBlock ) - { -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write: " << __LINE__ << ": Loading new block " << iCurBlock << "." - << sio.nl; -#endif - rMyriad.releaseBlock( pCurBlock ); - pCurBlock = rMyriad.getBlock( iCurBlock ); - } - pCurBlock->bChanged = true; - - // There are two main writing modes when it comes down to it. - // Overwrite mode and append mode. Append is what pretty much always - // happens when creating a new stream. - if( iPos < pStream->getSize() ) - { - int iAmnt = Bu::buMin( - Bu::buMin( - rMyriad.iBlockSize - iPos%rMyriad.iBlockSize, - iLeft - ), - pStream->getSize()-iPos - ); -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write (ovr): " << __LINE__ << ": Copying in bytes: " - << (iPos%rMyriad.iBlockSize) << "+" - << iAmnt - << ", " << iLeft << "b left." << sio.nl; -#endif - memcpy( - pCurBlock->pData+(iPos%rMyriad.iBlockSize), - pBuf, - iAmnt - ); - iPos += iAmnt; - pBuf = &((char *)pBuf)[iAmnt]; - iLeft -= iAmnt; - } - else - { - int iAmnt = Bu::buMin( - rMyriad.iBlockSize - iPos%rMyriad.iBlockSize, - iLeft - ); -#ifdef MYRIAD_STREAM_DEBUG - sio << "MyriadStream: write (app): " << __LINE__ << ": Copying in bytes: " - << (iPos%rMyriad.iBlockSize) << "+" - << iAmnt - << ", " << iLeft << "b left." << sio.nl; -#endif - memcpy( - pCurBlock->pData+(iPos%rMyriad.iBlockSize), - pBuf, - iAmnt - ); - iPos += iAmnt; - TRACE(Bu::String("Stream=%1 - pStream->iSize(%2) += iAmnt(%3)").arg(pStream->iId).arg( pStream->getSize() ).arg(iAmnt).end()); - pStream->growTo( pStream->getSize()+iAmnt ); - TRACE(Bu::String("Stream=%1 - pStream->iSize = %2").arg(pStream->iId).arg( pStream->getSize() ).end()); - rMyriad.headerChanged(); - pBuf = &((char *)pBuf)[iAmnt]; - iLeft -= iAmnt; - } - } +Bu::size Bu::MyriadStream::write( const void *pBuf, size iBytes ) +{ +} - return nBytes; +Bu::size Bu::MyriadStream::write( const Bu::String &sBuf ) +{ } Bu::size Bu::MyriadStream::tell() { - return iPos; } -void Bu::MyriadStream::seek( Bu::size offset ) +void Bu::MyriadStream::seek( size offset ) { - iPos += offset; } -void Bu::MyriadStream::setPos( Bu::size pos ) +void Bu::MyriadStream::setPos( size pos ) { - iPos = pos; } -void Bu::MyriadStream::setPosEnd( Bu::size pos ) +void Bu::MyriadStream::setPosEnd( size pos ) { - iPos = pStream->getSize()-pos; } bool Bu::MyriadStream::isEos() { - return iPos >= pStream->getSize(); } bool Bu::MyriadStream::isOpen() { - return true; } void Bu::MyriadStream::flush() @@ -257,59 +82,45 @@ void Bu::MyriadStream::flush() bool Bu::MyriadStream::canRead() { - return true; } bool Bu::MyriadStream::canWrite() { - return true; } bool Bu::MyriadStream::isReadable() { - return true; } bool Bu::MyriadStream::isWritable() { - return true; } bool Bu::MyriadStream::isSeekable() { - return true; } bool Bu::MyriadStream::isBlocking() { - return true; } -void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ ) +void Bu::MyriadStream::setBlocking( bool bBlocking ) { } -void Bu::MyriadStream::setSize( Bu::size iSize ) +void Bu::MyriadStream::setSize( size iSize ) { - if( iSize < 0 ) - iSize = 0; - rMyriad.setStreamSize( pStream, iSize ); - if( iPos > iSize ) - iPos = iSize; } Bu::size Bu::MyriadStream::getSize() const { - return pStream->getSize(); } Bu::size Bu::MyriadStream::getBlockSize() const { - return rMyriad.getBlockSize(); } -Bu::String Bu::MyriadStream::getLocation() const +Bu::String getLocation() const { - return Bu::String("%1").arg( pStream->iId ); } diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h index a94a9a2..87192a9 100644 --- a/src/stable/myriadstream.h +++ b/src/stable/myriadstream.h @@ -1,38 +1,31 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - #ifndef BU_MYRIAD_STREAM_H #define BU_MYRIAD_STREAM_H #include "bu/stream.h" +#include "bu/myriadstream.h" #include "bu/myriad.h" namespace Bu { class MyriadStream : public Bu::Stream { - friend class Myriad; + friend class Myriad; private: - /** - * These can only be created by the Myriad class. - */ - MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream ); - - public: + MyriadStream( Bu::Myriad &rMyriad, Bu::Myriad::Stream *pStream, + Bu::Myriad::Mode eMode ); virtual ~MyriadStream(); - + + public: virtual void close(); - virtual Bu::size read( void *pBuf, Bu::size nBytes ); - virtual Bu::size write( const void *pBuf, Bu::size nBytes ); - using Stream::write; - virtual Bu::size tell(); - virtual void seek( Bu::size offset ); - virtual void setPos( Bu::size pos ); - virtual void setPosEnd( Bu::size pos ); + virtual size read( void *pBuf, size iBytes ); + virtual Bu::String readLine(); + virtual Bu::String readAll(); + virtual size write( const void *pBuf, size iBytes ); + virtual size write( const Bu::String &sBuf ); + virtual size tell(); + virtual void seek( size offset ); + virtual void setPos( size pos ); + virtual void setPosEnd( size pos ); virtual bool isEos(); virtual bool isOpen(); virtual void flush(); @@ -43,18 +36,16 @@ namespace Bu virtual bool isSeekable(); virtual bool isBlocking(); virtual void setBlocking( bool bBlocking=true ); - virtual void setSize( Bu::size iSize ); - + virtual void setSize( size iSize ); virtual size getSize() const; virtual size getBlockSize() const; virtual Bu::String getLocation() const; private: - Myriad &rMyriad; - Myriad::Stream *pStream; - Myriad::Block *pCurBlock; - int iBlockSize; - int iPos; + mutable Bu::Mutex mAccess; + Bu::Myriad &rMyriad; + Bu::Myriad::Stream *pStream; + Bu::Myriad::Mode eMode; }; }; diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp index 9af301c..21c5ff2 100644 --- a/src/tests/bigmyriad.cpp +++ b/src/tests/bigmyriad.cpp @@ -5,17 +5,17 @@ int main() { Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create ); - Bu::Myriad m( f, 2048 ); + Bu::Myriad m( f, 8, 12 ); char *buf = new char[1024*1024*10]; memset( buf, 0, 1024*1024*10 ); for( int j = 0; j < 250; j++ ) { - m.openStream( m.createStream() ).write( buf, 1024*1024*10 ); +// m.openStream( m.createStream() ).write( buf, 1024*1024*10 ); // m.sync(); - printf("\r%03d%%", (j+1)*100/250 ); - fflush( stdout ); +// printf("\r%03d%%", (j+1)*100/250 ); +// fflush( stdout ); } printf("\n\n"); diff --git a/src/tests/cache.cpp b/src/tests/cache.cpp deleted file mode 100644 index 4751559..0000000 --- a/src/tests/cache.cpp +++ /dev/null @@ -1,289 +0,0 @@ -#include "bu/myriadcache.h" -#include "bu/uuid.h" -#include "bu/string.h" -#include "bu/sio.h" - -class Something : public Bu::CacheObject -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s ); -public: - Something() - { - } - - Something( const Bu::String &sName ) : - uId( Bu::Uuid::generate() ), - sName( sName ) - { - } - - virtual ~Something() - { - } - - virtual Bu::Uuid getKey() const - { - return uId; - } - - Bu::String getName() const - { - return sName; - } - - void setName( const Bu::String &sNewName ) - { - sName = sNewName; - changed(); - } - - virtual Bu::String toString() const=0; - -private: - Bu::Uuid uId; - Bu::String sName; -}; - -class SubSomethingA : public Something -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ); -public: - SubSomethingA() - { - } - - SubSomethingA( const Bu::String &sName, int iNumber ) : - Something( sName ), - iNumber( iNumber ) - { - } - - virtual Bu::String toString() const - { - return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber ); - } - -private: - int iNumber; -}; - -class SubSomethingB : public Something -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ); -public: - SubSomethingB() - { - } - - SubSomethingB( const Bu::String &sName, const Bu::String &sString ) : - Something( sName ), - sString( sString ) - { - } - - virtual Bu::String toString() const - { - return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString ); - } - -private: - Bu::String sString; -}; - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s ) -{ - return ar >> s.uId >> s.sName; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s ) -{ - return ar << s.uId << s.sName; -} - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ) -{ - return ar >> (Something &)s >> s.iNumber; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ) -{ - return ar << (Something &)s << s.iNumber; -} - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ) -{ - return ar >> (Something &)s >> s.sString; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ) -{ - return ar << (Something &)s << s.sString; -} - -namespace Bu -{ - template<> - void _cacheObjectSave( Bu::Stream &s, const Something *pObject ) - { - Bu::Archive ar( s, Bu::Archive::save ); - if( typeid(*pObject) == typeid(SubSomethingA) ) - { - ar << (uint8_t)1 << (SubSomethingA &)*pObject; - } - else if( typeid(*pObject) == typeid(SubSomethingB) ) - { - ar << (uint8_t)2 << (SubSomethingB &)*pObject; - } - else - { - Bu::println("Not a recognized type!"); - throw Bu::ExceptionBase("Not recognized type!"); - } - } - - template<> - Something *_cacheObjectLoad( Bu::CacheObject::Initializer &initObj, const Bu::Uuid &rKey, Bu::Stream &s ) - { - Bu::Archive ar( s, Bu::Archive::load ); - uint8_t uType; - ar >> uType; - switch( uType ) - { - case 1: - { - SubSomethingA *ret = initObj(new SubSomethingA()); - ar >> *ret; - return ret; - } - break; - - case 2: - { - SubSomethingB *ret = initObj(new SubSomethingB()); - ar >> *ret; - return ret; - } - break; - - default: - throw Bu::ExceptionBase("Flagrant error! Invalid type!"); - } - - return NULL; - } -} - -typedef Bu::CachePtr SomethingPtr; -typedef Bu::CachePtr SomethingAPtr; -typedef Bu::CachePtr SomethingBPtr; -typedef Bu::MyriadCache SomethingCache; - -int main( int, char *[] ) -{ - Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite); - SomethingCache c( fStore ); - - SomethingPtr ptr; - if( time(NULL)%2 ) - ptr = c.insert( new SubSomethingA("Hello", 55) ).cast(); - else - ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast(); - - Bu::println("Something[%1]: %2"). - arg( ptr.getKey() ). - arg( ptr->getName() ); - - SomethingCache::KeyList lKeys = c.getKeys(); - Bu::println("Count: %1").arg( lKeys.getSize() ); - for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ ) - { - Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->toString() ); - } - Bu::println("Count: %1").arg( c.getSize() ); - - SomethingPtr p2 = c.get( ptr.getKey() ); - Bu::println("%1 == %2").arg( p2->getName() ).arg( ptr->getName() ); - SomethingPtr p3 = ptr.cast(); - Bu::println("%1: %2").arg( p3.getKey() ).arg( p3->getName() ); - - return 0; -} - -/* -int main( int argc, char *argv[] ) -{ - Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite); - SomethingCache c( fStore ); - - for( int j = 1; j < argc; j++ ) - { - SomethingPtr ptr = c.insert( new Something(argv[j]) ); - - Bu::println("Something[%1]: %2"). - arg( ptr.getKey() ). - arg( ptr->getName() ); - } - - SomethingCache::KeyList lKeys = c.getKeys(); - Bu::println("Count: %1").arg( lKeys.getSize() ); - int j = 0; - for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ ) - { - Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() ); - if( ((j++)%2) ) - c.erase( *i ); - } - Bu::println("Count: %1").arg( c.getSize() ); - - c._debug(); - - SomethingPtr p2; - SomethingPtr p1( c.get( lKeys.first() ) ); - - c._debug(); - - { - SomethingPtr p2( p1 ); - c._debug(); - } - - c._debug(); - - p2 = p1; - - c._debug(); - - p1.unbind(); - - c._debug(); - - Bu::println("Name: %1").arg( p1->getName() ); - - p1.unbind(); - p1.lock(); - p1.unlock(); - - c._debug(); - - SomethingPtr p3 = c.getLazy( lKeys.first() ); - - c._debug(); - - { - SomethingPtr::Locker l( p3 ); - - Bu::println("Name again: %1").arg( p3->getName() ); - - p3->setName( p3->getName() + " - again" ); - } - - c.sync(); - - c._debug(); - - return 0; -} -*/ diff --git a/src/tests/cachedel.cpp b/src/tests/cachedel.cpp deleted file mode 100644 index 817757c..0000000 --- a/src/tests/cachedel.cpp +++ /dev/null @@ -1,288 +0,0 @@ -#include "bu/myriadcache.h" -#include "bu/uuid.h" -#include "bu/string.h" -#include "bu/sio.h" -#include "bu/membuf.h" - -class Something : public Bu::CacheObject -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s ); -public: - Something() - { - } - - Something( const Bu::String &sName ) : - uId( Bu::Uuid::generate() ), - sName( sName ) - { - } - - virtual ~Something() - { - } - - virtual Bu::Uuid getKey() const - { - return uId; - } - - Bu::String getName() const - { - return sName; - } - - void setName( const Bu::String &sNewName ) - { - sName = sNewName; - changed(); - } - - virtual Bu::String toString() const=0; - -private: - Bu::Uuid uId; - Bu::String sName; -}; - -class SubSomethingA : public Something -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ); -public: - SubSomethingA() - { - } - - SubSomethingA( const Bu::String &sName, int iNumber ) : - Something( sName ), - iNumber( iNumber ) - { - } - - virtual Bu::String toString() const - { - return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber ); - } - -private: - int iNumber; -}; - -class SubSomethingB : public Something -{ -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ); -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ); -public: - SubSomethingB() - { - } - - SubSomethingB( const Bu::String &sName, const Bu::String &sString ) : - Something( sName ), - sString( sString ) - { - } - - virtual Bu::String toString() const - { - return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString ); - } - -private: - Bu::String sString; -}; - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s ) -{ - return ar >> s.uId >> s.sName; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s ) -{ - return ar << s.uId << s.sName; -} - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s ) -{ - return ar >> (Something &)s >> s.iNumber; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s ) -{ - return ar << (Something &)s << s.iNumber; -} - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s ) -{ - return ar >> (Something &)s >> s.sString; -} - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s ) -{ - return ar << (Something &)s << s.sString; -} - -namespace Bu -{ - template<> - void _cacheObjectSave( Bu::Stream &s, const Something *pObject ) - { - Bu::Archive ar( s, Bu::Archive::save ); - if( typeid(*pObject) == typeid(SubSomethingA) ) - { - ar << (uint8_t)1 << (SubSomethingA &)*pObject; - } - else if( typeid(*pObject) == typeid(SubSomethingB) ) - { - ar << (uint8_t)2 << (SubSomethingB &)*pObject; - } - else - { - Bu::println("Not a recognized type!"); - throw Bu::ExceptionBase("Not recognized type!"); - } - } - - template<> - Something *_cacheObjectLoad( Bu::CacheObject::Initializer &initObj, const Bu::Uuid &rKey, Bu::Stream &s ) - { - Bu::Archive ar( s, Bu::Archive::load ); - uint8_t uType; - ar >> uType; - switch( uType ) - { - case 1: - { - SubSomethingA *ret = initObj(new SubSomethingA()); - ar >> *ret; - return ret; - } - break; - - case 2: - { - SubSomethingB *ret = initObj(new SubSomethingB()); - ar >> *ret; - return ret; - } - break; - - default: - throw Bu::ExceptionBase("Flagrant error! Invalid type!"); - } - - return NULL; - } -} - -typedef Bu::CachePtr SomethingPtr; -typedef Bu::CachePtr SomethingAPtr; -typedef Bu::CachePtr SomethingBPtr; -typedef Bu::MyriadCache SomethingCache; - -int main( int, char *[] ) -{ - Bu::MemBuf mbStore; - SomethingCache c( mbStore ); - - SomethingPtr ptr; - if( time(NULL)%2 ) - ptr = c.insert( new SubSomethingA("Hello", 55) ).cast(); - else - ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast(); - - Bu::Uuid id = ptr.getKey(); - Bu::println("Something[%1]: %2"). - arg( ptr.getKey() ). - arg( ptr->getName() ); - - Bu::println("has %1: %2").arg( id ).arg( c.has( id ) ); - c.erase( ptr.getKey() ); - Bu::println("has %1: %2").arg( id ).arg( c.has( id ) ); - SomethingPtr b = ptr; - - SomethingPtr p2 = c.insert( new SubSomethingA("new test", 123) ).cast(); - id = p2.getKey(); - p2.unbind(); - c.erase( id ); - - return 0; -} - -/* -int main( int argc, char *argv[] ) -{ - Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite); - SomethingCache c( fStore ); - - for( int j = 1; j < argc; j++ ) - { - SomethingPtr ptr = c.insert( new Something(argv[j]) ); - - Bu::println("Something[%1]: %2"). - arg( ptr.getKey() ). - arg( ptr->getName() ); - } - - SomethingCache::KeyList lKeys = c.getKeys(); - Bu::println("Count: %1").arg( lKeys.getSize() ); - int j = 0; - for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ ) - { - Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() ); - if( ((j++)%2) ) - c.erase( *i ); - } - Bu::println("Count: %1").arg( c.getSize() ); - - c._debug(); - - SomethingPtr p2; - SomethingPtr p1( c.get( lKeys.first() ) ); - - c._debug(); - - { - SomethingPtr p2( p1 ); - c._debug(); - } - - c._debug(); - - p2 = p1; - - c._debug(); - - p1.unbind(); - - c._debug(); - - Bu::println("Name: %1").arg( p1->getName() ); - - p1.unbind(); - p1.lock(); - p1.unlock(); - - c._debug(); - - SomethingPtr p3 = c.getLazy( lKeys.first() ); - - c._debug(); - - { - SomethingPtr::Locker l( p3 ); - - Bu::println("Name again: %1").arg( p3->getName() ); - - p3->setName( p3->getName() + " - again" ); - } - - c.sync(); - - c._debug(); - - return 0; -} -*/ diff --git a/src/tests/mfstest.cpp b/src/tests/mfstest.cpp deleted file mode 100644 index 813ef8e..0000000 --- a/src/tests/mfstest.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include -#include -#include - -class Obstore -{ -public: - Obstore() : - fStore("test.mfs", Bu::File::WriteNew|Bu::File::Read ), - fsOb( fStore, 1024 ) - { - } - - virtual ~Obstore() - { - } - - Bu::String getObject( const Bu::String &sPath ) - { - try - { - Bu::MyriadStream s = fsOb.open( - sPath, - Bu::MyriadFs::Read - ); - return s.readAll(); - } - catch( Bu::ExceptionBase &e ) - { - Bu::println("Exception opening file: %1").arg( e.what() ); - return Bu::String(); - } - } - - void storeObject( const Bu::String &sPath, const Bu::String &sData ) - { - for( Bu::String::const_iterator i = sPath.begin()+1; i; i++ ) - { - if( *i == '/' ) - { - Bu::String sChunk( sPath.begin(), i ); - fsOb.mkDir( sChunk, 0664 ); - } - } - try - { - Bu::MyriadStream s = fsOb.open( - sPath, - Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate - ); - s.write( sData ); - } - catch(Bu::ExceptionBase &e ) - { - Bu::println("Exception opening file: %1").arg( e.what() ); - } - } - - Bu::File fStore; - Bu::MyriadFs fsOb; -}; - -int main( int, char *[] ) -{ - Obstore os; - - os.storeObject("/killTeamState.json", "This is a json file."); - os.storeObject("/appCredentials/local", "App credentials"); - os.storeObject("/appCredentials/inckybot", "App credentials"); - os.storeObject("/appCredentials/playwith", "App credentials"); - os.storeObject("/restreamIo/users", "User data"); - os.storeObject("/youTube/users", "User data"); - os.storeObject("/topTippers.json", "{\"people\": [\"sam\", \"joe\", \"bob\"]}"); - os.storeObject("/wh40kDecks.json", ""); - - Bu::MyriadStream s = os.fsOb.open( - "/appCredentials/inckybot", - Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate - ); - s.write("Some new credentials, this is a bigger bit of data."); - - os.storeObject("/wh40kDecks.json", "{ \"op\": \"replaceDecks\", \"decks\": { \"lev\": { \"name\": \"Leviathan\", \"primary\": { \"pt\": {\"name\": \"Priority Targets\"}, \"vg\": {\"name\": \"Vital Ground\"}, \"se\": {\"name\": \"Scorched Earth\"}, \"pf\": {\"name\": \"Purge the Foe\"}, \"th\": {\"name\": \"Take and Hold\"}, \"sd\": {\"name\": \"Supply Drop\"}, \"tr\": {\"name\": \"The Ritual\"}, \"ds\": {\"name\": \"Deploy Servo-Skulls\"}, \"sp\": {\"name\": \"Sites of Power\"}, }, \"secondary\": { \"tt\": {\"name\": \"A Tempting Target\"}, \"ad\": {\"name\": \"Area Denial\"}, \"as\": {\"name\": \"Assassination\"}, \"be\": {\"name\": \"Behind Enemy Lines\"}, \"bd\": {\"name\": \"Bring It Down\"}, \"ce\": {\"name\": \"Capture Enemy Outpost\"}, \"cl\": {\"name\": \"Cleanse\"}, \"ds\": {\"name\": \"Defend Stronghold\"}, \"dt\": {\"name\": \"Deploy Teleport Homer\"}, \"ef\": {\"name\": \"Engage On All Fronts\"}, \"eb\": {\"name\": \"Extend Battle Lines\"}, \"is\": {\"name\": \"Investigate Signals\"}, \"np\": {\"name\": \"No Prisoners\"}, \"of\": {\"name\": \"Overwhelming Force\"}, \"sl\": {\"name\": \"Secure No Man’s Land\"}, \"sh\": {\"name\": \"Storm Hostile Objective\"}, }, \"gambit\": { \"dt\": {\"name\": \"Delaying Tactics\"}, \"ee\": {\"name\": \"Emergency Evacuation\"}, \"oc\": {\"name\": \"Orbital Strike Coordinates\"}, \"pp\": {\"name\": \"Proceed As Planned\"}, }, \"missionRule\": { \"cr\": {\"name\": \"Chilling Rain\"}, \"sc\": {\"name\": \"Sweep and Clear\"}, \"hs\": {\"name\": \"Hidden Supplies\"}, \"mi\": {\"name\": \"Minefields\"}, \"to\": {\"name\": \"Targets of Opportunity\"}, \"sf\": {\"name\": \"Scrambler Fields\"}, \"dr\": {\"name\": \"Delayed Reserves\"}, \"cb\": {\"name\": \"Chosen Battlefield\"}, \"mb\": {\"name\": \"Maelstrom of Battle\"}, \"sl\": {\"name\": \"Supply Lines\"}, \"si\": {\"name\": \"Secret Intel\"}, \"vs\": {\"name\": \"Vox Static\"}, }, }, } }"); - os.storeObject("/wh40kState.json", "{\"cards\": \"hello\"}"); - - return 0; -} - diff --git a/src/tests/myriadfs.cpp b/src/tests/myriadfs.cpp deleted file mode 100644 index 1266e4b..0000000 --- a/src/tests/myriadfs.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "bu/file.h" -#include "bu/membuf.h" -#include "bu/myriadfs.h" -#include "bu/myriadstream.h" -#include "bu/sio.h" - -using namespace Bu; - -int main() -{ -// Bu::MemBuf mb; - Bu::File mb("store.myr", File::Read|File::Write|File::Create ); - Bu::MyriadFs mfs( mb, 512 ); - - sio << "Creating dirs..." << sio.nl; - mfs.create("/etc", Bu::MyriadFs::typeDir|0755 ); - mfs.create("/dev", Bu::MyriadFs::typeDir|0755 ); - mfs.create("/usr", Bu::MyriadFs::typeDir|0755 ); - - mfs.create("/dev/null", Bu::MyriadFs::typeChrDev|0666, 1, 3 ); - mfs.create("/dev/zero", Bu::MyriadFs::typeChrDev|0666, 1, 5 ); - mfs.create("/dev/sda", Bu::MyriadFs::typeBlkDev|0660, 8, 0 ); - - sio << "Creating files..." << sio.nl; - { - Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read ); - ms.write("world!"); - } - { - Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read ); - ms.write("world, again!"); - } - { - Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write ); - ms.write("[longer text shouldn't be seen]"); - } - { - Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate ); - ms.write("[short text]"); - } - - sio << "Reading files..." << sio.nl; - { - Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read ); - char buf[512]; - buf[ms.read( buf, 512 )] = '\0'; - sio << "read: '" << buf << "'" << sio.nl; - } - { - Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read ); - char buf[512]; - buf[ms.read( buf, 512 )] = '\0'; - sio << "read: '" << buf << "'" << sio.nl; - } - { - Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Read ); - char buf[512]; - buf[ms.read( buf, 512 )] = '\0'; - sio << "read: '" << buf << "'" << sio.nl; - } -} - diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp deleted file mode 100644 index ccf3d3b..0000000 --- a/src/tools/myriad.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/sio.h" -#include "bu/file.h" -#include "bu/myriad.h" -#include "bu/myriadstream.h" -#include "bu/optparser.h" - -#include - -using namespace Bu; - -enum Mode -{ - modeCreate, - modeInfo, - modeStreamNew, - modeStreamDump, - modeStreamPut, - modeStreamGet, - modeBlockMap, - - modeNone -}; - -class Options : public OptParser -{ -public: - Options( int argc, char *argv[] ) : - eMode( modeNone ), - iBlockSize( 64 ), - iPreallocate( 0 ), - iStream( 0 ) - { - addHelpBanner("Mode of operation:"); - addOption( eMode, 'c', "create", - "Create a new Myriad file." ); - addOption( eMode, 'i', "info", - "Display some info about a Myriad file." ); - addOption( eMode, 'n', "new", - "Create a new sub-stream in a Myriad file."); - addOption( eMode, 'd', "dump", - "Display a hexdump of a stream from a Myriad file."); - addOption( eMode, "get", - "Get a file out of a Myriad stream (use --dst)."); - addOption( eMode, "put", - "Put a file into a Myriad stream (usr --src)."); - addOption( eMode, 'm', "block-map", - "Visualize block usage."); - addHelpOption(); - - addHelpBanner("\nGeneral options:"); - addOption( iBlockSize, 'b', "block-size", "Set the block size." ); - addOption( iPreallocate, 'p', "preallocate", - "Number of blocks to preallocate." ); - addOption( sFile, 'f', "file", "Set the Myriad filename." ); - addOption( iStream, 's', "stream", "Substream to work with."); - addOption( sSrc, "src", "Source file for copying into a Myriad file."); - addOption( sDst, "dst", - "Destination file for copying out of a Myriad file."); - - setOverride( "create", modeCreate ); - setOverride( "info", modeInfo ); - setOverride( "new", modeStreamNew ); - setOverride( "dump", modeStreamDump ); - setOverride( "put", modeStreamPut ); - setOverride( "get", modeStreamGet ); - setOverride( "block-map", modeBlockMap ); - - parse( argc, argv ); - } - - Mode eMode; - int iBlockSize; - int iPreallocate; - int iStream; - Bu::String sFile; - Bu::String sSrc; - Bu::String sDst; -}; - -Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ ) -{ - sio << "Uh oh, the formatter was called..." << sio.nl; - return f; -} - -int main( int argc, char *argv[] ) -{ - Options opts( argc, argv ); - - switch( opts.eMode ) - { - case modeCreate: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file to create." << sio.nl; - return 0; - } - else - { - File fOut( opts.sFile, File::WriteNew|File::Read ); - Myriad m( fOut, opts.iBlockSize, opts.iPreallocate ); - } - break; - - case modeInfo: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file to display info about." << sio.nl; - return 0; - } - else - { - File fIn( opts.sFile, File::Read ); - Myriad m( fIn ); - sio << "Myriad info:" << sio.nl - << " Block size: " << m.getBlockSize() << sio.nl - << " Block count: " << m.getNumBlocks() << sio.nl - << " Blocks used: " << m.getNumUsedBlocks() << " (" - << m.getNumUsedBlocks()*100/m.getNumBlocks() << "%)" - << sio.nl - << " Stream count: " << m.getNumStreams() << sio.nl - << " Used space: " << m.getTotalUsedBytes() << sio.nl - << " Unused space: " << m.getTotalUnusedBytes() << sio.nl - << " % of files: " << (double)(m.getNumBlocks()*m.getBlockSize())/(double)(m.getTotalUsedBytes() + m.getTotalUnusedBytes( 4096 ))*100.0 << sio.nl; -/* Bu::Array aStreams = m.getStreamIds(); - sio << " Stream info:" << sio.nl; - for( Bu::Array::iterator i = aStreams.begin(); i; i++ ) - { - sio << " " << Fmt(4) << *i << ") " - << m.getStreamSize( *i ) << "b" << sio.nl; - } */ - } - break; - - case modeStreamNew: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file manipulate." << sio.nl; - return 0; - } - else - { - File fOut( opts.sFile, File::Write|File::Read ); - Myriad m( fOut ); - m.createStream( opts.iPreallocate ); - } - break; - - case modeStreamDump: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file to manipulate." << sio.nl; - return 0; - } - else - { - File fOut( opts.sFile, File::Read ); - Myriad m( fOut ); - MyriadStream s = m.openStream( opts.iStream ); - sio << "Stream " << opts.iStream << ":" << sio.nl; - char buf[8]; - int iPos = 0; - while( !s.isEos() ) - { - size_t sAmnt = s.read( buf, 8 ); - sio << Fmt(5) << iPos << ": "; - iPos += sAmnt; - for( size_t j = 0; j < sAmnt; j++ ) - { - sio << Fmt::hex(2) << (int)((unsigned char)buf[j]) - << " "; - } - for( size_t j = sAmnt; j < 8; j++ ) - { - sio << "-- "; - } - sio << "| "; - for( size_t j = 0; j < sAmnt; j++ ) - { - if( buf[j] >= 32 && buf[j] <= 126 ) - sio << buf[j] << " "; - else - sio << " "; - } - sio << sio.nl; - } - sio << "Position: " << s.tell() << ", isEos()=" << s.isEos() - << sio.nl; - } - break; - - case modeStreamPut: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file manipulate." << sio.nl; - return 0; - } - else if( !opts.sSrc.isSet() ) - { - sio << "Please specify a source file to read." << sio.nl; - } - else - { - File fOut( opts.sFile, File::Write|File::Read ); - Myriad m( fOut ); - MyriadStream sOut = m.openStream( - m.createStream( opts.iPreallocate ) - ); - File fIn( opts.sSrc, File::Read ); - char buf[1024]; - while( !fIn.isEos() ) - { - sOut.write( buf, fIn.read( buf, 1024 ) ); - } - } - break; - - case modeStreamGet: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file manipulate." << sio.nl; - return 0; - } - else if( !opts.sDst.isSet() ) - { - sio << "Please specify a destination file to write." << sio.nl; - } - else - { - File fIn( opts.sFile, File::Write|File::Read ); - Myriad m( fIn ); - MyriadStream sIn = m.openStream( opts.iStream ); - File fOut( opts.sDst, File::Write|File::Create|File::Truncate ); - char buf[1024]; - while( !sIn.isEos() ) - { - fOut.write( buf, sIn.read( buf, 1024 ) ); - } - } - break; - - case modeBlockMap: - if( !opts.sFile.isSet() ) - { - sio << "Please specify a file manipulate." << sio.nl; - return 0; - } - { - File fIn( opts.sFile, File::Write|File::Read ); - Myriad m( fIn ); - Bu::BitString bs = m.getBlocksUsed(); - for( int j = 0; j < bs.getSize(); j++ ) - { - if( j>0 && (j%50) == 0 ) - Bu::println(""); - if( bs.getBit( j ) ) - Bu::print("#"); - else - Bu::print("-"); - } - Bu::println("\n"); - } - break; - - case modeNone: - sio << "Please select a mode, for more info, try --help." - << sio.nl << sio.nl; - break; - } - - return 0; -} - diff --git a/src/tools/myriadfs.cpp b/src/tools/myriadfs.cpp deleted file mode 100644 index 587bc89..0000000 --- a/src/tools/myriadfs.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/sio.h" -#include "bu/streamstack.h" -#include "bu/file.h" -#include "bu/myriadfs.h" -#include "bu/myriadstream.h" -#include "bu/optparser.h" - -enum Mode -{ - modeList, - modeCat, - modeCopyIn, - modeCopyOut, - modeMkdir, - modeInitialize, - modeRm, - - modeNone -}; - -Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ ) -{ - Bu::sio << "Uh oh, the formatter was called..." << Bu::sio.nl; - return f; -} - -class Options : public Bu::OptParser -{ -public: - Options( int argc, char *argv[] ) : - eMode( modeNone ), - iBlockSize( 64 ) - { - addHelpBanner("Options:"); - addOption( sFile, 'f', "file", "Myriadfs file"); - addOption( iBlockSize, 'b', "block-size", - "Specify the block size when initializing a new MyriadFs"); - - setNonOption( Bu::slot( this, &Options::nonOption ) ); - - addHelpOption(); - - parse( argc, argv ); - } - - int nonOption( Bu::Array aParams ) - { - if( eMode == modeNone ) - { - //Bu::println("Checking mode"); - // First param, must be the mode - if( aParams[0] == "ls" ) - eMode = modeList; - else if( aParams[0] == "cat" ) - eMode = modeCat; - else if( aParams[0] == "cp-in" ) - eMode = modeCopyIn; - else if( aParams[0] == "cp-out" ) - eMode = modeCopyOut; - else if( aParams[0] == "mkdir" ) - eMode = modeMkdir; - else if( aParams[0] == "initialize" ) - eMode = modeInitialize; - else if( aParams[0] == "rm" ) - eMode = modeRm; - else - Bu::println("Unknown command, try --help"); - return 0; - } else { - lParams.append( aParams[0] ); - } - //Bu::println("Got option: \"%1\"").arg( aParams[0] ); - return 0; - } - - Mode eMode; - Bu::String sFile; - Bu::StringList lParams; - int iBlockSize; -}; - -int main( int argc, char *argv[] ) -{ - Options opt( argc, argv ); - - if( opt.sFile.isEmpty() ) - { - Bu::println("You must specify a MyriadFs stream (see -f).\n"); - return 1; - } - - if( opt.eMode == modeNone ) - { - Bu::println("Specify an operation to perfrom.\n"); - return 1; - } - - int iFlags = Bu::File::ReadWrite; - - if( opt.eMode == modeInitialize ) - { - iFlags |= Bu::File::Create|Bu::File::Truncate; - } - - Bu::File fFs( opt.sFile, iFlags ); - Bu::MyriadFs mFs( fFs, opt.iBlockSize ); - - switch( opt.eMode ) - { - case modeList: - { - Bu::String sPath = "/"; - if( opt.lParams.getSize() > 0 ) - { - sPath = opt.lParams.first(); - } - Bu::MyriadFs::Dir lEnt = mFs.readDir( sPath ); - for( Bu::MyriadFs::Dir::iterator i = lEnt.begin(); i; i++ ) - { - Bu::String sPerm; - switch( (*i).uPerms&Bu::MyriadFs::typeMask ) - { - case Bu::MyriadFs::typeDir: sPerm += "d"; break; - case Bu::MyriadFs::typeChrDev: sPerm += "c"; break; - case Bu::MyriadFs::typeBlkDev: sPerm += "b"; break; - case Bu::MyriadFs::typeSymLink: sPerm += "l"; break; - case Bu::MyriadFs::typeSocket: sPerm += "s"; break; - default: sPerm += "-"; break; - } - sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrR)?"r":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrW)?"w":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrX)?"x":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpR)?"r":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpW)?"w":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpX)?"x":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permOthR)?"r":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permOthW)?"w":"-"; - sPerm += ((*i).uPerms&Bu::MyriadFs::permOthX)?"x":"-"; - Bu::println("%1 %2 %3:%4 %5 %6") - .arg( sPerm ) - .arg( (*i).iNode ) - .arg( (*i).iUser ) - .arg( (*i).iGroup ) - .arg( (*i).iSize ) - .arg( (*i).sName ); - } - } - break; - - case modeCat: - { - if( opt.lParams.isEmpty() ) - { - Bu::println("Specify at least one file."); - return 1; - } - int iBlockSize = 1024*1024; - char *pBuf = new char[iBlockSize]; - for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ ) - { - Bu::MyriadStream ms = mFs.open( *i, Bu::MyriadFs::Read ); - int iRead = 0; - do - { - iRead = ms.read( pBuf, iBlockSize ); - if( iRead > 0 ) - { - Bu::sioRaw.write( pBuf, iRead ); - } - } while( iRead == iBlockSize ); - } - delete[] pBuf; - } - break; - - case modeCopyIn: - { - if( opt.lParams.getSize() != 2 ) - { - Bu::println("Specify a source file and destination file."); - return 1; - } - int iBlockSize = 1024*1024; - char *pBuf = new char[iBlockSize]; - Bu::File fs = Bu::File( - opt.lParams.first(), Bu::File::Read ); - Bu::MyriadStream ms = mFs.open( - opt.lParams.last(), Bu::MyriadFs::WriteNew ); - int iRead = 0; - do - { - iRead = fs.read( pBuf, iBlockSize ); - if( iRead > 0 ) - { - ms.write( pBuf, iRead ); - } - } while( iRead == iBlockSize ); - delete[] pBuf; - } - break; - - case modeCopyOut: - { - if( opt.lParams.getSize() != 2 ) - { - Bu::println("Specify a source file and destination file."); - return 1; - } - int iBlockSize = 1024*1024; - char *pBuf = new char[iBlockSize]; - Bu::MyriadStream ms = mFs.open( - opt.lParams.first(), Bu::MyriadFs::Read ); - Bu::File fs = Bu::File( - opt.lParams.last(), Bu::File::WriteNew ); - int iRead = 0; - do - { - iRead = ms.read( pBuf, iBlockSize ); - if( iRead > 0 ) - { - fs.write( pBuf, iRead ); - } - } while( iRead == iBlockSize ); - delete[] pBuf; - } - break; - - case modeMkdir: - { - if( opt.lParams.isEmpty() ) - { - Bu::println("Specify at least one directory."); - return 1; - } - for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ ) - { - mFs.mkDir( *i, 0777 ); - } - } - break; - - case modeInitialize: - Bu::println("MyriadFs initialized.\n"); - break; - - case modeRm: - { - if( opt.lParams.getSize() != 1 ) - { - Bu::println("Specify a file path."); - return 1; - } - mFs.unlink( opt.lParams.first() ); - } - break; - - case modeNone: - break; - } - - return 0; -} diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit deleted file mode 100644 index 24d3116..0000000 --- a/src/unit/myriad.unit +++ /dev/null @@ -1,385 +0,0 @@ -// vim: syntax=cpp -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/string.h" -#include "bu/file.h" -#include "bu/myriad.h" -#include "bu/myriadstream.h" -#include "bu/array.h" - -#include "bu/sio.h" -#include "bu/archive.h" -#include "bu/md5.h" -#include "bu/unitsuite.h" - -#include - -using namespace Bu; - -class VerifyObject -{ -friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo ); -friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo ); -public: - VerifyObject( int iUnits ) : - iUnits( iUnits ), - iBytesWritten( 0 ) - { - } - - virtual ~VerifyObject() - { - } - - int getBytesWritten() - { - return iBytesWritten; - } - -private: - int iUnits; - mutable int iBytesWritten; -}; - -Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo ) -{ - Md5 sum; - ar << vo.iUnits; - vo.iBytesWritten = sizeof(int); - sum.addData( &vo.iUnits, sizeof(int) ); - for( int j = 0; j < vo.iUnits; j++ ) - { - int iRand = random()%128; -// ar << iRand; - Bu::String sDat( iRand ); - for( int j = 0; j < iRand; j++ ) - sDat[j] = (char)((uint8_t)(random()%256)); - ar << sDat; - sum.addData( &iRand, sizeof(int) ); - sum.addData( sDat.getStr(), iRand ); - vo.iBytesWritten += sizeof(long) + iRand; - } - Bu::String sRes = sum.getResult(); - ar << sRes; - vo.iBytesWritten += sizeof(long) + sRes.getSize(); - return ar; -} - -Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo ) -{ - Md5 sum; - ar >> vo.iUnits; - sum.addData( &vo.iUnits, sizeof(int) ); - for( int j = 0; j < vo.iUnits; j++ ) - { - int iRand; -// ar >> iRand; - Bu::String sStr; - ar >> sStr; - iRand = sStr.getSize(); - sum.addData( &iRand, sizeof(int) ); - sum.addData( sStr.getStr(), iRand ); - } - Bu::String sSum; - ar >> sSum; - unitTest( sSum == sum.getResult() ); - int iTooMuch; - try - { - ar >> iTooMuch; - unitFailed("should have thrown an exception."); - } - catch( Bu::ExceptionBase &e ) - { - } - return ar; -} - -suite Myriad -{ - test setSize - { - String sFileName("myriad-XXXXXXX"); - - File fMyriad = tempFile( sFileName ); - Myriad m( fMyriad, 32 ); - - MyriadStream ms = m.openStream( m.createStream() ); - ms.setSize( 150 ); - ms.setPos( 145 ); - char stuff[10]; - unitTest( ms.read( stuff, 10 ) == 5 ); - - ms.setSize( 12 ); - unitTest( ms.read( stuff, 10 ) == 0 ); - unitTest( ms.write( "hello", 5 ) == 5 ); - unitTest( ms.tell() == 17 ); - - ms.setSize( 500 ); - unitTest( ms.tell() == 17 ); - } - - void addBlock( Stream &s, bool bAppend=true ) - { - if( bAppend ) - s.setPosEnd( 0 ); - int iSize = (random()%1016)+8; - s.write( &iSize, 4 ); - char *buf = new char[iSize-8]; - for( int j = 0; j < iSize-8; j++ ) - { - buf[j] = (j+iSize)%256; - } - if( random()%2 == 0 ) - { - s.write( buf, iSize-8 ); - } - else - { - for( int j = 0; j < iSize-8; ) - { - int iAmnt = (random()%8)+1; - if( iAmnt+j > iSize-8 ) - iAmnt = iSize-8-j; - iAmnt = s.write( buf+j, iAmnt ); - j += iAmnt; - } - } - delete[] buf; - iSize = ~iSize; - s.write( &iSize, 4 ); - } - - void verifyBlock( Stream &s ) - { - int iSize, iInv; - if( s.read( &iSize, 4 ) == 0 ) - return; - if( iSize < 8 || iSize > 1024 ) - throw ExceptionBase("Read bad data, %d", iSize ); - char *buf = new char[iSize-8]; - if( s.read( buf, iSize-8 ) < (size_t)iSize-8 ) - { - delete[] buf; - throw ExceptionBase("Block failed verify (insuffient block data)."); - } - for( int j = 0; j < iSize-8; j++ ) - { - if( buf[j] != (char)((j+iSize)%256) ) - { - char b = buf[j]; - delete[] buf; - throw ExceptionBase("Block failed computed data verify " - "(%02X==%02X).", b, (char)((j+iSize)%256) ); - } - } - delete[] buf; - if( s.read( &iInv, 4 ) < 4 ) - throw ExceptionBase("Block failed verify (insufficient data)."); - if( iInv != ~iSize ) - throw ExceptionBase("Block failed inversion verify."); - } - - void verifyStream( Stream &s ) - { - s.setPos( 0 ); - while( !s.isEos() ) - verifyBlock( s ); - } - - test stressGrow - { - String sFileName("myriad-XXXXXXX"); - - File fMyriad = tempFile( sFileName ); - Myriad m( fMyriad ); - m.initialize( 64 ); - - Array aStreams; - for( int j = 0; j < 5; j++ ) - { - aStreams.append( m.createStream() ); - } - - srandom( 512 ); - - for( int j = 0; j < 2500; j++ ) - { - switch( random()%5 ) - { - case 0: - aStreams.append( m.createStream() ); - break; - - case 1: - if( aStreams.getSize() > 0 ) - { - int iStream = random()%aStreams.getSize(); - { - MyriadStream ms = m.openStream( aStreams[iStream] ); - verifyStream( ms ); - } - m.deleteStream( aStreams[iStream] ); - Array::iterator i = aStreams.begin(); - for( int k = 0; k < iStream; k++ ) - i++; - aStreams.erase( i ); - } - break; - - default: - if( aStreams.getSize() == 0 ) - { - aStreams.append( m.createStream() ); - } - { - int iStream = random()%aStreams.getSize(); - MyriadStream ms = m.openStream( aStreams[iStream] ); - addBlock( ms ); - verifyStream( ms ); - } - break; - } - } - - for( Array::iterator i = aStreams.begin(); i; i++ ) - { - MyriadStream ms = m.openStream( *i ); - verifyStream( ms ); - } - } - - test stressTruncate - { - String sFileName("myriad-XXXXXXX"); - - File fMyriad = tempFile( sFileName ); - Myriad m( fMyriad ); - m.initialize( 128 ); - - Array aStream; - - for( int j = 0; j < 5; j++ ) - { - aStream.append( m.createStream() ); - } - - srandom( 1024 ); - - char b; - for( int iter = 0; iter < 2500; iter++ ) - { - for( Array::iterator i = aStream.begin(); i; i++ ) - { - MyriadStream ms = m.openStream( *i ); - addBlock( ms, false ); - ms.setSize( ms.tell() ); - unitTest( ms.read( &b, 1 ) == 0 ); - ms.setPos( 0 ); - verifyBlock( ms ); - unitTest( ms.read( &b, 1 ) == 0 ); - } - } - } - - test stressTruncate2 - { - String sFileName("myriad-XXXXXXX"); - - Array aStream; - - setStepCount( 5*2500 + 5 ); - - { - File fMyriad = tempFile( sFileName ); - Myriad m( fMyriad, 128 ); - - for( int j = 0; j < 5; j++ ) - { - aStream.append( m.createStream() ); - incProgress(); - } - } - - srandom( 1024 ); - - char b; - for( int iter = 0; iter < 2500; iter++ ) - { - File fMyriad( sFileName, File::ReadWrite ); - Myriad m( fMyriad ); - for( Array::iterator i = aStream.begin(); i; i++ ) - { - MyriadStream ms = m.openStream( *i ); - addBlock( ms, false ); - ms.setSize( ms.tell() ); - unitTest( ms.read( &b, 1 ) == 0 ); - ms.setPos( 0 ); - verifyBlock( ms ); - unitTest( ms.read( &b, 1 ) == 0 ); - incProgress(); - } - } - } - - test stressArchive - { - String sFileName("myriad-XXXXXX"); - Array aStream; - - srandom( 2096 ); - - setStepCount( 15*250 + 15 ); - - { - File fMyriad = tempFile( sFileName ); - Myriad m( fMyriad, 1024 ); - - for( int j = 0; j < 15; j++ ) - { - int iStream = m.createStream(); - aStream.append( iStream ); - VerifyObject vo( random()%1024 ); - { - MyriadStream ms = m.openStream( iStream ); - Archive ar( ms, Archive::save ); - ar << vo; - unitTest( ms.tell() == vo.getBytesWritten() ); - ms.setSize( ms.tell() ); - } - unitTest( m.getStreamSize( iStream ) == vo.getBytesWritten() ); - incProgress(); - } - } - - for( int iter = 0; iter < 250; iter++ ) - { - File fMyriad( sFileName, File::ReadWrite ); - Myriad m( fMyriad ); - for( Array::iterator i = aStream.begin(); i; i++ ) - { - VerifyObject vo( random()%1024 ); - { - MyriadStream ms = m.openStream( *i ); - Archive ar( ms, Archive::load ); - ar >> vo; - } - { - MyriadStream ms = m.openStream( *i ); - Archive ar( ms, Archive::save ); - ar << vo; - unitTest( ms.tell() == vo.getBytesWritten() ); - ms.setSize( ms.tell() ); - } - unitTest( m.getStreamSize( *i ) == vo.getBytesWritten() ); - incProgress(); - } - } - } -} - diff --git a/src/unstable/myriadcache.cpp b/src/unstable/myriadcache.cpp deleted file mode 100644 index c9eb9c4..0000000 --- a/src/unstable/myriadcache.cpp +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/myriadcache.h" diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h deleted file mode 100644 index 24002b0..0000000 --- a/src/unstable/myriadcache.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef BU_MYRIAD_CACHE_H -#define BU_MYRIAD_CACHE_H - -#include "bu/cachebase.h" -#include "bu/myriad.h" -#include "bu/myriadstream.h" -#include "bu/file.h" -#include "bu/streamstack.h" - -namespace Bu -{ - template - class MyriadCache : public Bu::CacheBase - { - public: - MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) : - sStore( sStore ), - mStore( sStore, iBlockSize, iPreallocate ), - bStructureChanged( false ) - { - try - { - Bu::ReadWriteMutex::ReadLocker l( rwStore ); - Bu::MyriadStream ms = mStore.openStream( 1 ); - Bu::Archive ar( ms, Bu::Archive::load ); - uint8_t uVer; - ar >> uVer; - switch( uVer ) - { - case 0: - ar >> hIndex; - break; - } - } - catch(...) - { - if( mStore.createStreamWithId( 1 ) != 1 ) - throw Bu::ExceptionBase("Error creating index stream."); - - _sync(); - } - } - - virtual ~MyriadCache() - { - Bu::CacheBase::sync(); - } - - using typename Bu::CacheBase::KeyList; - using typename Bu::CacheBase::ObjectType; - - virtual typename Bu::CacheBase::KeyList getKeys() const - { - Bu::ReadWriteMutex::ReadLocker rl( rwStore ); - return hIndex.getKeys(); - } - - virtual int getSize() const - { - Bu::ReadWriteMutex::ReadLocker rl( rwStore ); - return hIndex.getSize(); - } - - protected: - virtual bool _has( const keytype &key ) - { - Bu::ReadWriteMutex::ReadLocker rl( rwStore ); - return hIndex.has( key ); - } - - virtual void _create( const obtype *o ) - { - Bu::ReadWriteMutex::WriteLocker wl( rwStore ); - hIndex.insert( o->getKey(), mStore.createStream() ); - _save( o ); - - bStructureChanged = true; - } - - virtual void _erase( const keytype &k ) - { - Bu::ReadWriteMutex::WriteLocker wl( rwStore ); - mStore.deleteStream( hIndex.get( k ) ); - hIndex.erase( k ); - - bStructureChanged = true; - } - - virtual obtype *_load( - typename Bu::CacheObject::Initializer &initObj, - const keytype &k - ) - { - Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) ); - return _cacheObjectLoad( initObj, k, ms ); - } - - virtual void _save( const obtype *o ) - { - Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) ); - _cacheObjectSave( ms, o ); - ms.setSize( ms.tell() ); - - mStore.sync(); - } - - virtual void _sync() - { - Bu::ReadWriteMutex::WriteLocker wl( rwStore ); - if( !bStructureChanged ) - return; - - Bu::MyriadStream ms = mStore.openStream( 1 ); - Bu::Archive ar( ms, Bu::Archive::save ); - ar << (uint8_t)0 << hIndex; - ar.close(); - ms.setSize( ms.tell() ); - - bStructureChanged = false; - - mStore.sync(); - } - - private: - Bu::Stream &sStore; - Bu::Myriad mStore; - Bu::Hash hIndex; - mutable Bu::ReadWriteMutex rwStore; - bool bStructureChanged; - }; -} - -#endif diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp deleted file mode 100644 index 991c21d..0000000 --- a/src/unstable/myriadfs.cpp +++ /dev/null @@ -1,722 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#include "bu/config.h" -#include "bu/myriadfs.h" -#include "bu/myriadstream.h" - -#include -#include -#include - -#include "bu/sio.h" -using Bu::sio; -using Bu::Fmt; - -namespace Bu { subExceptionDef( MyriadFsException ) } - -#define Myriad_Fs_MAGIC_CODE ((char *)"\xa7\x18\x8b\x39") - -Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) : - rStore( rStore ), - mStore( rStore, iBlockSize ), - iUser( 0 ), - iGroup( 0 ) -{ -#ifndef WIN32 - iUser = getuid(); - iGroup = getgid(); -#endif - - if( mStore.hasStream( 1 ) ) - { - // Check to see if this is a MyriadFs stream. - Bu::MyriadStream ms = mStore.openStream( 1 ); - char sMagic[4]; - if( ms.read( sMagic, 4 ) < 4 ) - throw MyriadFsException("The provided stream does not appear to be " - "a MyriadFs stream."); - if( ::strncmp( sMagic, Myriad_Fs_MAGIC_CODE, 4 ) ) - throw MyriadFsException("The provided stream does not appear to be " - "a MyriadFs stream."); - - int8_t iVer; - ms.read( &iVer, 1 ); - - int32_t iNumNodes; - ms.read( &iNumNodes, 4 ); - for( int32_t j = 0; j < iNumNodes; j++ ) - { - int32_t iNode; - int32_t iPos; - ms.read( &iNode, 4 ); - ms.read( &iPos, 4 ); - hNodeIndex.insert( iNode, iPos ); - } - } - else - { - // Create initial header stream - { - mStore.createStream( 1 ); - Bu::MyriadStream ms = mStore.openStream( 1 ); - ms.write( Myriad_Fs_MAGIC_CODE, 4 ); - int8_t iVer = 1; - int32_t iTmp = 1; - ms.write( &iVer, 1 ); - ms.write( &iTmp, 4 ); // iNumNodes - iTmp = 0; - ms.write( &iTmp, 4 ); // iInode - ms.write( &iTmp, 4 ); // iPosition - hNodeIndex.insert( 0, 0 ); - } - - // Create initial inode stream, with one root node. - { - mStore.createStream( 2 ); - Bu::MyriadStream ms = mStore.openStream( 2 ); - RawStat rs; - rs.iNode = 0; - rs.iUser = iUser; - rs.iGroup = iGroup; - rs.uPerms = 0755|typeDir; - rs.iLinks = 1; - rs.uStreamIndex = 3; - rs.iCTime = rs.iMTime = rs.iATime = time(NULL); - ms.write( &rs, sizeof(RawStat) ); - } - - // Create inode 0's storage stream. - { - mStore.createStream( 3 ); - Bu::MyriadStream ms = mStore.openStream( 3 ); - int32_t iTmp32 = 0; - ms.write( &iTmp32, 4 ); // iChildCount - } - } -} - -Bu::MyriadFs::~MyriadFs() -{ - writeHeader(); -} - -void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf ) -{ - int32_t iParent; - int32_t iNode = lookupInode( sPath, iParent ); - Bu::MyriadStream is = mStore.openStream( 2 ); - stat( iNode, rBuf, is ); -} - -Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode, - uint16_t uPerms ) -{ - int32_t iParent = -1; - int32_t iNode; - try - { - iNode = lookupInode( sPath, iParent ); -// sio << "File found." << sio.nl; - // The file was found - Bu::MyriadStream ms = openByInode( iNode ); - if( (iMode&Truncate) ) - { - ms.setSize( 0 ); - } - return ms; - } - catch( Bu::MyriadFsException &e ) - { - if( iParent < 0 ) - throw; - - // This means that an intermediate path component couldn't be found - if( e.getErrorCode() == 1 ) - throw; - - // The file wasn't found, but the path leading up to it was. - // first, figure out the final path element... - Bu::String sName = filePart( sPath ); -// sio << "End filename: " << sName << sio.nl; -// sio << "Parent inode: " << iParent << sio.nl; - iNode = create( iParent, sName, (uPerms&permMask)|typeRegFile, 0 ); -// sio << "New iNode: " << iNode << sio.nl; - return openByInode( iNode ); - } -} - -void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms ) -{ - create( sPath, iPerms, 0 ); -} - -void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms, - uint16_t iDevHi, uint16_t iDevLo ) -{ - create( sPath, iPerms, ((uint32_t)iDevHi<<16)|(uint32_t)iDevLo ); -} - -void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms, - uint32_t uSpecial ) -{ - int32_t iParent = -1; -// int32_t iNode; - try - { - /*iNode =*/ lookupInode( sPath, iParent ); -// sio << "File found." << sio.nl; - } - catch( Bu::MyriadFsException &e ) - { - if( iParent < 0 ) - throw; - - // This means that an intermediate path component couldn't be found - if( e.getErrorCode() == 1 ) - throw; - - // The file wasn't found, but the path leading up to it was. - // first, figure out the final path element... - Bu::String sName = filePart( sPath ); -// sio << "End filename: " << sName << sio.nl; -// sio << "Parent inode: " << iParent << sio.nl; - /*iNode =*/ create( iParent, sName, iPerms, uSpecial ); -// sio << "New iNode: " << iNode << sio.nl; - } - // The file was found - //throw Bu::MyriadFsException("Path already exists."); -} - -void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms ) -{ - create( sPath, (iPerms&permMask)|typeDir, 0 ); -} - -void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget, - const Bu::String &sPath ) -{ - int32_t iParent = -1; - int32_t iNode; - try - { - iNode = lookupInode( sPath, iParent ); - } - catch( Bu::MyriadFsException &e ) - { - if( iParent < 0 ) - throw; - - // This means that an intermediate path component couldn't be found - if( e.getErrorCode() == 1 ) - throw; - - // The file wasn't found, but the path leading up to it was. - // first, figure out the final path element... - Bu::String sName = filePart( sPath ); -// sio << "End filename: " << sName << sio.nl; -// sio << "Parent inode: " << iParent << sio.nl; - iNode = create( iParent, sName, 0777|typeSymLink, 0 ); -// sio << "New iNode: " << iNode << sio.nl; - MyriadStream ms = openByInode( iNode ); - ms.write( sTarget ); - return; - } - throw Bu::MyriadFsException("Path already exists."); -} - -void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget, - const Bu::String &sPath ) -{ - int32_t iParent = -1; - int32_t iNode; - - iNode = lookupInode( sTarget, iParent ); - - try - { - lookupInode( sPath, iParent ); - throw Bu::MyriadFsException("Path already exists."); - } - catch( Bu::MyriadFsException &e ) - { - if( iParent < 0 ) - throw; - - // This means that an intermediate path component couldn't be found - if( e.getErrorCode() == 1 ) - throw; - - // The file wasn't found, but the path leading up to it was. - // first, figure out the final path element... - Bu::String sName = filePart( sPath ); -// sio << "End filename: " << sName << sio.nl; -// sio << "Parent inode: " << iParent << sio.nl; - addToDir( iParent, iNode, sName ); - MyriadStream is = mStore.openStream( 2 ); - RawStat rs; - readInode( iNode, rs, is ); - rs.iLinks++; - writeInode( rs, is ); - } -} - -Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath ) -{ - int32_t iParent = -1; - int32_t iNode; - iNode = lookupInode( sPath, iParent ); - MyriadStream ms = openByInode( iNode ); - Bu::String sRet; - sRet.setSize( ms.getSize() ); - ms.read( sRet.getStr(), ms.getSize() ); - return sRet; -} - -Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath ) -{ - int32_t iParent = -1; - int32_t iNode = lookupInode( sPath, iParent ); - return readDir( iNode ); -} - -void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime, - int64_t iMTime ) -{ - int32_t iParent = -1; - int32_t iNode; - - iNode = lookupInode( sPath, iParent ); - - setTimes( iNode, iATime, iMTime ); -} - -void Bu::MyriadFs::unlink( const Bu::String &sPath ) -{ - int32_t iParent = -1; -// int32_t iNode; - - /*iNode =*/ lookupInode( sPath, iParent ); - - Dir lDir = readDir( iParent ); - - Bu::String sName = filePart( sPath ); - - for( Dir::iterator i = lDir.begin(); i; i++ ) - { - if( sName == (*i).sName ) - { - RawStat rs; - readInode( (*i).iNode, rs ); - if( (rs.uPerms&typeMask) == typeDir ) - { - MyriadStream msDir = mStore.openStream( rs.uStreamIndex ); - int32_t iCount; - msDir.read( &iCount, 4 ); - if( iCount > 0 ) - { - throw Bu::MyriadFsException("Directory not empty."); - } - } - if( --rs.iLinks == 0 ) - { - destroyNode( (*i).iNode ); - } - else - { - writeInode( rs ); - } - lDir.erase( i ); - break; - } - } - - Bu::MyriadStream ms = openByInode( iParent ); - int32_t iNumChildren = lDir.getSize(); - ms.write( &iNumChildren, 4 ); - for( Dir::iterator i = lDir.begin(); i; i++ ) - { - ms.write( &(*i).iNode, 4 ); - uint8_t iSize = (*i).sName.getSize(); - ms.write( &iSize, 1 ); - ms.write( (*i).sName.getStr(), iSize ); - } - ms.setSize( ms.tell() ); -} - -void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize ) -{ - int32_t iParent = -1; - int32_t iNode; - iNode = lookupInode( sPath, iParent ); - MyriadStream ms = openByInode( iNode ); - ms.setSize( iSize ); -} - -void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo ) -{ - mkHardLink( sFrom, sTo ); - unlink( sFrom ); -} - -dev_t Bu::MyriadFs::devToSys( uint32_t uDev ) -{ - return (((uDev&0xFFFF0000)>>8)&0xFF00) | ((uDev&0xFF)); -} - -uint32_t Bu::MyriadFs::sysToDev( dev_t uDev ) -{ - return (((uint32_t)uDev&0xFF00)<<8) | ((uint32_t)uDev&0xFF); -} - -int32_t Bu::MyriadFs::lookupInode( const Bu::String &sPath, int32_t &iParent ) -{ - if( sPath == "/" ) - { - return 0; - } - if( sPath[0] == '/' ) - { - // Absolute lookup - return lookupInode( sPath.begin()+1, 0, iParent ); - } - else - { - // Relative lookup - throw Bu::ExceptionBase( - "Relative lookups in MyriadFs are not working yet."); - } -} - -int32_t Bu::MyriadFs::lookupInode( Bu::String::const_iterator iStart, - int32_t iNode, int32_t &iParent ) -{ - iParent = iNode; - - Bu::String::const_iterator iEnd = iStart.find('/'); - Bu::String sTok( iStart, iEnd ); - -// sio << "Direcotry component: " << sTok << sio.nl; - - Dir lDir = readDir( iNode ); - - for( Dir::iterator i = lDir.begin(); i; i++ ) - { - if( (*i).sName == sTok ) - { - // We found an item - if( !iEnd ) - { - // It's the last one in the requested path, return it - return (*i).iNode; - } - else - { - // Not the last one in our path, double check it's a dir - if( ((*i).uPerms&typeMask) == typeDir ) - { - return lookupInode( iEnd+1, (*i).iNode, iParent ); - } - else - { - iParent = -1; - throw Bu::MyriadFsException( - "Element '%s' in given path is not a directory.", - sTok.getStr() ); - } - } - } - } - - if( iEnd ) - throw Bu::MyriadFsException( 1, "Path not found"); - else - throw Bu::MyriadFsException( 2, "Path not found"); -} - -void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs ) -{ - rIs.setPos( hNodeIndex.get( iNode )*sizeof(RawStat) ); - if( rIs.read( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) ) - throw Bu::MyriadFsException("Filesystem corruption detected."); - if( rs.iNode != iNode ) - throw Bu::MyriadFsException("Filesystem corruption detected."); -} - -void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs ) -{ - MyriadStream ms = mStore.openStream( 2 ); - readInode( iNode, rs, ms ); -} - -void Bu::MyriadFs::writeInode( const RawStat &rs, - MyriadStream &rOs ) -{ - rOs.setSize( hNodeIndex.getSize()*sizeof(RawStat) ); - rOs.setPos( hNodeIndex.get( rs.iNode )*sizeof(RawStat) ); - if( rOs.write( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) ) - throw Bu::MyriadFsException("Error writing inode to header stream."); -} - -void Bu::MyriadFs::writeInode( const RawStat &rs ) -{ - MyriadStream ms = mStore.openStream( 2 ); - writeInode( rs, ms ); -} - -Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode ) -{ - Bu::MyriadStream ms = openByInode( iNode ); - int32_t iNumChildren = 0; - ms.read( &iNumChildren, 4 ); - - Bu::MyriadStream is = mStore.openStream( 2 ); - Dir lDir; - // sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl; - for( int32_t j = 0; j < iNumChildren; j++ ) - { - int32_t iChildNode = 0; - if( ms.read( &iChildNode, 4 ) < 4 ) - { - throw Bu::MyriadFsException( - "Failed to read iChildNode from directory."); - } - Stat s; - stat( iChildNode, s, is ); - uint8_t uLen; - if( ms.read( &uLen, 1 ) < 1 ) - { - throw Bu::MyriadFsException( - "Failed to read uLen from directory."); - } - s.sName.setSize( uLen ); - if( ms.read( s.sName.getStr(), uLen ) < uLen ) - { - throw Bu::MyriadFsException( - "Failed to read sName from directory."); - } - lDir.append( s ); - -// sio << " " << s.sName << sio.nl; - } - - return lDir; -} - -Bu::MyriadStream Bu::MyriadFs::openByInode( int32_t iNode ) -{ - RawStat rs; - readInode( iNode, rs ); - switch( (rs.uPerms&typeMask) ) - { - case typeDir: - case typeSymLink: - case typeRegFile: - return mStore.openStream( rs.uStreamIndex ); - - default: - throw Bu::MyriadFsException( - "inode incorrect type for low-level openByInode."); - } -} - -void Bu::MyriadFs::addToDir( int32_t iDir, int32_t iNode, - const Bu::String &sName ) -{ - if( sName.getSize() > 255 ) - { - throw Bu::MyriadFsException("Filename too long, max is 255 bytes."); - } - Bu::MyriadStream ms = openByInode( iDir ); - int32_t iNumChildren = 0; - ms.read( &iNumChildren, 4 ); - iNumChildren++; - ms.setPos( 0 ); - ms.write( &iNumChildren, 4 ); - ms.setPosEnd( 0 ); - ms.write( &iNode, 4 ); - uint8_t uLen = sName.getSize(); - ms.write( &uLen, 1 ); - ms.write( sName.getStr(), uLen ); -} - -int32_t Bu::MyriadFs::create( int32_t iParent, const Bu::String &sName, - uint16_t uPerms, uint32_t uSpecial ) -{ - int32_t iNode = allocInode( uPerms, uSpecial ); - addToDir( iParent, iNode, sName ); - return iNode; -} - -int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial ) -{ - int32_t iNode = 0; - for(; iNode < 0xfffffff; iNode++ ) - { - if( !hNodeIndex.has( iNode ) ) - { - hNodeIndex.insert( iNode, hNodeIndex.getSize() ); - RawStat rs; - rs.iNode = iNode; - rs.iUser = iUser; - rs.iGroup = iGroup; - rs.uPerms = uPerms; - rs.iLinks = 1; - switch( (uPerms&typeMask) ) - { - case typeRegFile: - case typeSymLink: - rs.uStreamIndex = mStore.createStream(); - break; - - case typeDir: - rs.uStreamIndex = mStore.createStream(); -// sio << "Creating directory node, storage: " -// << rs.uStreamIndex << sio.nl; - { - Bu::MyriadStream msDir = mStore.openStream( - rs.uStreamIndex - ); - uint32_t uSize = 0; - msDir.write( &uSize, 4 ); - } - break; - - case typeChrDev: - case typeBlkDev: - rs.uStreamIndex = uSpecial; - break; - - default: - rs.uStreamIndex = 0; - break; - } - rs.iATime = time(NULL); - rs.iMTime = time(NULL); - rs.iCTime = time(NULL); - writeInode( rs ); - - return iNode; - } - } - - throw Bu::MyriadFsException( - "No inode could be allocated. You've run out!"); -} - -void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs ) -{ - RawStat rs; - readInode( iNode, rs, rIs ); - rBuf.iNode = iNode; - rBuf.iUser = rs.iUser; - rBuf.iGroup = rs.iGroup; - rBuf.uPerms = rs.uPerms; - rBuf.iLinks = rs.iLinks; - rBuf.iATime = rs.iATime; - rBuf.iMTime = rs.iMTime; - rBuf.iCTime = rs.iCTime; - rBuf.uDev = 0; - rBuf.iSize = 0; - switch( (rBuf.uPerms&typeMask) ) - { - case typeRegFile: - case typeSymLink: - rBuf.iSize = mStore.getStreamSize( rs.uStreamIndex ); - break; - - case typeChrDev: - case typeBlkDev: - rBuf.uDev = rs.uStreamIndex; - break; - - default: - rBuf.iSize = 0; - break; - } -} - -void Bu::MyriadFs::writeHeader() -{ - Bu::MyriadStream ms = mStore.openStream( 1 ); - ms.write( Myriad_Fs_MAGIC_CODE, 4 ); - int8_t iVer = 1; - int32_t iNumNodes = hNodeIndex.getSize(); - ms.write( &iVer, 1 ); - ms.write( &iNumNodes, 4 ); // iNumNodes - for( NodeIndex::iterator i = hNodeIndex.begin(); i; i++ ) - { - int32_t iNode = i.getKey(); - int32_t iPosition = i.getValue(); - ms.write( &iNode, 4 ); - ms.write( &iPosition, 4 ); - } - - // Truncate the stream afterwards so we don't use up too much space. - ms.setSize( ms.tell() ); -} - -void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ) -{ - RawStat rs; - Bu::MyriadStream is = mStore.openStream( 2 ); - - readInode( iNode, rs, is ); - rs.iATime = iATime; - rs.iMTime = iMTime; - writeInode( rs, is ); -} - -void Bu::MyriadFs::destroyNode( int32_t iNode ) -{ - if( iNode == 0 ) - throw Bu::MyriadFsException("You cannot destroy the root."); - - Bu::MyriadStream is = mStore.openStream( 2 ); - - // This will be overwritten with the last node - uint32_t iPosition = hNodeIndex.get( iNode ); - RawStat rsOld; - readInode( iNode, rsOld, is ); - switch( (rsOld.uPerms&typeMask) ) - { - case typeRegFile: - case typeDir: - case typeSymLink: - mStore.deleteStream( rsOld.uStreamIndex ); - break; - } - - hNodeIndex.erase( iNode ); - - // Read the last node, can't use the helpers, because we don't know the - // iNode yet. - if( iPosition != hNodeIndex.getSize() ) - { - // If this is the last node, then we don't need to do anything, but - // this case handles what to do if we aren't on the last node - RawStat rs; - is.setPos( (hNodeIndex.getSize())*sizeof(RawStat) ); - is.read( &rs, sizeof(RawStat) ); - - hNodeIndex.get( rs.iNode ) = iPosition; - writeInode( rs, is ); - } - - is.setSize( hNodeIndex.getSize() * sizeof(RawStat) ); -} - -Bu::String Bu::MyriadFs::filePart( const Bu::String &sPath ) -{ - Bu::String::const_iterator iStart = sPath.begin(); - if( *iStart == '/' ) - iStart++; - for( Bu::String::const_iterator iEnd = iStart.find('/'); iEnd; - iStart = iEnd+1, iEnd = iStart.find('/') ) { } - return Bu::String( iStart, sPath.end() ); -} - diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h deleted file mode 100644 index ff14292..0000000 --- a/src/unstable/myriadfs.h +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (C) 2007-2023 Xagasoft, All rights reserved. - * - * This file is part of the libbu++ library and is released under the - * terms of the license contained in the file LICENSE. - */ - -#ifndef MYRIAD_FS_H -#define MYRIAD_FS_H - -#include - -#include "bu/myriad.h" -#include "bu/readwritemutex.h" - -namespace Bu -{ - class Stream; - - subExceptionDecl( MyriadFsException ); - - /** - * A POSIX compliant, node based filesystem built on top of Myriad. - * - * A header is placed into stream 1. - * Header format: - * int32_t iMagicHeader (A7188B39) - * int8_t iVersion (1) - * int32_t iNumNodes - * NodeLookup[iNumNodes] nNode - * - * Node lookup: - * int32_t iInode - * int32_t iPosition - * - * The node headers or inode structures have a base size of 44 bytes. - * The name is stored in the directory format. - * Basic node header format: - * int32_t iNode - * int32_t iUser - * int32_t iGroup - * uint16_t uPerms - * int16_t iLinks - * uint32_t uStreamIndex - * int64_t iATime - * int64_t iMTime - * int64_t iCTime - * - * Some types get special formats for their assosiated data stream, or - * other special considerations, here's a list: - * - * - typeFifo: No stream, uStreamIndex unused (probably) - * - typeChrDev: No stream, uStreamIndex is device hi/lo - * - typeDir: The stream contains a directory contents listing, described - * below - * - typeBlkDev: No stream, uStreamIndex is device hi/lo - * - typeRegFile: The stream is the file data - * - typeSymLink: The stream is the destination of the symlink - * - typeSocket: No steram, uStreamIndex unused (probably) - * - * Directory streams have this simple listing format. They contain a list - * of all child elements, with no particular order at the moment. The . and - * .. entries are not listed, they are implicit: - * int32_t iNumNodes - * NodeTable[iNumNodes] nChildren - * - * NodeTable: - * int32_t iInode - * uint8_t uNameSize - * char[uNameSize] sName - */ - class MyriadFs - { - public: - MyriadFs( Bu::Stream &rStore, int iBlockSize=512 ); - virtual ~MyriadFs(); - - enum - { - permOthX = 0000001, - permOthW = 0000002, - permOthR = 0000004, - permGrpX = 0000010, - permGrpW = 0000020, - permGrpR = 0000040, - permUsrX = 0000100, - permUsrW = 0000200, - permUsrR = 0000400, - permSticky = 0001000, - permSetGid = 0002000, - permSetUid = 0004000, - permMask = 0007777, - typeFifo = 0010000, - typeChrDev = 0020000, - typeDir = 0040000, - typeBlkDev = 0060000, - typeRegFile = 0100000, - typeSymLink = 0120000, - typeSocket = 0140000, - typeMask = 0170000 - }; - - enum - { - Read = 0x01, ///< Open file for reading - Write = 0x02, ///< Open file for writing - Create = 0x04, ///< Create file if it doesn't exist - Truncate = 0x08, ///< Truncate file if it does exist - Append = 0x10, ///< Always append on every write - NonBlock = 0x20, ///< Open file in non-blocking mode - Exclusive = 0x44, ///< Create file, if it exists then fail - - // Helpful mixes - ReadWrite = 0x03, ///< Open for reading and writing - WriteNew = 0x0E ///< Create a file (or truncate) for writing. - /// Same as Write|Create|Truncate - }; - - class Stat - { - public: - int32_t iNode; - int32_t iUser; - int32_t iGroup; - uint16_t uPerms; - int16_t iLinks; - int64_t iATime; - int64_t iMTime; - int64_t iCTime; - int32_t iSize; - uint32_t uDev; - Bu::String sName; - }; - typedef Bu::List Dir; - - void stat( const Bu::String &sPath, Stat &rBuf ); - MyriadStream open( const Bu::String &sPath, int iMode, - uint16_t uPerms=0664 ); - void create( const Bu::String &sPath, uint16_t iPerms ); - void create( const Bu::String &sPath, uint16_t iPerms, - uint16_t iDevHi, uint16_t iDevLo ); - void create( const Bu::String &sPath, uint16_t iPerms, - uint32_t uSpecial ); - void mkDir( const Bu::String &sPath, uint16_t iPerms ); - void mkSymLink( const Bu::String &sTarget, const Bu::String &sPath ); - void mkHardLink( const Bu::String &sTarget, const Bu::String &sPath ); - Bu::String readSymLink( const Bu::String &sPath ); - Dir readDir( const Bu::String &sPath ); - void setTimes( const Bu::String &sPath, int64_t iATime, - int64_t iMTime ); - void unlink( const Bu::String &sPath ); - void setFileSize( const Bu::String &sPath, int32_t iSize ); - void rename( const Bu::String &sFrom, const Bu::String &sTo ); - - static dev_t devToSys( uint32_t uDev ); - static uint32_t sysToDev( dev_t uDev ); - - private: - class RawStat - { - public: - int32_t iNode; - int32_t iUser; - int32_t iGroup; - uint16_t uPerms; - int16_t iLinks; - uint32_t uStreamIndex; - int64_t iATime; - int64_t iMTime; - int64_t iCTime; - }; - typedef Bu::Hash NodeIndex; - - private: - int32_t lookupInode( const Bu::String &sPath, int32_t &iParent ); - int32_t lookupInode( Bu::String::const_iterator iStart, - int32_t iNode, int32_t &iParent ); - void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs ); - void readInode( int32_t iNode, RawStat &rs ); - void writeInode( const RawStat &rs ); - void writeInode( const RawStat &rs, MyriadStream &rOs ); - Dir readDir( int32_t iNode ); - MyriadStream openByInode( int32_t iNode ); - void addToDir( int32_t iDir, int32_t iNode, const Bu::String &sName ); - int32_t create( int32_t iParent, const Bu::String &sName, - uint16_t uPerms, uint32_t uSpecial ); - int32_t allocInode( uint16_t uPerms, uint32_t uSpecial ); - void stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs ); - void writeHeader(); - void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ); - void destroyNode( int32_t iNode ); - - Bu::String filePart( const Bu::String &sPath ); - - private: - Bu::Stream &rStore; - Bu::Myriad mStore; - Bu::ReadWriteMutex mNodeIndex; - NodeIndex hNodeIndex; - int32_t iUser; - int32_t iGroup; - }; -}; - -#endif -- cgit v1.2.3