From 76821551f312dd447a03748a01670f3718cd8345 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Thu, 26 Sep 2024 14:43:22 -0700 Subject: Basic update to new API for existing components. This may not all work yet, but it all compiles! --- src/stable/myriad.cpp | 21 +- src/stable/myriad.h | 24 +- src/stable/myriadstream.cpp | 6 + src/stable/myriadstream.h | 1 + src/tools/myriadfs.cpp | 269 ++++++++++++++++ src/unstable/myriadcache.cpp | 8 + src/unstable/myriadcache.h | 150 +++++++++ src/unstable/myriadfs.cpp | 737 +++++++++++++++++++++++++++++++++++++++++++ src/unstable/myriadfs.h | 205 ++++++++++++ 9 files changed, 1418 insertions(+), 3 deletions(-) create mode 100644 src/tools/myriadfs.cpp create mode 100644 src/unstable/myriadcache.cpp create mode 100644 src/unstable/myriadcache.h create mode 100644 src/unstable/myriadfs.cpp create mode 100644 src/unstable/myriadfs.h diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp index 1bf2301..0bd2ff2 100644 --- a/src/stable/myriad.cpp +++ b/src/stable/myriad.cpp @@ -74,7 +74,7 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode, mhStream.unlock(); bStructureChanged = true; - return Bu::MyriadStream( *this, pStream, (Mode)(eMode&ReadWrite) ); + return Bu::MyriadStream( *this, pStream, eMode&ReadWrite ); } Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, @@ -144,6 +144,18 @@ void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream, pStream->setSize( iNewSize ); } +int32_t Bu::Myriad::getSize( StreamId iStream ) const +{ + Bu::MutexLocker l( mhStream ); + return hStream.get( iStream )->getSize(); +} + +bool Bu::Myriad::exists( StreamId iStream ) const +{ + Bu::MutexLocker l( mhStream ); + return hStream.has( iStream ); +} + Bu::String Bu::Myriad::getLocation() const { Bu::MutexLocker l( mAccess ); @@ -218,6 +230,11 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const return iTotal; } +void Bu::Myriad::sync() +{ + writeHeader(); +} + bool Bu::Myriad::loadMyriad() { Bu::println("Load myriad!"); @@ -417,12 +434,12 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ) void Bu::Myriad::writeHeader() { + Bu::MutexLocker l( mAccess ); if( !rBacking.isWritable() ) return; Bu::println("Writing stream breakdown:"); Bu::MemBuf mbHeader; { - Bu::MutexLocker l( mAccess ); Bu::MutexLocker l2( mhStream ); int32_t iHdrStreamSize = __calcHeaderSize(); diff --git a/src/stable/myriad.h b/src/stable/myriad.h index 58f700a..c3f682d 100644 --- a/src/stable/myriad.h +++ b/src/stable/myriad.h @@ -32,7 +32,7 @@ namespace Bu { public: typedef int32_t StreamId; - enum Mode { + enum Mode : int32_t { None = 0x00, // Flags @@ -75,6 +75,8 @@ namespace Bu MyriadStream open( StreamId iStream, Mode eMode ); void erase( StreamId iStream ); void setSize( StreamId iStream, int32_t iNewSize ); + int32_t getSize( StreamId iStream ) const; + bool exists( StreamId iStream ) const; Bu::String getLocation() const; int32_t getBlockSize() const; int32_t getTotalBlocks() const; @@ -84,6 +86,12 @@ namespace Bu int32_t getTotalUsedBytes() const; int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const; + /** + * Flush all caches to the backing stream, write all structural and + * header changes. + */ + void sync(); + private: bool loadMyriad(); void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks ); @@ -175,6 +183,20 @@ namespace Bu IndexList lFreeBlocks; StreamId iLastUsedIndex; }; + constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b ) + { + return static_cast( + static_cast::type>(a) & + static_cast::type>(b) + ); + } + constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b ) + { + return static_cast( + static_cast::type>(a) | + static_cast::type>(b) + ); + } }; #endif diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp index 9ea2e17..eaf91a5 100644 --- a/src/stable/myriadstream.cpp +++ b/src/stable/myriadstream.cpp @@ -166,3 +166,9 @@ Bu::String Bu::MyriadStream::getLocation() const return pStream->getLocation(); } +Bu::Myriad::StreamId Bu::MyriadStream::getId() const +{ + Bu::MutexLocker l( mAccess ); + return pStream->getStreamId(); +} + diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h index b86dbd7..27a15d5 100644 --- a/src/stable/myriadstream.h +++ b/src/stable/myriadstream.h @@ -39,6 +39,7 @@ namespace Bu virtual size getSize() const; virtual size getBlockSize() const; virtual Bu::String getLocation() const; + Myriad::StreamId getId() const; private: mutable Bu::Mutex mAccess; diff --git a/src/tools/myriadfs.cpp b/src/tools/myriadfs.cpp new file mode 100644 index 0000000..587bc89 --- /dev/null +++ b/src/tools/myriadfs.cpp @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/sio.h" +#include "bu/streamstack.h" +#include "bu/file.h" +#include "bu/myriadfs.h" +#include "bu/myriadstream.h" +#include "bu/optparser.h" + +enum Mode +{ + modeList, + modeCat, + modeCopyIn, + modeCopyOut, + modeMkdir, + modeInitialize, + modeRm, + + modeNone +}; + +Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ ) +{ + Bu::sio << "Uh oh, the formatter was called..." << Bu::sio.nl; + return f; +} + +class Options : public Bu::OptParser +{ +public: + Options( int argc, char *argv[] ) : + eMode( modeNone ), + iBlockSize( 64 ) + { + addHelpBanner("Options:"); + addOption( sFile, 'f', "file", "Myriadfs file"); + addOption( iBlockSize, 'b', "block-size", + "Specify the block size when initializing a new MyriadFs"); + + setNonOption( Bu::slot( this, &Options::nonOption ) ); + + addHelpOption(); + + parse( argc, argv ); + } + + int nonOption( Bu::Array aParams ) + { + if( eMode == modeNone ) + { + //Bu::println("Checking mode"); + // First param, must be the mode + if( aParams[0] == "ls" ) + eMode = modeList; + else if( aParams[0] == "cat" ) + eMode = modeCat; + else if( aParams[0] == "cp-in" ) + eMode = modeCopyIn; + else if( aParams[0] == "cp-out" ) + eMode = modeCopyOut; + else if( aParams[0] == "mkdir" ) + eMode = modeMkdir; + else if( aParams[0] == "initialize" ) + eMode = modeInitialize; + else if( aParams[0] == "rm" ) + eMode = modeRm; + else + Bu::println("Unknown command, try --help"); + return 0; + } else { + lParams.append( aParams[0] ); + } + //Bu::println("Got option: \"%1\"").arg( aParams[0] ); + return 0; + } + + Mode eMode; + Bu::String sFile; + Bu::StringList lParams; + int iBlockSize; +}; + +int main( int argc, char *argv[] ) +{ + Options opt( argc, argv ); + + if( opt.sFile.isEmpty() ) + { + Bu::println("You must specify a MyriadFs stream (see -f).\n"); + return 1; + } + + if( opt.eMode == modeNone ) + { + Bu::println("Specify an operation to perfrom.\n"); + return 1; + } + + int iFlags = Bu::File::ReadWrite; + + if( opt.eMode == modeInitialize ) + { + iFlags |= Bu::File::Create|Bu::File::Truncate; + } + + Bu::File fFs( opt.sFile, iFlags ); + Bu::MyriadFs mFs( fFs, opt.iBlockSize ); + + switch( opt.eMode ) + { + case modeList: + { + Bu::String sPath = "/"; + if( opt.lParams.getSize() > 0 ) + { + sPath = opt.lParams.first(); + } + Bu::MyriadFs::Dir lEnt = mFs.readDir( sPath ); + for( Bu::MyriadFs::Dir::iterator i = lEnt.begin(); i; i++ ) + { + Bu::String sPerm; + switch( (*i).uPerms&Bu::MyriadFs::typeMask ) + { + case Bu::MyriadFs::typeDir: sPerm += "d"; break; + case Bu::MyriadFs::typeChrDev: sPerm += "c"; break; + case Bu::MyriadFs::typeBlkDev: sPerm += "b"; break; + case Bu::MyriadFs::typeSymLink: sPerm += "l"; break; + case Bu::MyriadFs::typeSocket: sPerm += "s"; break; + default: sPerm += "-"; break; + } + sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrR)?"r":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrW)?"w":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrX)?"x":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpR)?"r":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpW)?"w":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpX)?"x":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permOthR)?"r":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permOthW)?"w":"-"; + sPerm += ((*i).uPerms&Bu::MyriadFs::permOthX)?"x":"-"; + Bu::println("%1 %2 %3:%4 %5 %6") + .arg( sPerm ) + .arg( (*i).iNode ) + .arg( (*i).iUser ) + .arg( (*i).iGroup ) + .arg( (*i).iSize ) + .arg( (*i).sName ); + } + } + break; + + case modeCat: + { + if( opt.lParams.isEmpty() ) + { + Bu::println("Specify at least one file."); + return 1; + } + int iBlockSize = 1024*1024; + char *pBuf = new char[iBlockSize]; + for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ ) + { + Bu::MyriadStream ms = mFs.open( *i, Bu::MyriadFs::Read ); + int iRead = 0; + do + { + iRead = ms.read( pBuf, iBlockSize ); + if( iRead > 0 ) + { + Bu::sioRaw.write( pBuf, iRead ); + } + } while( iRead == iBlockSize ); + } + delete[] pBuf; + } + break; + + case modeCopyIn: + { + if( opt.lParams.getSize() != 2 ) + { + Bu::println("Specify a source file and destination file."); + return 1; + } + int iBlockSize = 1024*1024; + char *pBuf = new char[iBlockSize]; + Bu::File fs = Bu::File( + opt.lParams.first(), Bu::File::Read ); + Bu::MyriadStream ms = mFs.open( + opt.lParams.last(), Bu::MyriadFs::WriteNew ); + int iRead = 0; + do + { + iRead = fs.read( pBuf, iBlockSize ); + if( iRead > 0 ) + { + ms.write( pBuf, iRead ); + } + } while( iRead == iBlockSize ); + delete[] pBuf; + } + break; + + case modeCopyOut: + { + if( opt.lParams.getSize() != 2 ) + { + Bu::println("Specify a source file and destination file."); + return 1; + } + int iBlockSize = 1024*1024; + char *pBuf = new char[iBlockSize]; + Bu::MyriadStream ms = mFs.open( + opt.lParams.first(), Bu::MyriadFs::Read ); + Bu::File fs = Bu::File( + opt.lParams.last(), Bu::File::WriteNew ); + int iRead = 0; + do + { + iRead = ms.read( pBuf, iBlockSize ); + if( iRead > 0 ) + { + fs.write( pBuf, iRead ); + } + } while( iRead == iBlockSize ); + delete[] pBuf; + } + break; + + case modeMkdir: + { + if( opt.lParams.isEmpty() ) + { + Bu::println("Specify at least one directory."); + return 1; + } + for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ ) + { + mFs.mkDir( *i, 0777 ); + } + } + break; + + case modeInitialize: + Bu::println("MyriadFs initialized.\n"); + break; + + case modeRm: + { + if( opt.lParams.getSize() != 1 ) + { + Bu::println("Specify a file path."); + return 1; + } + mFs.unlink( opt.lParams.first() ); + } + break; + + case modeNone: + break; + } + + return 0; +} diff --git a/src/unstable/myriadcache.cpp b/src/unstable/myriadcache.cpp new file mode 100644 index 0000000..c9eb9c4 --- /dev/null +++ b/src/unstable/myriadcache.cpp @@ -0,0 +1,8 @@ +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/myriadcache.h" diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h new file mode 100644 index 0000000..d6842a5 --- /dev/null +++ b/src/unstable/myriadcache.h @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#ifndef BU_MYRIAD_CACHE_H +#define BU_MYRIAD_CACHE_H + +#include "bu/cachebase.h" +#include "bu/myriad.h" +#include "bu/myriadstream.h" +#include "bu/file.h" +#include "bu/streamstack.h" + +namespace Bu +{ + template + class MyriadCache : public Bu::CacheBase + { + public: + MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) : + sStore( sStore ), + mStore( sStore, iBlockSize, iPreallocate ), + bStructureChanged( false ) + { + try + { + Bu::ReadWriteMutex::ReadLocker l( rwStore ); + Bu::MyriadStream ms = mStore.open( + 1, Bu::Myriad::WriteNew|Bu::Myriad::Read + ); + Bu::Archive ar( ms, Bu::Archive::load ); + uint8_t uVer; + ar >> uVer; + switch( uVer ) + { + case 0: + ar >> hIndex; + break; + } + } + catch(...) + { + try + { + mStore.open( 1, Bu::Myriad::Create|Bu::Myriad::ReadWrite ); + _sync(); + } + catch(...) + { + throw Bu::ExceptionBase("Error creating index stream."); + } + } + } + + virtual ~MyriadCache() + { + Bu::CacheBase::sync(); + } + + using typename Bu::CacheBase::KeyList; + using typename Bu::CacheBase::ObjectType; + + virtual typename Bu::CacheBase::KeyList getKeys() const + { + Bu::ReadWriteMutex::ReadLocker rl( rwStore ); + return hIndex.getKeys(); + } + + virtual int getSize() const + { + Bu::ReadWriteMutex::ReadLocker rl( rwStore ); + return hIndex.getSize(); + } + + protected: + virtual bool _has( const keytype &key ) + { + Bu::ReadWriteMutex::ReadLocker rl( rwStore ); + return hIndex.has( key ); + } + + virtual void _create( const obtype *o ) + { + Bu::ReadWriteMutex::WriteLocker wl( rwStore ); + { + Bu::MyriadStream ms = mStore.create( Bu::Myriad::Create ); + hIndex.insert( o->getKey(), ms.getId() ); + } + _save( o ); + + bStructureChanged = true; + } + + virtual void _erase( const keytype &k ) + { + Bu::ReadWriteMutex::WriteLocker wl( rwStore ); + mStore.erase( hIndex.get( k ) ); + hIndex.erase( k ); + + bStructureChanged = true; + } + + virtual obtype *_load( + typename Bu::CacheObject::Initializer &initObj, + const keytype &k + ) + { + Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) ); + return _cacheObjectLoad( initObj, k, ms ); + } + + virtual void _save( const obtype *o ) + { + Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) ); + _cacheObjectSave( ms, o ); + ms.setSize( ms.tell() ); + + mStore.sync(); + } + + virtual void _sync() + { + Bu::ReadWriteMutex::WriteLocker wl( rwStore ); + if( !bStructureChanged ) + return; + + Bu::MyriadStream ms = mStore.openStream( 1 ); + Bu::Archive ar( ms, Bu::Archive::save ); + ar << (uint8_t)0 << hIndex; + ar.close(); + ms.setSize( ms.tell() ); + + bStructureChanged = false; + + mStore.sync(); + } + + private: + Bu::Stream &sStore; + Bu::Myriad mStore; + Bu::Hash hIndex; + mutable Bu::ReadWriteMutex rwStore; + bool bStructureChanged; + }; +} + +#endif diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp new file mode 100644 index 0000000..ab9ca74 --- /dev/null +++ b/src/unstable/myriadfs.cpp @@ -0,0 +1,737 @@ +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/config.h" +#include "bu/myriadfs.h" +#include "bu/myriadstream.h" + +#include +#include +#include + +#include "bu/sio.h" +using Bu::sio; +using Bu::Fmt; + +namespace Bu { subExceptionDef( MyriadFsException ) } + +#define Myriad_Fs_MAGIC_CODE ((char *)"\xa7\x18\x8b\x39") + +Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) : + rStore( rStore ), + mStore( rStore, iBlockSize ), + iUser( 0 ), + iGroup( 0 ) +{ +#ifndef WIN32 + iUser = getuid(); + iGroup = getgid(); +#endif + + if( mStore.exists( 1 ) ) + { + // Check to see if this is a MyriadFs stream. + Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Read ); + char sMagic[4]; + if( ms.read( sMagic, 4 ) < 4 ) + throw MyriadFsException("The provided stream does not appear to be " + "a MyriadFs stream."); + if( ::strncmp( sMagic, Myriad_Fs_MAGIC_CODE, 4 ) ) + throw MyriadFsException("The provided stream does not appear to be " + "a MyriadFs stream."); + + int8_t iVer; + ms.read( &iVer, 1 ); + + int32_t iNumNodes; + ms.read( &iNumNodes, 4 ); + for( int32_t j = 0; j < iNumNodes; j++ ) + { + int32_t iNode; + int32_t iPos; + ms.read( &iNode, 4 ); + ms.read( &iPos, 4 ); + hNodeIndex.insert( iNode, iPos ); + } + } + else + { + // Create initial header stream + { + Bu::MyriadStream ms = mStore.open( + 1, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive ); + ms.write( Myriad_Fs_MAGIC_CODE, 4 ); + int8_t iVer = 1; + int32_t iTmp = 1; + ms.write( &iVer, 1 ); + ms.write( &iTmp, 4 ); // iNumNodes + iTmp = 0; + ms.write( &iTmp, 4 ); // iInode + ms.write( &iTmp, 4 ); // iPosition + hNodeIndex.insert( 0, 0 ); + } + + // Create initial inode stream, with one root node. + { + Bu::MyriadStream ms = mStore.open( + 2, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive ); + RawStat rs; + rs.iNode = 0; + rs.iUser = iUser; + rs.iGroup = iGroup; + rs.uPerms = 0755|typeDir; + rs.iLinks = 1; + rs.uStreamIndex = 3; + rs.iCTime = rs.iMTime = rs.iATime = time(NULL); + ms.write( &rs, sizeof(RawStat) ); + } + + // Create inode 0's storage stream. + { + Bu::MyriadStream ms = mStore.open( + 3, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive ); + int32_t iTmp32 = 0; + ms.write( &iTmp32, 4 ); // iChildCount + } + } +} + +Bu::MyriadFs::~MyriadFs() +{ + writeHeader(); +} + +void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf ) +{ + int32_t iParent; + int32_t iNode = lookupInode( sPath, iParent ); + Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read ); + stat( iNode, rBuf, is ); +} + +Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode, + uint16_t uPerms ) +{ + int32_t iParent = -1; + int32_t iNode; + try + { + iNode = lookupInode( sPath, iParent ); +// sio << "File found." << sio.nl; + // The file was found + Bu::MyriadStream ms = openByInode( iNode ); + if( (iMode&Truncate) ) + { + ms.setSize( 0 ); + } + return ms; + } + catch( Bu::MyriadFsException &e ) + { + if( iParent < 0 ) + throw; + + // This means that an intermediate path component couldn't be found + if( e.getErrorCode() == 1 ) + throw; + + // The file wasn't found, but the path leading up to it was. + // first, figure out the final path element... + Bu::String sName = filePart( sPath ); +// sio << "End filename: " << sName << sio.nl; +// sio << "Parent inode: " << iParent << sio.nl; + iNode = create( iParent, sName, (uPerms&permMask)|typeRegFile, 0 ); +// sio << "New iNode: " << iNode << sio.nl; + return openByInode( iNode ); + } +} + +void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms ) +{ + create( sPath, iPerms, 0 ); +} + +void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms, + uint16_t iDevHi, uint16_t iDevLo ) +{ + create( sPath, iPerms, ((uint32_t)iDevHi<<16)|(uint32_t)iDevLo ); +} + +void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms, + uint32_t uSpecial ) +{ + int32_t iParent = -1; +// int32_t iNode; + try + { + /*iNode =*/ lookupInode( sPath, iParent ); +// sio << "File found." << sio.nl; + } + catch( Bu::MyriadFsException &e ) + { + if( iParent < 0 ) + throw; + + // This means that an intermediate path component couldn't be found + if( e.getErrorCode() == 1 ) + throw; + + // The file wasn't found, but the path leading up to it was. + // first, figure out the final path element... + Bu::String sName = filePart( sPath ); +// sio << "End filename: " << sName << sio.nl; +// sio << "Parent inode: " << iParent << sio.nl; + /*iNode =*/ create( iParent, sName, iPerms, uSpecial ); +// sio << "New iNode: " << iNode << sio.nl; + } + // The file was found + //throw Bu::MyriadFsException("Path already exists."); +} + +void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms ) +{ + create( sPath, (iPerms&permMask)|typeDir, 0 ); +} + +void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget, + const Bu::String &sPath ) +{ + int32_t iParent = -1; + int32_t iNode; + try + { + iNode = lookupInode( sPath, iParent ); + } + catch( Bu::MyriadFsException &e ) + { + if( iParent < 0 ) + throw; + + // This means that an intermediate path component couldn't be found + if( e.getErrorCode() == 1 ) + throw; + + // The file wasn't found, but the path leading up to it was. + // first, figure out the final path element... + Bu::String sName = filePart( sPath ); +// sio << "End filename: " << sName << sio.nl; +// sio << "Parent inode: " << iParent << sio.nl; + iNode = create( iParent, sName, 0777|typeSymLink, 0 ); +// sio << "New iNode: " << iNode << sio.nl; + MyriadStream ms = openByInode( iNode ); + ms.write( sTarget ); + return; + } + throw Bu::MyriadFsException("Path already exists."); +} + +void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget, + const Bu::String &sPath ) +{ + int32_t iParent = -1; + int32_t iNode; + + iNode = lookupInode( sTarget, iParent ); + + try + { + lookupInode( sPath, iParent ); + throw Bu::MyriadFsException("Path already exists."); + } + catch( Bu::MyriadFsException &e ) + { + if( iParent < 0 ) + throw; + + // This means that an intermediate path component couldn't be found + if( e.getErrorCode() == 1 ) + throw; + + // The file wasn't found, but the path leading up to it was. + // first, figure out the final path element... + Bu::String sName = filePart( sPath ); +// sio << "End filename: " << sName << sio.nl; +// sio << "Parent inode: " << iParent << sio.nl; + addToDir( iParent, iNode, sName ); + MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite ); + RawStat rs; + readInode( iNode, rs, is ); + rs.iLinks++; + writeInode( rs, is ); + } +} + +Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath ) +{ + int32_t iParent = -1; + int32_t iNode; + iNode = lookupInode( sPath, iParent ); + MyriadStream ms = openByInode( iNode ); + Bu::String sRet; + sRet.setSize( ms.getSize() ); + ms.read( sRet.getStr(), ms.getSize() ); + return sRet; +} + +Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath ) +{ + int32_t iParent = -1; + int32_t iNode = lookupInode( sPath, iParent ); + return readDir( iNode ); +} + +void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime, + int64_t iMTime ) +{ + int32_t iParent = -1; + int32_t iNode; + + iNode = lookupInode( sPath, iParent ); + + setTimes( iNode, iATime, iMTime ); +} + +void Bu::MyriadFs::unlink( const Bu::String &sPath ) +{ + int32_t iParent = -1; +// int32_t iNode; + + /*iNode =*/ lookupInode( sPath, iParent ); + + Dir lDir = readDir( iParent ); + + Bu::String sName = filePart( sPath ); + + for( Dir::iterator i = lDir.begin(); i; i++ ) + { + if( sName == (*i).sName ) + { + RawStat rs; + readInode( (*i).iNode, rs ); + if( (rs.uPerms&typeMask) == typeDir ) + { + MyriadStream msDir = mStore.open( + rs.uStreamIndex, Bu::Myriad::Read + ); + int32_t iCount; + msDir.read( &iCount, 4 ); + if( iCount > 0 ) + { + throw Bu::MyriadFsException("Directory not empty."); + } + } + if( --rs.iLinks == 0 ) + { + destroyNode( (*i).iNode ); + } + else + { + writeInode( rs ); + } + lDir.erase( i ); + break; + } + } + + Bu::MyriadStream ms = openByInode( iParent ); + int32_t iNumChildren = lDir.getSize(); + ms.write( &iNumChildren, 4 ); + for( Dir::iterator i = lDir.begin(); i; i++ ) + { + ms.write( &(*i).iNode, 4 ); + uint8_t iSize = (*i).sName.getSize(); + ms.write( &iSize, 1 ); + ms.write( (*i).sName.getStr(), iSize ); + } + ms.setSize( ms.tell() ); +} + +void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize ) +{ + int32_t iParent = -1; + int32_t iNode; + iNode = lookupInode( sPath, iParent ); + MyriadStream ms = openByInode( iNode ); + ms.setSize( iSize ); +} + +void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo ) +{ + mkHardLink( sFrom, sTo ); + unlink( sFrom ); +} + +dev_t Bu::MyriadFs::devToSys( uint32_t uDev ) +{ + return (((uDev&0xFFFF0000)>>8)&0xFF00) | ((uDev&0xFF)); +} + +uint32_t Bu::MyriadFs::sysToDev( dev_t uDev ) +{ + return (((uint32_t)uDev&0xFF00)<<8) | ((uint32_t)uDev&0xFF); +} + +int32_t Bu::MyriadFs::lookupInode( const Bu::String &sPath, int32_t &iParent ) +{ + if( sPath == "/" ) + { + return 0; + } + if( sPath[0] == '/' ) + { + // Absolute lookup + return lookupInode( sPath.begin()+1, 0, iParent ); + } + else + { + // Relative lookup + throw Bu::ExceptionBase( + "Relative lookups in MyriadFs are not working yet."); + } +} + +int32_t Bu::MyriadFs::lookupInode( Bu::String::const_iterator iStart, + int32_t iNode, int32_t &iParent ) +{ + iParent = iNode; + + Bu::String::const_iterator iEnd = iStart.find('/'); + Bu::String sTok( iStart, iEnd ); + +// sio << "Direcotry component: " << sTok << sio.nl; + + Dir lDir = readDir( iNode ); + + for( Dir::iterator i = lDir.begin(); i; i++ ) + { + if( (*i).sName == sTok ) + { + // We found an item + if( !iEnd ) + { + // It's the last one in the requested path, return it + return (*i).iNode; + } + else + { + // Not the last one in our path, double check it's a dir + if( ((*i).uPerms&typeMask) == typeDir ) + { + return lookupInode( iEnd+1, (*i).iNode, iParent ); + } + else + { + iParent = -1; + throw Bu::MyriadFsException( + "Element '%s' in given path is not a directory.", + sTok.getStr() ); + } + } + } + } + + if( iEnd ) + throw Bu::MyriadFsException( 1, "Path not found"); + else + throw Bu::MyriadFsException( 2, "Path not found"); +} + +void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs ) +{ + rIs.setPos( hNodeIndex.get( iNode )*sizeof(RawStat) ); + if( rIs.read( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) ) + throw Bu::MyriadFsException("Filesystem corruption detected."); + if( rs.iNode != iNode ) + throw Bu::MyriadFsException("Filesystem corruption detected."); +} + +void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs ) +{ + MyriadStream ms = mStore.open( 2, Bu::Myriad::Read ); + readInode( iNode, rs, ms ); +} + +void Bu::MyriadFs::writeInode( const RawStat &rs, + MyriadStream &rOs ) +{ + rOs.setSize( hNodeIndex.getSize()*sizeof(RawStat) ); + rOs.setPos( hNodeIndex.get( rs.iNode )*sizeof(RawStat) ); + if( rOs.write( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) ) + throw Bu::MyriadFsException("Error writing inode to header stream."); +} + +void Bu::MyriadFs::writeInode( const RawStat &rs ) +{ + MyriadStream ms = mStore.open( 2, Bu::Myriad::Write ); + writeInode( rs, ms ); +} + +Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode ) +{ + Bu::MyriadStream ms = openByInode( iNode ); + int32_t iNumChildren = 0; + ms.read( &iNumChildren, 4 ); + + Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read ); + Dir lDir; + // sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl; + for( int32_t j = 0; j < iNumChildren; j++ ) + { + int32_t iChildNode = 0; + if( ms.read( &iChildNode, 4 ) < 4 ) + { + throw Bu::MyriadFsException( + "Failed to read iChildNode from directory."); + } + Stat s; + stat( iChildNode, s, is ); + uint8_t uLen; + if( ms.read( &uLen, 1 ) < 1 ) + { + throw Bu::MyriadFsException( + "Failed to read uLen from directory."); + } + s.sName.setSize( uLen ); + if( ms.read( s.sName.getStr(), uLen ) < uLen ) + { + throw Bu::MyriadFsException( + "Failed to read sName from directory."); + } + lDir.append( s ); + +// sio << " " << s.sName << sio.nl; + } + + return lDir; +} + +Bu::MyriadStream Bu::MyriadFs::openByInode( int32_t iNode ) +{ + RawStat rs; + readInode( iNode, rs ); + switch( (rs.uPerms&typeMask) ) + { + case typeDir: + case typeSymLink: + case typeRegFile: + return mStore.open( rs.uStreamIndex, Bu::Myriad::ReadWrite ); + + default: + throw Bu::MyriadFsException( + "inode incorrect type for low-level openByInode."); + } +} + +void Bu::MyriadFs::addToDir( int32_t iDir, int32_t iNode, + const Bu::String &sName ) +{ + if( sName.getSize() > 255 ) + { + throw Bu::MyriadFsException("Filename too long, max is 255 bytes."); + } + Bu::MyriadStream ms = openByInode( iDir ); + int32_t iNumChildren = 0; + ms.read( &iNumChildren, 4 ); + iNumChildren++; + ms.setPos( 0 ); + ms.write( &iNumChildren, 4 ); + ms.setPosEnd( 0 ); + ms.write( &iNode, 4 ); + uint8_t uLen = sName.getSize(); + ms.write( &uLen, 1 ); + ms.write( sName.getStr(), uLen ); +} + +int32_t Bu::MyriadFs::create( int32_t iParent, const Bu::String &sName, + uint16_t uPerms, uint32_t uSpecial ) +{ + int32_t iNode = allocInode( uPerms, uSpecial ); + addToDir( iParent, iNode, sName ); + return iNode; +} + +int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial ) +{ + int32_t iNode = 0; + for(; iNode < 0xfffffff; iNode++ ) + { + if( !hNodeIndex.has( iNode ) ) + { + hNodeIndex.insert( iNode, hNodeIndex.getSize() ); + RawStat rs; + rs.iNode = iNode; + rs.iUser = iUser; + rs.iGroup = iGroup; + rs.uPerms = uPerms; + rs.iLinks = 1; + switch( (uPerms&typeMask) ) + { + case typeRegFile: + case typeSymLink: + { + Bu::MyriadStream ms = mStore.create( + Bu::Myriad::Create + ); + rs.uStreamIndex = ms.getId(); + } + break; + + case typeDir: + { + Bu::MyriadStream ms = mStore.create( + Bu::Myriad::Create + ); + rs.uStreamIndex = ms.getId(); + } +// sio << "Creating directory node, storage: " +// << rs.uStreamIndex << sio.nl; + { + Bu::MyriadStream msDir = mStore.open( + rs.uStreamIndex, Bu::Myriad::Write + ); + uint32_t uSize = 0; + msDir.write( &uSize, 4 ); + } + break; + + case typeChrDev: + case typeBlkDev: + rs.uStreamIndex = uSpecial; + break; + + default: + rs.uStreamIndex = 0; + break; + } + rs.iATime = time(NULL); + rs.iMTime = time(NULL); + rs.iCTime = time(NULL); + writeInode( rs ); + + return iNode; + } + } + + throw Bu::MyriadFsException( + "No inode could be allocated. You've run out!"); +} + +void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs ) +{ + RawStat rs; + readInode( iNode, rs, rIs ); + rBuf.iNode = iNode; + rBuf.iUser = rs.iUser; + rBuf.iGroup = rs.iGroup; + rBuf.uPerms = rs.uPerms; + rBuf.iLinks = rs.iLinks; + rBuf.iATime = rs.iATime; + rBuf.iMTime = rs.iMTime; + rBuf.iCTime = rs.iCTime; + rBuf.uDev = 0; + rBuf.iSize = 0; + switch( (rBuf.uPerms&typeMask) ) + { + case typeRegFile: + case typeSymLink: + rBuf.iSize = mStore.getSize( rs.uStreamIndex ); + break; + + case typeChrDev: + case typeBlkDev: + rBuf.uDev = rs.uStreamIndex; + break; + + default: + rBuf.iSize = 0; + break; + } +} + +void Bu::MyriadFs::writeHeader() +{ + Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Write ); + ms.write( Myriad_Fs_MAGIC_CODE, 4 ); + int8_t iVer = 1; + int32_t iNumNodes = hNodeIndex.getSize(); + ms.write( &iVer, 1 ); + ms.write( &iNumNodes, 4 ); // iNumNodes + for( NodeIndex::iterator i = hNodeIndex.begin(); i; i++ ) + { + int32_t iNode = i.getKey(); + int32_t iPosition = i.getValue(); + ms.write( &iNode, 4 ); + ms.write( &iPosition, 4 ); + } + + // Truncate the stream afterwards so we don't use up too much space. + ms.setSize( ms.tell() ); +} + +void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ) +{ + RawStat rs; + Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite ); + + readInode( iNode, rs, is ); + rs.iATime = iATime; + rs.iMTime = iMTime; + writeInode( rs, is ); +} + +void Bu::MyriadFs::destroyNode( int32_t iNode ) +{ + if( iNode == 0 ) + throw Bu::MyriadFsException("You cannot destroy the root."); + + uint32_t iPosition; + RawStat rsOld; + + Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite ); + + // This will be overwritten with the last node + iPosition = hNodeIndex.get( iNode ); + readInode( iNode, rsOld, is ); + + switch( (rsOld.uPerms&typeMask) ) + { + case typeRegFile: + case typeDir: + case typeSymLink: + mStore.erase( rsOld.uStreamIndex ); + break; + } + + hNodeIndex.erase( iNode ); + + // Read the last node, can't use the helpers, because we don't know the + // iNode yet. + if( iPosition != hNodeIndex.getSize() ) + { + // If this is the last node, then we don't need to do anything, but + // this case handles what to do if we aren't on the last node + RawStat rs; + is.setPos( (hNodeIndex.getSize())*sizeof(RawStat) ); + is.read( &rs, sizeof(RawStat) ); + + hNodeIndex.get( rs.iNode ) = iPosition; + writeInode( rs, is ); + } + + is.setSize( hNodeIndex.getSize() * sizeof(RawStat) ); +} + +Bu::String Bu::MyriadFs::filePart( const Bu::String &sPath ) +{ + Bu::String::const_iterator iStart = sPath.begin(); + if( *iStart == '/' ) + iStart++; + for( Bu::String::const_iterator iEnd = iStart.find('/'); iEnd; + iStart = iEnd+1, iEnd = iStart.find('/') ) { } + return Bu::String( iStart, sPath.end() ); +} + diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h new file mode 100644 index 0000000..ff14292 --- /dev/null +++ b/src/unstable/myriadfs.h @@ -0,0 +1,205 @@ +/* + * Copyright (C) 2007-2023 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#ifndef MYRIAD_FS_H +#define MYRIAD_FS_H + +#include + +#include "bu/myriad.h" +#include "bu/readwritemutex.h" + +namespace Bu +{ + class Stream; + + subExceptionDecl( MyriadFsException ); + + /** + * A POSIX compliant, node based filesystem built on top of Myriad. + * + * A header is placed into stream 1. + * Header format: + * int32_t iMagicHeader (A7188B39) + * int8_t iVersion (1) + * int32_t iNumNodes + * NodeLookup[iNumNodes] nNode + * + * Node lookup: + * int32_t iInode + * int32_t iPosition + * + * The node headers or inode structures have a base size of 44 bytes. + * The name is stored in the directory format. + * Basic node header format: + * int32_t iNode + * int32_t iUser + * int32_t iGroup + * uint16_t uPerms + * int16_t iLinks + * uint32_t uStreamIndex + * int64_t iATime + * int64_t iMTime + * int64_t iCTime + * + * Some types get special formats for their assosiated data stream, or + * other special considerations, here's a list: + * + * - typeFifo: No stream, uStreamIndex unused (probably) + * - typeChrDev: No stream, uStreamIndex is device hi/lo + * - typeDir: The stream contains a directory contents listing, described + * below + * - typeBlkDev: No stream, uStreamIndex is device hi/lo + * - typeRegFile: The stream is the file data + * - typeSymLink: The stream is the destination of the symlink + * - typeSocket: No steram, uStreamIndex unused (probably) + * + * Directory streams have this simple listing format. They contain a list + * of all child elements, with no particular order at the moment. The . and + * .. entries are not listed, they are implicit: + * int32_t iNumNodes + * NodeTable[iNumNodes] nChildren + * + * NodeTable: + * int32_t iInode + * uint8_t uNameSize + * char[uNameSize] sName + */ + class MyriadFs + { + public: + MyriadFs( Bu::Stream &rStore, int iBlockSize=512 ); + virtual ~MyriadFs(); + + enum + { + permOthX = 0000001, + permOthW = 0000002, + permOthR = 0000004, + permGrpX = 0000010, + permGrpW = 0000020, + permGrpR = 0000040, + permUsrX = 0000100, + permUsrW = 0000200, + permUsrR = 0000400, + permSticky = 0001000, + permSetGid = 0002000, + permSetUid = 0004000, + permMask = 0007777, + typeFifo = 0010000, + typeChrDev = 0020000, + typeDir = 0040000, + typeBlkDev = 0060000, + typeRegFile = 0100000, + typeSymLink = 0120000, + typeSocket = 0140000, + typeMask = 0170000 + }; + + enum + { + Read = 0x01, ///< Open file for reading + Write = 0x02, ///< Open file for writing + Create = 0x04, ///< Create file if it doesn't exist + Truncate = 0x08, ///< Truncate file if it does exist + Append = 0x10, ///< Always append on every write + NonBlock = 0x20, ///< Open file in non-blocking mode + Exclusive = 0x44, ///< Create file, if it exists then fail + + // Helpful mixes + ReadWrite = 0x03, ///< Open for reading and writing + WriteNew = 0x0E ///< Create a file (or truncate) for writing. + /// Same as Write|Create|Truncate + }; + + class Stat + { + public: + int32_t iNode; + int32_t iUser; + int32_t iGroup; + uint16_t uPerms; + int16_t iLinks; + int64_t iATime; + int64_t iMTime; + int64_t iCTime; + int32_t iSize; + uint32_t uDev; + Bu::String sName; + }; + typedef Bu::List Dir; + + void stat( const Bu::String &sPath, Stat &rBuf ); + MyriadStream open( const Bu::String &sPath, int iMode, + uint16_t uPerms=0664 ); + void create( const Bu::String &sPath, uint16_t iPerms ); + void create( const Bu::String &sPath, uint16_t iPerms, + uint16_t iDevHi, uint16_t iDevLo ); + void create( const Bu::String &sPath, uint16_t iPerms, + uint32_t uSpecial ); + void mkDir( const Bu::String &sPath, uint16_t iPerms ); + void mkSymLink( const Bu::String &sTarget, const Bu::String &sPath ); + void mkHardLink( const Bu::String &sTarget, const Bu::String &sPath ); + Bu::String readSymLink( const Bu::String &sPath ); + Dir readDir( const Bu::String &sPath ); + void setTimes( const Bu::String &sPath, int64_t iATime, + int64_t iMTime ); + void unlink( const Bu::String &sPath ); + void setFileSize( const Bu::String &sPath, int32_t iSize ); + void rename( const Bu::String &sFrom, const Bu::String &sTo ); + + static dev_t devToSys( uint32_t uDev ); + static uint32_t sysToDev( dev_t uDev ); + + private: + class RawStat + { + public: + int32_t iNode; + int32_t iUser; + int32_t iGroup; + uint16_t uPerms; + int16_t iLinks; + uint32_t uStreamIndex; + int64_t iATime; + int64_t iMTime; + int64_t iCTime; + }; + typedef Bu::Hash NodeIndex; + + private: + int32_t lookupInode( const Bu::String &sPath, int32_t &iParent ); + int32_t lookupInode( Bu::String::const_iterator iStart, + int32_t iNode, int32_t &iParent ); + void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs ); + void readInode( int32_t iNode, RawStat &rs ); + void writeInode( const RawStat &rs ); + void writeInode( const RawStat &rs, MyriadStream &rOs ); + Dir readDir( int32_t iNode ); + MyriadStream openByInode( int32_t iNode ); + void addToDir( int32_t iDir, int32_t iNode, const Bu::String &sName ); + int32_t create( int32_t iParent, const Bu::String &sName, + uint16_t uPerms, uint32_t uSpecial ); + int32_t allocInode( uint16_t uPerms, uint32_t uSpecial ); + void stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs ); + void writeHeader(); + void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ); + void destroyNode( int32_t iNode ); + + Bu::String filePart( const Bu::String &sPath ); + + private: + Bu::Stream &rStore; + Bu::Myriad mStore; + Bu::ReadWriteMutex mNodeIndex; + NodeIndex hNodeIndex; + int32_t iUser; + int32_t iGroup; + }; +}; + +#endif -- cgit v1.2.3