aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/experimental/debugmutex.cpp78
-rw-r--r--src/experimental/debugmutex.h102
-rw-r--r--src/stable/file.cpp22
-rw-r--r--src/stable/mutex.h8
-rw-r--r--src/stable/myriad.cpp1226
-rw-r--r--src/stable/myriad.h347
-rw-r--r--src/stable/myriadstream.cpp284
-rw-r--r--src/stable/myriadstream.h46
-rw-r--r--src/tests/bigmyriad.cpp15
-rw-r--r--src/tests/cachedel.cpp27
-rw-r--r--src/tests/myriad.cpp23
-rw-r--r--src/tests/myriadfs.cpp2
-rw-r--r--src/tools/myriad.cpp97
-rw-r--r--src/unit/myriad.unit46
-rw-r--r--src/unstable/bitstring.cpp12
-rw-r--r--src/unstable/bitstring.h10
-rw-r--r--src/unstable/cachebase.h10
-rw-r--r--src/unstable/myriadcache.h28
-rw-r--r--src/unstable/myriadfs.cpp76
-rw-r--r--src/unstable/myriadfs.h14
20 files changed, 1367 insertions, 1106 deletions
diff --git a/src/experimental/debugmutex.cpp b/src/experimental/debugmutex.cpp
new file mode 100644
index 0000000..2b61ae2
--- /dev/null
+++ b/src/experimental/debugmutex.cpp
@@ -0,0 +1,78 @@
1#include "bu/debugmutex.h"
2
3#include "bu/exceptionbase.h"
4
5Bu::DebugMutex::DebugMutex()
6{
7}
8
9Bu::DebugMutex::~DebugMutex()
10{
11}
12
13int Bu::DebugMutex::lock()
14{
15 pthread_t self = pthread_self();
16 mState.lock();
17 bool bFound = false;
18 for( ThreadList::iterator i = lThreads.begin(); i; i++ )
19 {
20 if( (*i) == self )
21 {
22 bFound = true;
23 if( (*i).bLocked == true )
24 {
25 throw Bu::ExceptionBase( Bu::String("Double lock in thread: %1").arg( (*i).sName ).end().getStr() );
26
27 }
28 else
29 {
30 (*i).bLocked = true;
31 }
32 break;
33 }
34 }
35 if( bFound == false )
36 {
37 lThreads.append( ThreadInfo( true ) );
38 }
39 mState.unlock();
40 return Bu::Mutex::lock();
41}
42
43int Bu::DebugMutex::unlock()
44{
45 pthread_t self = pthread_self();
46 mState.lock();
47 bool bFound = false;
48 for( ThreadList::iterator i = lThreads.begin(); i; i++ )
49 {
50 if( (*i) == self )
51 {
52 bFound = true;
53 if( (*i).bLocked == false )
54 {
55 throw Bu::ExceptionBase( Bu::String("Unlock in thread that did not lock: %1").arg( (*i).sName ).end().getStr() );
56
57 }
58 else
59 {
60 (*i).bLocked = false;
61 }
62 break;
63 }
64 }
65 if( bFound == false )
66 {
67 ThreadInfo info( false );
68 throw Bu::ExceptionBase( Bu::String("Unlock in thread that never locked mutex: %1").arg( info.sName ).end().getStr() );
69 }
70 mState.unlock();
71 return Bu::Mutex::unlock();
72}
73
74int Bu::DebugMutex::trylock()
75{
76 return Bu::Mutex::trylock();
77}
78
diff --git a/src/experimental/debugmutex.h b/src/experimental/debugmutex.h
new file mode 100644
index 0000000..ca8ef9f
--- /dev/null
+++ b/src/experimental/debugmutex.h
@@ -0,0 +1,102 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_DEBUG_MUTEX_H
9#define BU_DEBUG_MUTEX_H
10
11#include "bu/mutex.h"
12#include "bu/list.h"
13#include "bu/string.h"
14
15namespace Bu
16{
17 /**
18 * Simple mutex wrapper. Currently this doesn't do anything extra for you
19 * except keep all of the functionality together in an OO sorta' way and
20 * keep you from having to worry about cleaning up your mutexes properly,
21 * or initing them.
22 *@ingroup Threading
23 */
24 class DebugMutex : public Mutex
25 {
26 public:
27 /**
28 * Create an unlocked mutex.
29 */
30 DebugMutex();
31
32 /**
33 * Destroy a mutex. This can only be done when a mutex is unlocked.
34 * Failure to unlock before destroying a mutex object could cause it to
35 * wait for the mutex to unlock, the odds of which are usually farily
36 * low at deconstruction time.
37 */
38 virtual ~DebugMutex();
39
40 /**
41 * Lock the mutex. This causes all future calls to lock on this
42 * instance of mutex to block until the first thread that called mutex
43 * unlocks it. At that point the next thread that called lock will get
44 * a chance to go to work. Because of the nature of a mutex lock it is
45 * a very bad idea to do any kind of serious or rather time consuming
46 * computation within a locked section. This can cause thread-deadlock
47 * and your program may hang.
48 */
49 virtual int lock();
50
51 /**
52 * Unlock the mutex. This allows the next thread that asked for a lock
53 * to lock the mutex and continue with execution.
54 */
55 virtual int unlock();
56
57 /**
58 * Try to lock the mutex. This is the option to go with if you cannot
59 * avoid putting lengthy operations within a locked section. trylock
60 * will attempt to lock the mutex, if the mutex is already locked this
61 * function returns immediately with an error code.
62 */
63 virtual int trylock();
64
65 private:
66 Bu::Mutex mState;
67
68 class ThreadInfo
69 {
70 public:
71 ThreadInfo( bool bLocked=false ) :
72 idThread( pthread_self() ),
73 bLocked( bLocked )
74 {
75 char buf[64];
76 if( pthread_getname_np( idThread, buf, 64 ) == 0 )
77 sName = buf;
78 }
79 ~ThreadInfo() {}
80
81 bool operator==( const ThreadInfo &rhs )
82 {
83 return pthread_equal( idThread, rhs.idThread );
84 }
85
86 bool operator==( const pthread_t &rhs )
87 {
88 return pthread_equal( idThread, rhs );
89 }
90
91 pthread_t idThread;
92 Bu::String sName;
93 bool bLocked;
94 };
95 typedef Bu::List<ThreadInfo> ThreadList;
96 ThreadList lThreads;
97 };
98}
99
100#endif
101
102
diff --git a/src/stable/file.cpp b/src/stable/file.cpp
index 05e4af1..55766ea 100644
--- a/src/stable/file.cpp
+++ b/src/stable/file.cpp
@@ -142,6 +142,16 @@ bool Bu::File::isEos()
142 142
143bool Bu::File::canRead() 143bool Bu::File::canRead()
144{ 144{
145 return isReadable();
146}
147
148bool Bu::File::canWrite()
149{
150 return isWritable();
151}
152
153bool Bu::File::isReadable()
154{
145#ifdef WIN32 155#ifdef WIN32
146 return true; 156 return true;
147#else 157#else
@@ -152,7 +162,7 @@ bool Bu::File::canRead()
152#endif 162#endif
153} 163}
154 164
155bool Bu::File::canWrite() 165bool Bu::File::isWritable()
156{ 166{
157#ifdef WIN32 167#ifdef WIN32
158 return true; 168 return true;
@@ -164,16 +174,6 @@ bool Bu::File::canWrite()
164#endif 174#endif
165} 175}
166 176
167bool Bu::File::isReadable()
168{
169 return true;
170}
171
172bool Bu::File::isWritable()
173{
174 return true;
175}
176
177bool Bu::File::isSeekable() 177bool Bu::File::isSeekable()
178{ 178{
179 return true; 179 return true;
diff --git a/src/stable/mutex.h b/src/stable/mutex.h
index d9e8910..8034974 100644
--- a/src/stable/mutex.h
+++ b/src/stable/mutex.h
@@ -33,7 +33,7 @@ namespace Bu
33 * wait for the mutex to unlock, the odds of which are usually farily 33 * wait for the mutex to unlock, the odds of which are usually farily
34 * low at deconstruction time. 34 * low at deconstruction time.
35 */ 35 */
36 ~Mutex(); 36 virtual ~Mutex();
37 37
38 /** 38 /**
39 * Lock the mutex. This causes all future calls to lock on this 39 * Lock the mutex. This causes all future calls to lock on this
@@ -44,13 +44,13 @@ namespace Bu
44 * computation within a locked section. This can cause thread-deadlock 44 * computation within a locked section. This can cause thread-deadlock
45 * and your program may hang. 45 * and your program may hang.
46 */ 46 */
47 int lock(); 47 virtual int lock();
48 48
49 /** 49 /**
50 * Unlock the mutex. This allows the next thread that asked for a lock 50 * Unlock the mutex. This allows the next thread that asked for a lock
51 * to lock the mutex and continue with execution. 51 * to lock the mutex and continue with execution.
52 */ 52 */
53 int unlock(); 53 virtual int unlock();
54 54
55 /** 55 /**
56 * Try to lock the mutex. This is the option to go with if you cannot 56 * Try to lock the mutex. This is the option to go with if you cannot
@@ -58,7 +58,7 @@ namespace Bu
58 * will attempt to lock the mutex, if the mutex is already locked this 58 * will attempt to lock the mutex, if the mutex is already locked this
59 * function returns immediately with an error code. 59 * function returns immediately with an error code.
60 */ 60 */
61 int trylock(); 61 virtual int trylock();
62 62
63 protected: 63 protected:
64 pthread_mutex_t mutex; /**< The internal mutex reference. */ 64 pthread_mutex_t mutex; /**< The internal mutex reference. */
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 86f651e..53250c2 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -1,24 +1,25 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/config.h"
9#include "bu/myriad.h" 1#include "bu/myriad.h"
10#include "bu/stream.h"
11#include "bu/myriadstream.h" 2#include "bu/myriadstream.h"
3
4#include "bu/membuf.h"
12#include "bu/mutexlocker.h" 5#include "bu/mutexlocker.h"
13#include <stdio.h> 6#include "bu/util.h"
14 7
15#include "bu/sio.h" 8#include "bu/sio.h"
16using Bu::sio;
17using Bu::Fmt;
18 9
19#define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84") 10#define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84")
20 11
21#define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(sStore.getLocation()).arg(x) 12#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
13{ \
14 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
15 "Insufficient data reading myriad data from backing stream."); \
16} (void)0
17
18#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
19{ \
20 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
21 "Insufficient data reading from myriad stream."); \
22} (void)0
22 23
23namespace Bu 24namespace Bu
24{ 25{
@@ -28,779 +29,844 @@ namespace Bu
28 } 29 }
29} 30}
30 31
31Bu::Myriad::Myriad( Bu::Stream &sStore, int iBlockSize, int iPreallocate ) : 32Bu::Myriad::Myriad( Bu::Stream &rBacking, int iBlockSize,
32 sStore( sStore ), 33 int iPreallocateBlocks ) :
34 rBacking( rBacking ),
33 iBlockSize( iBlockSize ), 35 iBlockSize( iBlockSize ),
34 iBlocks( 0 ), 36 iBlockCount( 0 ),
35 iUsed( 0 ), 37 bIsNewStream( true ),
36 bHeaderChanged( false ) 38 bStructureChanged( false ),
39 iLastUsedIndex( -1 )
37{ 40{
38 try 41 if( !rBacking.isSeekable() )
39 { 42 {
40 initialize(); 43 throw Bu::MyriadException( Bu::MyriadException::invalidBackingStream,
44 "Myriad backing stream must be random access (seekable).");
41 } 45 }
42 catch( Bu::MyriadException &e ) 46 if( rBacking.getSize() == 0 )
43 { 47 {
44 if( e.getErrorCode() == MyriadException::emptyStream ) 48 createMyriad( iBlockSize, iPreallocateBlocks );
45 { 49 }
46 initialize( iBlockSize, iPreallocate ); 50 else
47 } 51 {
48 else 52 loadMyriad();
49 {
50 throw;
51 }
52 } 53 }
53} 54}
54 55
55Bu::Myriad::~Myriad() 56Bu::Myriad::~Myriad()
56{ 57{
57 mActiveBlocks.lock(); 58 writeHeader();
58 TRACE("mActiveBlocks locked."); 59}
59 if( !hActiveBlocks.isEmpty() ) 60
60 { 61Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode,
61 sio << "Bu::Myriad::~Myriad(): Error: There are " 62 int32_t iPreallocateBytes )
62 << hActiveBlocks.getSize() << " unsynced blocks!" << sio.nl; 63{
63 } 64 Bu::MutexLocker l( mAccess );
64 TRACE("mActiveBlocks unlocking...");
65 mActiveBlocks.unlock();
66 sync();
67 65
68 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 66 Stream *pStream = new Stream( *this, ++iLastUsedIndex, 0 );
67 int iBlocks = std::max(1, blkDiv( iPreallocateBytes, iBlockSize ));
68 for( int j = 0; j < iBlocks; j++ )
69 { 69 {
70 delete *i; 70 pStream->aBlocks.append( __allocateBlock() );
71 } 71 }
72 mhStream.lock();
73 hStream.insert( pStream->iStream, pStream );
74 mhStream.unlock();
75 bStructureChanged = true;
76
77 return Bu::MyriadStream( *this, pStream, eMode&ReadWrite );
72} 78}
73 79
74void Bu::Myriad::sync() 80Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
81 Bu::Myriad::Mode eMode )
75{ 82{
76 updateHeader(); 83 Stream *pStream = NULL;
77 84 Bu::MutexLocker l( mhStream );
78 mActiveBlocks.lock(); 85 if( (eMode&Create) )
79 TRACE("mActiveBlocks locked.");
80 for( BlockHash::iterator i = hActiveBlocks.begin(); i; i++ )
81 { 86 {
82 if( (*i)->bChanged ) 87 if( hStream.has( iStream ) )
88 {
89 if( (eMode&Exclusive) )
90 {
91 throw Bu::MyriadException( MyriadException::noSuchStream,
92 "Stream exists.");
93 }
94 }
95 else
83 { 96 {
84 syncBlock( *i ); 97 Bu::MutexLocker l( mAccess );
98 if( iStream >= iLastUsedIndex )
99 {
100 iLastUsedIndex = iStream;
101 }
102 pStream = new Stream( *this, iStream, 0 );
103 pStream->aBlocks.append( __allocateBlock() );
104 hStream.insert( pStream->iStream, pStream );
105 bStructureChanged = true;
85 } 106 }
86 } 107 }
87 TRACE("mActiveBlocks unlocked..."); 108 if( !hStream.has( iStream ) )
88 mActiveBlocks.unlock();
89}
90
91void Bu::Myriad::initialize()
92{
93 MutexLocker mLock( mHeader );
94 TRACE("mHeader locked.");
95 lFreeBlocks.clear();
96 sStore.setPosEnd( 0 );
97 Bu::size iSize = sStore.tell();
98 sStore.setPos( 0 );
99
100 unsigned char buf[4];
101 if( sStore.read( buf, 4 ) < 4 )
102 { 109 {
103 TRACE("mHeader unlocked..."); 110 throw Bu::MyriadException( MyriadException::noSuchStream,
104 throw MyriadException( MyriadException::emptyStream, 111 "No such stream.");
105 "Input stream appears to be empty.");
106 } 112 }
107 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
108 { 113 {
109 TRACE("mHeader unlocked..."); 114 Bu::MutexLocker l2( mBacking );
110 throw MyriadException( MyriadException::invalidFormat, 115 if( (eMode&Write) && !rBacking.isWritable() )
111 "Stream does not appear to be a valid Myriad format."); 116 {
117 throw Bu::MyriadException( MyriadException::badMode,
118 "Backing stream does not support writing.");
119 }
112 } 120 }
113 sStore.read( buf, 2 ); 121 if( pStream == NULL )
114 if( buf[0] != 1 )
115 { 122 {
116 TRACE("mHeader unlocked..."); 123 pStream = hStream.get( iStream );
117 throw MyriadException( MyriadException::badVersion,
118 "We can only handle version 1 for now.");
119 } 124 }
120 if( buf[1] != 32 ) 125 if( (eMode&Truncate) )
121 { 126 {
122 TRACE("mHeader unlocked..."); 127 pStream->setSize( 0 );
123 throw MyriadException( MyriadException::invalidWordSize,
124 "We can only handle 32-bit words at the moment.");
125 } 128 }
126 sStore.read( &iBlockSize, 4 ); 129 return Bu::MyriadStream( *this, pStream, eMode );
127 int iStreams; 130}
128 sStore.read( &iStreams, 4 );
129
130 iBlocks = iSize/iBlockSize;
131 //sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize
132 // << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl;
133 131
134 int iHeaderSize = 14 + 8 + 4; 132Bu::Myriad::StreamId Bu::Myriad::allocate()
135 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize ); 133{
134 Bu::MutexLocker l( mAccess );
136 135
137 while( iHeaderSize > iHeaderBlocks*iBlockSize ) 136 Stream *pStream = new Stream( *this, ++iLastUsedIndex, 0 );
138 { 137 mhStream.lock();
139 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize ); 138 hStream.insert( pStream->iStream, pStream );
140 iHeaderSize = 14 + 8 + 4*iHeaderBlocks; 139 mhStream.unlock();
141 } 140 bStructureChanged = true;
142 141
143 //sio << "Myriad: iHeaderSize=" << iHeaderSize 142 return pStream->iStream;
144 // << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; 143}
145 144
146 Stream *pFakeHdr = new Stream; 145void Bu::Myriad::erase( Bu::Myriad::StreamId iStream )
147 pFakeHdr->iId = 0; 146{
148 pFakeHdr->iSize = iHeaderSize; 147 // For now, let's prevent you from erasing a stream if it's open.
149 for( int j = 0; j < iHeaderBlocks; j++ ) 148 Bu::MutexLocker l( mhStream );
149 if( !hStream.has( iStream ) )
150 { 150 {
151 pFakeHdr->aBlocks.append( j ); 151 throw Bu::MyriadException( Bu::MyriadException::noSuchStream,
152 "No such stream exists.");
152 } 153 }
153 154 Stream *pStream = hStream.get( iStream );
154// sio << "Blocks: " << iBlocks << " (size = " << iSize << "/" << iBlockSize
155// << ")" << sio.nl;
156 Bu::BitString bsBlockUsed( iBlocks, false );
157 bsBlockUsed.clear();
158
159// bool bCanSkip = false; // Can skip around, post initial header stream i/o
160 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
161 pIn->setPos( sStore.tell() );
162 for( int j = 0; j < iStreams; j++ )
163 { 155 {
164 aStreams.append( new Stream() ); 156 Bu::MutexLocker sl( pStream->mAccess );
165 Stream &s = *aStreams[j]; 157 if( pStream->iOpenCount > 0 )
166 pIn->read( &s.iId, 4 );
167 pIn->read( &s.iSize, 4 );
168 int iSBlocks = blkDiv(s.iSize, iBlockSize);
169 // sio << "Myriad: - Stream::iId=" << s.iId
170 // << ", Stream::iSize=" << s.iSize
171 // << ", Stream::aBlocks=" << iSBlocks
172 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
173 for( int k = 0; k < iSBlocks; k++ )
174 { 158 {
175 int iBId; 159 throw Bu::MyriadException( Bu::MyriadException::streamOpen,
176 pIn->read( &iBId, 4 ); 160 "Cannot currently erase a stream while it is open.");
177 // sio << "Myriad: - iBId=" << iBId
178 // << ", iStartPos=" << iBId*iBlockSize
179 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
180 s.aBlocks.append( iBId );
181 bsBlockUsed.setBit( iBId );
182 iUsed++;
183 if( (j == 0 && k == iHeaderBlocks-1) )
184 {
185 // sio << "Myriad: - End of prepartition, unlocking skipping."
186 // << sio.nl;
187// bCanSkip = true;
188 MyriadStream *pTmp = new MyriadStream( *this, aStreams[0] );
189 // sio << "Myriad - Position = " << pIn->tell() << sio.nl;
190 pTmp->setPos( pIn->tell() );
191 delete pIn;
192 delete pFakeHdr;
193 pIn = pTmp;
194 }
195 } 161 }
196 }
197 delete pIn;
198 162
199 for( int j = 0; j < iBlocks; j++ ) 163 for( Bu::Array<int32_t>::iterator i = pStream->aBlocks.begin(); i; i++ )
200 {
201 if( bsBlockUsed.getBit( j ) == false )
202 { 164 {
203// sio << "Preinitialized block " << j << " is free." << sio.nl; 165 releaseBlock( *i, false );
204 lFreeBlocks.append( j );
205 } 166 }
167 pStream->aBlocks.clear();
168 hStream.erase( iStream );
206 } 169 }
207// sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; 170 delete pStream;
208 TRACE("mHeader unlocked...");
209} 171}
210 172
211void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) 173void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
174 int32_t iNewSize )
212{ 175{
213 MutexLocker mLock( mHeader ); 176 Stream *pStream;
214 TRACE("mHeader locked.");
215 lFreeBlocks.clear();
216
217 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
218 { 177 {
219 delete *i; 178 Bu::MutexLocker l( mhStream );
220 } 179 pStream = hStream.get( iStream );
221 aStreams.clear();
222 iUsed = 0;
223
224 int iHeaderSize = 14 + 8 + 4;
225 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
226 char cBuf = 1;
227 int iBuf = 0;
228
229 Stream *pStr = new Stream;
230 pStr->iId = 0;
231
232 while( iHeaderSize > iHeaderBlocks*iBlockSize )
233 {
234 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
235 iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
236 } 180 }
181 pStream->setSize( iNewSize );
182}
237 183
238 iPreAllocate += iHeaderBlocks; 184int32_t Bu::Myriad::getSize( StreamId iStream ) const
185{
186 Bu::MutexLocker l( mhStream );
187 return hStream.get( iStream )->getSize();
188}
239 189
240 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" 190bool Bu::Myriad::exists( StreamId iStream ) const
241 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; 191{
242 192 Bu::MutexLocker l( mhStream );
243// bsBlockUsed.setSize( iPreAllocate, true ); 193 return hStream.has( iStream );
244 iUsed++; 194}
245 195
246 char *pBlock = new char[iBlockSize]; 196Bu::String Bu::Myriad::getLocation() const
247 memset( pBlock, 0, iBlockSize ); 197{
248 for( int j = 0; j < iPreAllocate; j++ ) 198 Bu::MutexLocker l( mAccess );
249 { 199 Bu::MutexLocker l2( mBacking );
250 sStore.write( pBlock, iBlockSize ); 200 return Bu::String("myriad(%1,%2):%3")
251 } 201 .arg( 1 ).arg( iBlockSize ).arg( rBacking.getLocation() );
252 delete[] (char *)pBlock; 202}
253 203
254 sStore.setPos( 0 ); 204int32_t Bu::Myriad::getBlockSize() const
205{
206 Bu::MutexLocker l( mAccess );
207 return iBlockSize;
208}
255 209
256 // Magic number 210int32_t Bu::Myriad::getTotalBlocks() const
257 sStore.write( Myriad_MAGIC_CODE, 4 ); 211{
212 Bu::MutexLocker l( mAccess );
213 return iBlockCount;
214}
258 215
259 // Version (0) 216int32_t Bu::Myriad::getUsedBlocks() const
260 sStore.write( &cBuf, 1 ); 217{
218 Bu::MutexLocker l( mAccess );
219 return iBlockCount-lFreeBlocks.getSize();
220}
261 221
262 // Bits per int 222int32_t Bu::Myriad::getFreeBlocks() const
263 cBuf = 32; 223{
264 sStore.write( &cBuf, 1 ); 224 Bu::MutexLocker l( mAccess );
225 return lFreeBlocks.getSize();
226}
265 227
266 // The size of each block 228int32_t Bu::Myriad::getTotalStreams() const
267 sStore.write( &iBlockSize, 4 ); 229{
230 Bu::MutexLocker l( mhStream );
231 return hStream.getSize();
232}
268 233
269 iBuf = 1; 234int32_t Bu::Myriad::getTotalUsedBytes() const
270 // The number of streams 235{
271 sStore.write( &iBuf, 4 ); 236 Bu::MutexLocker l( mhStream );
272 237 int32_t iTotal = 0;
273 // Stream header 238 for( StreamHash::const_iterator i = hStream.begin(); i; i++ )
274 iBuf = 0;
275 sStore.write( &iBuf, 4 );
276 sStore.write( &iHeaderSize, 4 );
277 for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ )
278 { 239 {
279 sStore.write( &iBuf, 4 ); 240 iTotal += i.getValue()->getSize();
280 } 241 }
281 242
282 this->iBlockSize = iBlockSize; 243 return iTotal;
283 this->iBlocks = iPreAllocate; 244}
284
285 pStr->iSize = sStore.tell();
286// sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl;
287 245
288 pStr->iSize = iHeaderSize; 246int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
289 for( int j = 0; j < iHeaderBlocks; j++ ) 247{
248 if( iAssumeBlockSize < 0 )
290 { 249 {
291// sio << "Started block " << j << " is header." << sio.nl; 250 iAssumeBlockSize = getBlockSize();
292 pStr->aBlocks.append( j );
293// bsBlockUsed.setBit( j );
294 iUsed++;
295 } 251 }
296 for( int j = iHeaderBlocks; j < this->iBlocks; j++ ) 252 int32_t iTotal = 0;
297 { 253 {
298// sio << "Started block " << j << " is free." << sio.nl; 254 Bu::MutexLocker l( mhStream );
299 lFreeBlocks.append( j ); 255 for( StreamHash::const_iterator i = hStream.begin(); i; i++ )
256 {
257 if( (i.getValue()->getSize()%iAssumeBlockSize) > 0 )
258 iTotal += iBlockSize-(i.getValue()->getSize()%iAssumeBlockSize);
259 }
300 } 260 }
301 261
302 aStreams.append( pStr ); 262 {
263 Bu::MutexLocker l( mAccess );
264 iTotal += lFreeBlocks.getSize()*iBlockSize;
265 }
303 266
304 //sio << bsBlockUsed.toString() << " - " << pStr->aBlocks << sio.nl; 267 return iTotal;
268}
305 269
306 bHeaderChanged = true; 270Bu::Myriad::StreamIdList Bu::Myriad::getStreamList() const
307 //hStreams.insert( 0, BlockArray( 0 ) ); 271{
308 TRACE("mHeader unlocked..."); 272 mhStream.lock();
273 StreamIdList lIds = hStream.getKeys();
274 mhStream.unlock();
275 lIds.sort();
276 if( lIds.first() == 0 )
277 {
278 lIds.eraseFirst();
279 }
280 return lIds;
309} 281}
310 282
311void Bu::Myriad::updateHeader() 283Bu::BitString Bu::Myriad::buildBlockUseMap() const
312{ 284{
313 MutexLocker mLock( mHeader ); 285 Bu::MutexLocker l( mAccess );
314 TRACE("mHeader locked."); 286 Bu::BitString bsMap( iBlockCount );
287 bsMap.fill();
288 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
289 {
290 bsMap.setBit( *i, false );
291 }
292 return bsMap;
293}
315 294
316 if( bHeaderChanged == false ) 295Bu::Myriad::StreamIdArray Bu::Myriad::buildBlockMap() const
296{
297 Bu::MutexLocker l( mAccess );
298 StreamIdArray bm( iBlockCount );
299 for( int j = 0; j < iBlockCount; j++ )
317 { 300 {
318 TRACE("mHeader unlocked..."); 301 bm.append( -1 );
319 return;
320 } 302 }
321 if( !sStore.canWrite() ) 303 Bu::MutexLocker l2( mhStream );
304 for( StreamHash::const_iterator iStream = hStream.begin();
305 iStream; iStream++ )
322 { 306 {
323 TRACE("mHeader unlocked..."); 307 int32_t iId = iStream.getKey();
324 return; 308 Stream *pStream = iStream.getValue();
309 for( Bu::Array<int32_t>::const_iterator iBlock =
310 pStream->aBlocks.begin(); iBlock; iBlock++ )
311 {
312 bm[*iBlock] = iId;
313 }
325 } 314 }
315 return bm;
316}
326 317
327 char cBuf; 318void Bu::Myriad::sync()
328 int iBuf; 319{
320 writeHeader();
321}
329 322
330 //for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 323bool Bu::Myriad::loadMyriad()
331 //{ 324{
332 // sio << "Myriad: Stream " << Fmt(4) << (*i)->iId << ": " << (*i)->aBlocks << sio.nl; 325 //Bu::println("Load myriad!");
333 //} 326 char sMagicCode[4];
327 rBacking.setPos( 0 );
328 MyriadRead( sMagicCode, 4 );
329 if( memcmp( sMagicCode, Myriad_MAGIC_CODE, 4 ) )
330 {
331 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
332 "Backing stream does not seem to be a Myriad structure.");
333 }
334 uint8_t uVer;
335 uint8_t uBitsPerInt;
336 MyriadRead( &uVer, 1 );
337 if( uVer != 1 )
338 {
339 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
340 "Only version 1 myriad structures are supported.");
341 }
342 MyriadRead( &uBitsPerInt, 1 );
343 if( uBitsPerInt != 32 )
344 {
345 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
346 "Only 32 bits per int are supported at this time.");
347 }
348 MyriadRead( &iBlockSize, 4 );
334 349
335 // Compute the new size of the header. 350 iBlockCount = rBacking.getSize()/iBlockSize;
336 int iHeaderSize = 14 + 8*aStreams.getSize(); 351 if( (rBacking.getSize()%iBlockSize) != 0 )
337// sio << "Myriad: updateHeader: aStreams.getSize() = " << aStreams.getSize()
338// << sio.nl;
339 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
340 { 352 {
341 iHeaderSize += 4*(*i)->aBlocks.getSize(); 353 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
342// sio << "Myriad: updateHeader: (*i)->aBlocks.getSize() = " 354 "Backing stream is not cleanly divisibly by the block size.");
343// << (*i)->aBlocks.getSize() << sio.nl;
344 } 355 }
345 int iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); 356
346 while( iNewBlocks > aStreams[0]->aBlocks.getSize() ) 357 Bu::Hash<int32_t,bool> hUnusedBlocks;
358 for( int32_t j = 0; j < iBlockCount; j++ )
347 { 359 {
348 int iBlock = findEmptyBlock(); 360 hUnusedBlocks.insert( j, true );
349// sio << "Myriad: updateHeader: Appending block " << iBlock
350// << " to header." << sio.nl;
351 aStreams[0]->aBlocks.append( iBlock );
352// bsBlockUsed.setBit( iBlock );
353 iUsed++;
354 iHeaderSize += 4;
355 iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
356 } 361 }
357 aStreams[0]->iSize = iHeaderSize;
358// sio << "Myriad: updateHeader: iHeaderSize=" << iHeaderSize
359// << ", iNewBlocks=" << iNewBlocks << ", curBlocks="
360// << aStreams[0]->aBlocks.getSize() << sio.nl;
361 362
362 MyriadStream sHdr( *this, aStreams[0] ); 363 int iStreamCount;
363 sHdr.write( Myriad_MAGIC_CODE, 4 ); 364 MyriadRead( &iStreamCount, 4 );
365
366 //
367 // Read stream data -- Bootstrap the zero stream
368 //
369 StreamId iStream;
370 MyriadRead( &iStream, 4 );
371 if( iStream != 0 )
372 {
373 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
374 "The first stream defined must be the header/zero stream.");
375 }
376 iLastUsedIndex = iStream;
377 int32_t iHeaderStreamBytes;
378 MyriadRead( &iHeaderStreamBytes, 4 );
364 379
365 // Version (1) 380 Stream *pHeaderStream = new Stream( *this, iStream, iHeaderStreamBytes );
366 cBuf = 1; 381 hStream.insert( iStream, pHeaderStream );
367 sHdr.write( &cBuf, 1 ); 382 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes, iBlockSize );
383 MyriadStream sHeader( *this, pHeaderStream, Read );
368 384
369 // Bits per int 385 // We need to read enough so that we can gurantee that we're within a block
370 cBuf = 32; 386 // that we have read the index to, plus one index.
371 sHdr.write( &cBuf, 1 ); 387 for( int32_t j = 0; j < iHeaderStreamBlocks; j++ )
388 {
389 int32_t iBlockIndex;
390 MyriadRead( &iBlockIndex, 4 );
391 hUnusedBlocks.erase( iBlockIndex );
392 pHeaderStream->aBlocks.append( iBlockIndex );
393 if( rBacking.tell()+4 <= (j+1)*iBlockSize )
394 break;
395 }
372 396
373 // The size of each block 397 // Bootstrap now using the header stream to read the rest of the data.
374 sHdr.write( &iBlockSize, 4 ); 398 sHeader.setPos( rBacking.tell() );
399 while( pHeaderStream->aBlocks.getSize() < iHeaderStreamBlocks )
400 {
401 int32_t iBlockIndex;
402 ReqRead( sHeader, &iBlockIndex, 4 );
403 hUnusedBlocks.erase( iBlockIndex );
404 pHeaderStream->aBlocks.append( iBlockIndex );
405 }
375 406
376 iBuf = aStreams.getSize(); 407 // Ok, now we can read the rest of the header in.
377 // The number of streams 408 for( int j = 1; j < iStreamCount; j++ )
378 sHdr.write( &iBuf, 4 );
379
380 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
381 { 409 {
382 sHdr.write( &(*i)->iId, 4 ); 410 int32_t iStreamBytes;
383 sHdr.write( &(*i)->iSize, 4 ); 411 ReqRead( sHeader, &iStream, 4 );
384 int iUsedBlocks = blkDiv( (*i)->iSize, iBlockSize ); 412 ReqRead( sHeader, &iStreamBytes, 4 );
385// for( BlockArray::iterator j = (*i)->aBlocks.begin(); j; j++ ) 413 Stream *pStream = new Stream( *this, iStream, iStreamBytes );
386 for( int j = 0; j < iUsedBlocks; j++ ) 414 int32_t iBlocks = blkDiv(iStreamBytes, iBlockSize );
415 for( int k = 0; k < iBlocks; k++ )
387 { 416 {
388 sHdr.write( &(*i)->aBlocks[j], 4 ); 417 int32_t iBlockIndex;
418 ReqRead( sHeader, &iBlockIndex, 4 );
419 hUnusedBlocks.erase( iBlockIndex );
420 pStream->aBlocks.append( iBlockIndex );
389 } 421 }
422 hStream.insert( iStream, pStream );
423 if( iLastUsedIndex < iStream )
424 iLastUsedIndex = iStream;
390 } 425 }
426
427 lFreeBlocks = hUnusedBlocks.getKeys();
428 //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
391 429
392 bHeaderChanged = false; 430 bIsNewStream = false;
393 TRACE("mHeader unlocked..."); 431
432 return true;
394} 433}
395 434
396int Bu::Myriad::createStream( int iPreAllocate ) 435void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
397{ 436{
398 MutexLocker mLock( mHeader ); 437 if( iBlockSize < 8 )
399 TRACE("mHeader locked."); 438 {
439 throw Bu::MyriadException( Bu::MyriadException::invalidParameter,
440 "iBlockSize cannot be below 8");
441 }
442 if( rBacking.getSize() )
443 {
444 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
445 "Backing stream contains data, but not a myriad structure.");
446 }
447
448 // Start with the bytes for the file header and initial stream header
449 int iHeaderStreamBytes
450 = 14 // Base header
451 + 8; // Stream header
452
453 // Pick the block count that matches our current estimate for the header
454 // plus one block index.
455 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
400 456
401 Stream *pStr = new Stream(); 457 //Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
402 pStr->iId = aStreams.last()->iId+1; 458 while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
403 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate=" 459 > iHeaderStreamBlocks*iBlockSize )
404 // << iPreAllocate << sio.nl; 460 {
405 pStr->iSize = 0; 461 iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize);
406 aStreams.append( pStr ); 462 if( iHeaderStreamBlocks > 100 )
463 break;
464 //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
465 }
407 466
408 for( int j = 0; j < iPreAllocate; j++ ) 467 if( iPreallocateBlocks < iHeaderStreamBlocks )
409 { 468 {
410 int iFreeBlock = findEmptyBlock(); 469 iPreallocateBlocks = iHeaderStreamBlocks;
411// sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
412 pStr->aBlocks.append( iFreeBlock );
413// bsBlockUsed.setBit( iFreeBlock );
414 iUsed++;
415 } 470 }
471 rBacking.setSize( iBlockSize*iPreallocateBlocks );
416 472
417 bHeaderChanged = true; 473 //
474 // Write Myriad header
475 //
476 uint8_t uVer = 1;
477 uint8_t uBpi = 32;
478 int32_t iStreamCount = 1;
479 rBacking.setPos( 0 );
480 rBacking.write( Myriad_MAGIC_CODE, 4 );
481 rBacking.write( &uVer, 1 );
482 rBacking.write( &uBpi, 1 );
483 rBacking.write( &iBlockSize, 4 );
484 rBacking.write( &iStreamCount, 4 );
418 485
419 TRACE("mHeader unlocked..."); 486 Stream *pHeadStream = new Stream( *this, 0, Bu::Myriad::ReadWrite );
420 return pStr->iId; 487 //
421} 488 // Write stream header
489 //
490 uint32_t uStreamId = 0;
491 uint32_t uStreamSize = iHeaderStreamBytes+iHeaderStreamBlocks*4;
492 rBacking.write( &uStreamId, 4 );
493 rBacking.write( &uStreamSize, 4 );
494 for( int iBlockIndex = 0; iBlockIndex < iHeaderStreamBlocks; iBlockIndex++ )
495 {
496 rBacking.write( &iBlockIndex, 4 );
497 pHeadStream->aBlocks.append( iBlockIndex );
498 }
499 rBacking.flush();
422 500
423int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate ) 501 hStream.insert( pHeadStream->iStream, pHeadStream );
424{
425 MutexLocker mLock( mHeader );
426 TRACE("mHeader locked.");
427 502
428 try 503 for( int32_t j = iHeaderStreamBlocks; j < iPreallocateBlocks; j++ )
429 { 504 {
430 findStream( iId ); 505 lFreeBlocks.append( j );
431 TRACE("mHeader unlocked...");
432 throw MyriadException( MyriadException::streamExists,
433 "There is already a stream with the given id.");
434 } 506 }
435 catch( MyriadException &e ) 507 iLastUsedIndex = 0;
508 iBlockCount = iPreallocateBlocks;
509}
510
511void Bu::Myriad::writeHeader()
512{
513 Bu::MutexLocker l( mAccess );
514 if( !rBacking.isWritable() )
515 return;
516 //Bu::println("Writing stream breakdown:");
517 Bu::MemBuf mbHeader;
436 { 518 {
437 Stream *pStr = new Stream(); 519 Bu::MutexLocker l2( mhStream );
438 pStr->iId = iId; 520
439 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate=" 521 int32_t iHdrStreamSize = __calcHeaderSize();
440 // << iPreAllocate << sio.nl; 522 // Maybe just do stream surgery here.
441 pStr->iSize = 0;
442 if( aStreams.last()->iId < iId )
443 {
444 aStreams.append( pStr );
445 }
446 else
447 { 523 {
448 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 524 Stream *psHeader = hStream.get( 0 );
525 Bu::MutexLocker l2( psHeader->mAccess );
526 int iNewBlocks = Bu::blkDiv( iHdrStreamSize, iBlockSize );
527 if( iHdrStreamSize < psHeader->iSize )
449 { 528 {
450 if( (*i)->iId > iId ) 529 while( psHeader->aBlocks.getSize() > iNewBlocks )
451 { 530 {
452 aStreams.insert( i, pStr ); 531 __releaseBlock( psHeader->aBlocks.last(), false );
453 break; 532 psHeader->aBlocks.eraseLast();
454 } 533 }
455 } 534 }
535 else if( iHdrStreamSize > psHeader->iSize )
536 {
537 while( psHeader->aBlocks.getSize() < iNewBlocks )
538 {
539 psHeader->aBlocks.append( __allocateBlock() );
540 }
541 }
542 psHeader->iSize = iHdrStreamSize;
456 } 543 }
457 544
458 for( int j = 0; j < iPreAllocate; j++ ) 545 //Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
546
547 uint8_t uVer = 1;
548 uint8_t uBpi = 32;
549 int32_t iStreamCount = hStream.getSize();
550
551 mbHeader.write( Myriad_MAGIC_CODE, 4 );
552 mbHeader.write( &uVer, 1 );
553 mbHeader.write( &uBpi, 1 );
554 mbHeader.write( &iBlockSize, 4 );
555 mbHeader.write( &iStreamCount, 4 );
556 StreamHash::KeyList lStreamId = hStream.getKeys();
557 lStreamId.sort();
558 if( lStreamId.first() != 0 )
559 {
560 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
561 "There doesn't appear to be a zero (header) stream.");
562 }
563 for( StreamHash::KeyList::iterator i = lStreamId.begin(); i; i++ )
459 { 564 {
460 int iFreeBlock = findEmptyBlock(); 565 uint32_t uStreamId = *i;
461 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 566 Stream *pStream = hStream.get( uStreamId );
462 pStr->aBlocks.append( iFreeBlock ); 567 uint32_t uStreamSize = pStream->getSize();
463// bsBlockUsed.setBit( iFreeBlock ); 568 mbHeader.write( &uStreamId, 4 );
464 iUsed++; 569 mbHeader.write( &uStreamSize, 4 );
570 int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize );
571 Bu::Array<int32_t> aBlocks = pStream->getBlockList();
572
573 //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
574
575// for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
576 for( int j = 0; j < iBlocks; j++ )
577 {
578 mbHeader.write( &aBlocks[j], 4 );
579 }
465 } 580 }
466 581
467 bHeaderChanged = true;
468 582
469 TRACE("mHeader unlocked...");
470 return pStr->iId;
471 } 583 }
472 TRACE("mHeader unlocked...");
473}
474 584
475int Bu::Myriad::findEmptyBlock() 585 Bu::MyriadStream sHeader( *this, hStream.get( 0 ), Bu::Myriad::Write );
476{ 586 sHeader.write( mbHeader.getString() );
477 bHeaderChanged = true; 587 bStructureChanged = false;
478
479 if( lFreeBlocks.isEmpty() )
480 {
481 sStore.setSize( (iBlocks+1)*(Bu::size)iBlockSize );
482 return iBlocks++;
483 }
484 else
485 {
486 return lFreeBlocks.dequeue();
487 }
488} 588}
489 589
490void Bu::Myriad::deleteStream( int iId ) 590int32_t Bu::Myriad::__calcHeaderSize()
491{ 591{
492 MutexLocker mLock( mHeader ); 592 int32_t iHdrSize = 4+1+1+4+4;
493 TRACE("mHeader locked.");
494 593
495 if( iId < 0 ) 594 StreamHash::KeyList lStreamId = hStream.getKeys();
595 lStreamId.sort();
596 if( lStreamId.first() != 0 )
496 { 597 {
497 TRACE("mHeader unlocked..."); 598 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
498 throw MyriadException( MyriadException::invalidStreamId, 599 "There doesn't appear to be a zero (header) stream.");
499 "Invalid stream id.");
500 } 600 }
501 if( iId == 0 ) 601 for( StreamHash::KeyList::iterator i = lStreamId.begin(); i; i++ )
502 { 602 {
503 TRACE("mHeader unlocked..."); 603 iHdrSize += 4+4;
504 throw MyriadException( MyriadException::protectedStream, 604 int32_t iStreamSize = hStream.get( *i )->getSize();
505 "You cannot delete stream zero, it is protected."); 605 if( (*i) != 0 )
506 }
507 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
508 {
509 if( (*i)->iId == iId )
510 { 606 {
511 Stream *pStream = *i; 607 iHdrSize += Bu::blkDiv( iStreamSize, iBlockSize )*4;
512 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
513 {
514 lFreeBlocks.append( *j );
515// bsBlockUsed.setBit( *j, false );
516 iUsed--;
517 }
518 aStreams.erase( i );
519 bHeaderChanged = true;
520 delete pStream;
521 TRACE("mHeader unlocked...");
522 return;
523 } 608 }
524 } 609 }
525 TRACE("mHeader unlocked...");
526}
527 610
528Bu::Array<int> Bu::Myriad::getStreamIds() 611 //Bu::println("HeaderCalc:");
529{ 612 //Bu::println(" Base (no header stream): %1").arg( iHdrSize );
530 MutexLocker mLock( mHeader ); 613 int32_t iNewSize = iHdrSize;
531 TRACE("mHeader locked."); 614 int32_t iOldSize;
532 615
533 Bu::Array<int> aRet( aStreams.getSize() ); 616 do {
534 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 617 iOldSize = iNewSize;
535 { 618 iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4;
536 aRet.append( (*i)->iId ); 619 //Bu::println(" Recomp: %1").arg( iNewSize );
537 } 620 } while( iOldSize != iNewSize );
538 621
539 TRACE("mHeader unlocked..."); 622 return iNewSize;
540 return aRet;
541} 623}
542 624
543int Bu::Myriad::getStreamSize( int iId ) 625int32_t Bu::Myriad::allocateBlock()
544{ 626{
545 MutexLocker mLock( mHeader ); 627 Bu::MutexLocker l( mAccess );
546 TRACE("mHeader locked."); 628 return __allocateBlock();
547
548 TRACE("mHeader unlocked...");
549 return findStream( iId )->iSize;
550} 629}
551 630
552bool Bu::Myriad::hasStream( int iId ) 631int32_t Bu::Myriad::__allocateBlock()
553{ 632{
554 MutexLocker mLock( mHeader ); 633 bStructureChanged = true;
555 TRACE("mHeader locked."); 634 if( lFreeBlocks.isEmpty() )
556
557 try
558 { 635 {
559 findStream( iId ); 636 // Increase the size of the backing stream
560 TRACE("mHeader unlocked..."); 637 int32_t iIndex = iBlockCount++;
561 return true; 638 rBacking.setSize( iBlockCount*iBlockSize );
562 }catch(...) 639 return iIndex;
640 }
641 else
563 { 642 {
564 TRACE("mHeader unlocked..."); 643 // Provide an existing free block.
565 return false; 644 return lFreeBlocks.peekPop();
566 } 645 }
567} 646}
568 647
569Bu::MyriadStream Bu::Myriad::openStream( int iId ) 648void Bu::Myriad::releaseBlock( int32_t iBlockId, bool bBlank )
570{ 649{
571 MutexLocker mLock( mHeader ); 650 Bu::MutexLocker l( mAccess );
572 TRACE("mHeader locked."); 651 __releaseBlock( iBlockId, bBlank );
573
574 TRACE("mHeader unlocked...");
575 //sio << "Myriad: Request to open stream: " << iId << sio.nl;
576 return MyriadStream( *this, findStream( iId ) );
577} 652}
578 653
579int Bu::Myriad::getNumStreams() 654void Bu::Myriad::__releaseBlock( int32_t iBlockId, bool bBlank )
580{ 655{
581 MutexLocker mLock( mHeader ); 656 bStructureChanged = true;
582 TRACE("mHeader locked."); 657 lFreeBlocks.append( iBlockId );
583 658 if( bBlank )
584 TRACE("mHeader unlocked..."); 659 {
585 return aStreams.getSize(); 660 blankBlock( iBlockId );
661 }
586} 662}
587 663
588int Bu::Myriad::getBlockSize() 664void Bu::Myriad::blankBlock( int32_t iBlockId )
589{ 665{
590 return iBlockSize; 666 Bu::MutexLocker l( mBacking );
667 rBacking.setPos( iBlockId*iBlockSize );
668 int32_t iChunk = std::min( iBlockSize, 4096 );
669 uint8_t *pChunk = new uint8_t[iChunk];
670 memset( pChunk, 0, iChunk );
671 int iLeft = iBlockSize;
672 while( iLeft > 0 )
673 {
674 int32_t iWrite = rBacking.write( pChunk, std::min( iChunk, iLeft ) );
675 iLeft -= iWrite;
676 }
677 delete[] pChunk;
591} 678}
592 679
593int Bu::Myriad::getNumBlocks() 680void Bu::Myriad::openStream( StreamId id )
594{ 681{
595 return iBlocks; 682 Bu::MutexLocker l( mhStream );
683 hStream.get( id )->open();
596} 684}
597 685
598int Bu::Myriad::getNumUsedBlocks() 686void Bu::Myriad::closeStream( StreamId id )
599{ 687{
600 return iUsed; 688 Bu::MutexLocker l( mhStream );
689 hStream.get( id )->close();
601} 690}
602 691
603Bu::size Bu::Myriad::getTotalUsedBytes() 692int32_t Bu::Myriad::blockRead( int32_t iBlock, int32_t iStart,
693 void *pTarget, int32_t iSize )
604{ 694{
605 MutexLocker mLock( mHeader ); 695 int32_t iUpperSize = iBlockSize - (iStart%iBlockSize);
606 TRACE("mHeader locked."); 696/* Bu::println("Max read within block: %1 vs %2 (start=%3, blocksize=%4)")
697 .arg( iUpperSize ).arg( iSize )
698 .arg( iStart ).arg( iBlockSize );
699*/
700 int32_t iAmnt = std::min( iSize, iUpperSize );
701 Bu::MutexLocker l( mBacking );
702 rBacking.setPos( iBlockSize*iBlock + iStart );
607 703
608 Bu::size iTotalSize = 0; 704 return rBacking.read( pTarget, iAmnt );
609 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
610 {
611 iTotalSize += (*i)->iSize;
612 }
613 TRACE("mHeader unlocked...");
614 return iTotalSize;
615} 705}
616 706
617Bu::size Bu::Myriad::getTotalUnusedBytes() 707int32_t Bu::Myriad::blockWrite( int32_t iBlock, int32_t iStart,
708 const void *pTarget, int32_t iSize )
618{ 709{
619 MutexLocker mLock( mHeader ); 710 int32_t iUpperSize = iBlockSize - (iStart%iBlockSize);
620 TRACE("mHeader locked."); 711/* Bu::println("Max write within block: %1 vs %2 (start=%3, blocksize=%4)")
712 .arg( iUpperSize ).arg( iSize )
713 .arg( iStart ).arg( iBlockSize );
714*/
715 int32_t iAmnt = std::min( iSize, iUpperSize );
716 Bu::MutexLocker l( mBacking );
717 rBacking.setPos( iBlock*iBlockSize + iStart );
621 718
622 Bu::size iTotalSize = (iBlocks-iUsed)*iBlockSize; 719 return rBacking.write( pTarget, iAmnt );
623 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
624 {
625 iTotalSize += iBlockSize - ((Bu::size)(*i)->iSize%iBlockSize);
626 }
627 TRACE("mHeader unlocked...");
628 return iTotalSize;
629} 720}
630 721
631Bu::size Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize ) 722/////////
632{ 723// Bu::Myriad::Stream
633 MutexLocker mLock( mHeader ); 724//
634 TRACE("mHeader locked.");
635 725
636 Bu::size iTotalSize = (iBlocks-iUsed)*iFakeBlockSize; 726Bu::Myriad::Stream::Stream( Bu::Myriad &rParent, Bu::Myriad::StreamId iStream,
637 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 727 int32_t iSize ) :
638 { 728 rParent( rParent ),
639 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize); 729 iStream( iStream ),
640 } 730 iSize( iSize ),
641 TRACE("mHeader unlocked..."); 731 iOpenCount( 0 )
642 return iTotalSize; 732{
643} 733}
644 734
645Bu::Myriad::Stream *Bu::Myriad::findStream( int iId ) 735Bu::Myriad::Stream::~Stream()
646{ 736{
647 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
648 {
649 if( (*i)->iId == iId )
650 return *i;
651 }
652
653 throw MyriadException( MyriadException::noSuchStream,
654 "The requested stream doesn't exist and cannot be opened." );
655
656 return NULL;
657} 737}
658 738
659Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock ) 739int32_t Bu::Myriad::Stream::getSize() const
660{ 740{
661// sio << "Myriad: Reading block " << iBlock << ", bytes " 741 Bu::MutexLocker l( mAccess );
662// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl; 742 return iSize;
663 Block *pBlock = new Block; 743}
664 pBlock->pData = new char[iBlockSize];
665 sStore.setPos( iBlockSize * (Bu::size)iBlock );
666 sStore.read( pBlock->pData, iBlockSize );
667 pBlock->bChanged = false;
668 pBlock->iBlockIndex = iBlock;
669
670 mActiveBlocks.lock();
671 TRACE("mHeader locked.");
672 hActiveBlocks.insert( iBlock, pBlock );
673 TRACE("mHeader unlocked...");
674 mActiveBlocks.unlock();
675 744
676 return pBlock; 745int32_t Bu::Myriad::Stream::getBlockSize() const
746{
747 Bu::MutexLocker l( mAccess );
748 return rParent.iBlockSize;
677} 749}
678 750
679void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock ) 751Bu::Myriad::StreamId Bu::Myriad::Stream::getStreamId() const
680{ 752{
681 if( pBlock == NULL ) 753 return iStream;
682 return; 754}
683// sio << "Myriad: Releasing block " << pBlock->iBlockIndex << sio.nl;
684 syncBlock( pBlock );
685 mActiveBlocks.lock();
686 TRACE("mHeader locked.");
687 hActiveBlocks.erase( pBlock->iBlockIndex );
688 TRACE("mHeader unlocked...");
689 mActiveBlocks.unlock();
690 755
691 delete[] pBlock->pData; 756int32_t Bu::Myriad::Stream::getOpenCount() const
692 delete pBlock; 757{
758 Bu::MutexLocker l( mAccess );
759 return iOpenCount;
693} 760}
694 761
695void Bu::Myriad::syncBlock( Block *pBlock ) 762void Bu::Myriad::Stream::setSize( int32_t iNewSize )
696{ 763{
697 if( pBlock->bChanged ) 764 // Two possible modes, shrink or grow.
765 Bu::MutexLocker l( mAccess );
766 int iNewBlocks = Bu::blkDiv( iNewSize, rParent.iBlockSize );
767 if( iNewSize < iSize )
698 { 768 {
699// sio << "Myriad: - Block changed, writing back to stream." << sio.nl; 769 // Shrink it
700 sStore.setPos( iBlockSize * (Bu::size)pBlock->iBlockIndex ); 770 while( aBlocks.getSize() > iNewBlocks )
701 sStore.write( pBlock->pData, iBlockSize ); 771 {
702 pBlock->bChanged = false; 772 rParent.releaseBlock( aBlocks.last(), false );
773 aBlocks.eraseLast();
774 }
775 iSize = iNewSize;
776 }
777 else if( iNewSize > iSize )
778 {
779 // Grow it
780 while( aBlocks.getSize() < iNewBlocks )
781 {
782 aBlocks.append( rParent.allocateBlock() );
783 }
784 iSize = iNewSize;
703 } 785 }
704} 786}
705 787
706int Bu::Myriad::streamAddBlock( Stream *pStream ) 788int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
789 int32_t iSize )
707{ 790{
708 MutexLocker mLock( mHeader ); 791 int32_t iRead = 0;
709 TRACE("mHeader locked."); 792 Bu::MutexLocker l( mAccess );
710 793
711 int iBlock = findEmptyBlock(); 794 if( iStart >= this->iSize )
712 pStream->aBlocks.append( iBlock ); 795 return 0;
713// bsBlockUsed.setBit( iBlock );
714// bHeaderChanged = true;
715 iUsed++;
716 TRACE("mHeader unlocked...");
717 return iBlock;
718}
719
720void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
721{
722 MutexLocker mLock( mHeader );
723 TRACE("mHeader locked.");
724 796
725 if( pStream->iSize == iSize ) 797 if( iStart+iSize >= this->iSize )
726 { 798 {
727 TRACE("mHeader unlocked..."); 799 iSize = this->iSize-iStart;
728 return;
729 } 800 }
730 else if( pStream->iSize > iSize ) 801
802 while( iSize > 0 )
731 { 803 {
732 // Shrink 804 int32_t iBlock = aBlocks[iStart/rParent.iBlockSize];
733 TRACE(Bu::String("Shrink stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->iSize).arg(iSize).end() ); 805 int32_t iChunkRead = rParent.blockRead(
734 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; 806 iBlock, iStart%rParent.iBlockSize, pTarget, iSize
735 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize ) 807 );
736 { 808 if( iChunkRead == 0 )
737// if( bsBlockUsed.getBit( pStream->aBlocks.last() ) ) 809 break;
738 iUsed--; 810 iRead += iChunkRead;
739// else 811 iStart += iChunkRead;
740// sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl; 812 reinterpret_cast<ptrdiff_t &>(pTarget) += iChunkRead;
741 lFreeBlocks.enqueue( pStream->aBlocks.last() ); 813 iSize -= iChunkRead;
742// bsBlockUsed.setBit( pStream->aBlocks.last(), false );
743 pStream->aBlocks.eraseLast();
744 }
745 pStream->iSize = iSize;
746 bHeaderChanged = true;
747 } 814 }
748 else 815
816 return iRead;
817}
818
819int32_t Bu::Myriad::Stream::write( int32_t iStart, const void *pTarget,
820 int32_t iSize )
821{
822 int32_t iWrite = 0;
823 Bu::MutexLocker l( mAccess );
824 while( iSize > 0 )
749 { 825 {
750 // Grow 826 int32_t iBlockIdx = iStart/rParent.iBlockSize;
751 TRACE(Bu::String("Grow stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->iSize).arg(iSize).end() ); 827 while( iBlockIdx >= aBlocks.getSize() )
752 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
753 iNewSize < iSize; iNewSize += iBlockSize )
754 { 828 {
755 //streamAddBlock( pStream ); 829 aBlocks.append( rParent.allocateBlock() );
756 int iBlock = findEmptyBlock();
757 pStream->aBlocks.append( iBlock );
758// bsBlockUsed.setBit( iBlock );
759// bHeaderChanged = true;
760 iUsed++;
761 } 830 }
762 pStream->iSize = iSize; 831 int32_t iBlock = aBlocks[iBlockIdx];
763 bHeaderChanged = true; 832 int32_t iChunkWrite = rParent.blockWrite(
833 iBlock, iStart%rParent.iBlockSize, pTarget, iSize
834 );
835 if( iChunkWrite == 0 )
836 break;
837 iWrite += iChunkWrite;
838 iStart += iChunkWrite;
839 reinterpret_cast<ptrdiff_t &>(pTarget) += iChunkWrite;
840 iSize -= iChunkWrite;
764 } 841 }
765 TRACE("mHeader unlocked..."); 842 if( this->iSize < iStart )
843 this->iSize = iStart;
844
845 return iWrite;
766} 846}
767 847
768void Bu::Myriad::headerChanged() 848Bu::String Bu::Myriad::Stream::getLocation() const
769{ 849{
770 bHeaderChanged = true; 850 Bu::MutexLocker l( mAccess );
851 return Bu::String("%1:stream %2")\
852 .arg( rParent.getLocation() ).arg( iStream );
771} 853}
772 854
773bool Bu::Myriad::isMyriad( Bu::Stream &sStore ) 855Bu::Array<int32_t> Bu::Myriad::Stream::getBlockList() const
774{ 856{
775 uint8_t uTmp; 857 Bu::MutexLocker l( mAccess );
776 858 return aBlocks.clone();
777 return isMyriad( sStore, uTmp );
778} 859}
779 860
780bool Bu::Myriad::isMyriad( Bu::Stream &sStore, uint8_t &uTmp ) 861void Bu::Myriad::Stream::open()
781{ 862{
782 sStore.setPos( 0 ); 863 Bu::MutexLocker l( mAccess );
783 864 iOpenCount++;
784 unsigned char buf[4];
785 if( sStore.read( buf, 4 ) < 4 )
786 throw MyriadException( MyriadException::emptyStream,
787 "Input stream appears to be empty.");
788 sStore.read( &uTmp, 1 );
789 sStore.setPos( 0 );
790 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
791 {
792 return false;
793 }
794 return true;
795} 865}
796 866
797const Bu::BitString Bu::Myriad::getBlocksUsed() const 867bool Bu::Myriad::Stream::close()
798{ 868{
799 Bu::BitString bs( iBlocks, false ); 869 Bu::MutexLocker l( mAccess );
800 for( int j = 0; j < iBlocks; j++ ) 870 return (bool)(--iOpenCount);
801 bs.setBit( j );
802 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
803 bs.setBit( *i, false );
804 return bs;
805} 871}
806 872
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 14467a4..5accd1e 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -1,24 +1,16 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_H 1#ifndef BU_MYRIAD_H
9#define BU_MYRIAD_H 2#define BU_MYRIAD_H
10 3
11#include <stdint.h> 4#include "bu/stream.h"
12#include "bu/bitstring.h"
13#include "bu/exceptionbase.h" 5#include "bu/exceptionbase.h"
6#include "bu/mutex.h"
14#include "bu/array.h" 7#include "bu/array.h"
15#include "bu/hash.h" 8#include "bu/hash.h"
16#include "bu/mutex.h" 9
17#include "bu/extratypes.h" 10#include "bu/bitstring.h"
18 11
19namespace Bu 12namespace Bu
20{ 13{
21 class Stream;
22 class MyriadStream; 14 class MyriadStream;
23 15
24 subExceptionDeclBegin( MyriadException ) 16 subExceptionDeclBegin( MyriadException )
@@ -31,206 +23,229 @@ namespace Bu
31 noSuchStream, 23 noSuchStream,
32 streamExists, 24 streamExists,
33 invalidStreamId, 25 invalidStreamId,
34 protectedStream 26 protectedStream,
27 invalidParameter,
28 invalidBackingStream,
29 badMode,
30 streamOpen,
35 }; 31 };
36 subExceptionDeclEnd(); 32 subExceptionDeclEnd();
37 33
38 /** 34 /**
39 * Myriad block-allocated stream multiplexing system. This is a system for 35 * Myriad Stream Multiplexer. This is a system that allows you to store
40 * creating streams that contain other streams in a flexible and lightweight 36 * many streams within a single backing stream. This is great for databases,
41 * manner. Basically, you can create a file (or any other stream) that can 37 * caching, etc. It's fairly lightweight, and allows all streams to grow
42 * store any number of flexible, growing streams. The streams within the 38 * dynamically using a block-allocation scheme. This is used extensively
43 * Myriad stream are automatically numbered, not named. This works more 39 * by the caching system and MyriadFs as well as other systems within
44 * or less like a filesystem, but without the extra layer for managing 40 * libbu++.
45 * file and directory links. This would actually be very easy to add
46 * on top of Myriad, but is not required.
47 *
48 * Header format is as follows:
49 *
50 * MMMMvBssssSSSS*
51 * M = Magic number (0AD3FA84)
52 * v = version number
53 * B = Bits per int
54 * s = Blocksize in bytes
55 * S = Number of Streams
56 *
57 * The * represents the Stream headers, one per stream, as follows:
58 * IIIIssss$
59 * I = Id number of the stream
60 * s = size of stream in bytes
61 *
62 * The $ represents the Block headers, one per used block, as follows:
63 * IIII
64 * I = Index of the block
65 *
66 * The stream/block data is interleaved in the header, so all blocks stored
67 * with one stream are together. The block headers are in order, and the
68 * data in them is required to be "solid" you cannot fill partial blocks
69 * mid-way through a stream.
70 *
71 * The initial block starts with the nids header, and is both the zero block
72 * and the zero stream. For now, the minimum block size is the size needed
73 * to store the base header, the zero stream header, and the first two
74 * blocks of the zero stream, so 30 bytes. Since it's reccomended to use
75 * a size that will fit evenly into filesystem blocks, then a size of 32 is
76 * probably the smallest reccomended size because all powers of two equal
77 * to or greater than 32 are evenly divisible by 32.
78 *
79 * I have had a thought that if the block size were smaller than 42 bytes
80 * the header would consume the first N blocks where N * block size is
81 * enough space to house the initial header, the first stream header, and
82 * the first N block headers. This, of course, causes you to hit an
83 * infinite header if the block size is small enough.
84 */ 41 */
85 class Myriad 42 class Myriad
86 { 43 {
87 friend class MyriadStream; 44 public:
45 typedef int32_t StreamId;
46 typedef Bu::Array<StreamId> StreamIdArray;
47 typedef Bu::List<StreamId> StreamIdList;
48 enum Mode : int32_t {
49 None = 0x00,
50
51 // Flags
52 Read = 0x01, ///< Open file for reading
53 Write = 0x02, ///< Open file for writing
54 Create = 0x04, ///< Create file if it doesn't exist
55 Truncate = 0x08, ///< Truncate file if it does exist
56 Append = 0x10, ///< Start writing at end of file
57 //NonBlock = 0x20, ///< Open file in non-blocking mode
58 Exclusive = 0x40, ///< Create file, if it exists then fail
59
60 // Helpful mixes
61 ReadWrite = 0x03, ///< Open for reading and writing
62 WriteNew = 0x0E ///< Create a file (or truncate) for writing.
63 /// Same as Write|Create|Truncate
64 };
65
88 public: 66 public:
89 /** 67 /**
90 * Create a Myriad object that uses the given stream to store data. 68 * Open existing Myriad container, or initialize a new one if the
91 * This stream must be random access. The block size and preallocate 69 * backing stream is empty. If other data is already in the provided
92 * values passed in are values that will be used if the given stream 70 * backing stream an error is thrown.
93 * is empty. In that case the stream will be "formatted" for myriad 71 *
94 * with the specified block size. If there is already a viable Myriad 72 * Myriad format V0
95 * format present in the stream, then the blocksize and preallocate 73 * 0 - 3: Myriad_MAGIC_CODE (0ad3fa84)
96 * values will be ignored and the values from the stream will be used 74 * 4 - 4: Version Id (1)
97 * instead. If the stream doesn't appear to be Myriad formatted an 75 * 5 - 5: Bits per integer (32)
98 * exception will be thrown. 76 * 6 - 9: Block size in bytes.
77 * 10 - 13: Number of streams.
78 * 14 - ...: Stream Data
79 *
80 * Stream Data:
81 * 0 - 3: Stream Id
82 * 4 - 7: Size of stream in bytes
83 * 8 - ...: List of blocks in stream (4 bytes per block
99 */ 84 */
100 Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ); 85 Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1,
86 int32_t iPreallocateBlocks=-1 );
101 virtual ~Myriad(); 87 virtual ~Myriad();
102 88
103 /** 89 /**
104 * Destroy whatever data may be in the base stream and create a new 90 * Creates a new stream open in the specified eMode and, optionally,
105 * Myriad system there with the given blocksize. Use this with care, 91 * preallocates the specificed amount of space. The stream is zero
106 * it will destroy anything that was already in the stream, and 92 * bytes even if space is preallocated. The open stream is returned,
107 * generally, should not ever have to be used. 93 * ready for use. Use this if you don't care what the id is of the
94 * newly created stream.
108 */ 95 */
109 void initialize( int iBlockSize, int iPreAllocate=1 ); 96 MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
110 97
111 /** 98 /**
112 * Create a new stream within the Myriad system. The ID of the new 99 * Open an existing stream or create a new stream with the specified
113 * stream is returned. 100 * id (iStream) with the specified eMode. This respects the normal file
101 * modes, see Bu::Myriad::Mode for details.
114 */ 102 */
115 int createStream( int iPreAllocate=1 ); 103 MyriadStream open( StreamId iStream, Mode eMode );
116 104
117 /** 105 /**
118 * Create a new stream within the Myriad system with a given id. The 106 * Allocate a new stream but do not open it, just ensure it exists and
119 * id that you provide will be the new id of the stream unless it's 107 * return the id of the newly allocated stream.
120 * already used, in which case an error is thrown. This is primarilly
121 * useful when copying an old Myriad file into a new one.
122 */ 108 */
123 int createStreamWithId( int iId, int iPreAllocate=1 ); 109 StreamId allocate();
124 110
125 /** 111 /**
126 * Delete a stream that's already within the Myriad. 112 * Erase the stream specified by iStream. This only can work when the
113 * stream is not open at the moment.
127 */ 114 */
128 void deleteStream( int iId ); 115 void erase( StreamId iStream );
129 116 void setSize( StreamId iStream, int32_t iNewSize );
117 int32_t getSize( StreamId iStream ) const;
118 bool exists( StreamId iStream ) const;
119 Bu::String getLocation() const;
120 int32_t getBlockSize() const;
121 int32_t getTotalBlocks() const;
122 int32_t getUsedBlocks() const;
123 int32_t getFreeBlocks() const;
124 int32_t getTotalStreams() const;
125 int32_t getTotalUsedBytes() const;
126 int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
127 Bu::BitString buildBlockUseMap() const;
128 StreamIdArray buildBlockMap() const;
129
130 /** 130 /**
131 * Return a new Stream object assosiated with the given stream ID. 131 * Lists all stream ids that you are allowed to open. Technically there
132 * is always a zero stream, but it is used by Myriad for stream/block
133 * accounting. It works like a normal stream but you should not open
134 * it.
132 */ 135 */
133 MyriadStream openStream( int iId ); 136 StreamIdList getStreamList() const;
134
135 Bu::Array<int> getStreamIds();
136 int getStreamSize( int iId );
137 bool hasStream( int iId );
138
139 int getNumStreams();
140 int getBlockSize();
141 int getNumBlocks();
142 int getNumUsedBlocks();
143 Bu::size getTotalUsedBytes();
144 Bu::size getTotalUnusedBytes();
145 Bu::size getTotalUnusedBytes( int iFakeBlockSize );
146 137
147 /** 138 /**
148 * Syncronize the header data, etc. with the storage stream. It's not 139 * Flush all caches to the backing stream, write all structural and
149 * a bad idea to call this periodically. 140 * header changes.
150 */ 141 */
151 void sync(); 142 void sync();
152 143
144 private:
145 bool loadMyriad();
146 void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks );
147 void writeHeader();
148 int32_t __calcHeaderSize();
149 int32_t allocateBlock();
150 int32_t __allocateBlock();
151 void releaseBlock( int32_t iBlockId, bool bBlank=true );
152 void __releaseBlock( int32_t iBlockId, bool bBlank=true );
153 void blankBlock( int32_t iBlockId );
154
155 void openStream( StreamId id );
156 void closeStream( StreamId id );
153 /** 157 /**
154 * Read the first few bytes from the given stream and return true/false 158 * Block restricted read, it will not read past the end of the block
155 * depending on weather or not it's a Myriad stream. This will throw 159 * that iStart places it in.
156 * an exception if the stream is empty, or is not random access.
157 */ 160 */
158 static bool isMyriad( Bu::Stream &sStore, uint8_t &uVer ); 161 int32_t blockRead( int32_t iBlock, int32_t iStart,
159 162 void *pTarget, int32_t iSize );
163
160 /** 164 /**
161 * Read the first few bytes from the given stream and return true/false 165 * Block restricted write, it will not write past the end of the block
162 * depending on weather or not it's a Myriad stream. This will throw 166 * that iStart places it in. If this returns a non-zero number it's an
163 * an exception if the stream is empty, or is not random access. 167 * indication that you need to allocate a new block.
164 */ 168 */
165 static bool isMyriad( Bu::Stream &sStore ); 169 int32_t blockWrite( int32_t iBlock, int32_t iStart,
170 const void *pTarget, int32_t iSize );
166 171
167 const Bu::BitString getBlocksUsed() const; 172 public:
168
169 private:
170 /** 173 /**
171 * Initialize this object based on the data already in the assosiated 174 * Bridge/communication/tracking class for individual Myriad streams.
172 * stream. This will be called automatically for you if you forget, 175 * Not for general use, this is used by Myriad and MyriadStream to
173 * but if you want to pre-initialize for some reason, just call this 176 * control access.
174 * once before you actually start doing anything with your Myriad.
175 */ 177 */
176 void initialize();
177
178 enum
179 {
180 blockUnused = 0xFFFFFFFFUL
181 };
182
183 typedef Bu::Array<int> BlockArray;
184 class Stream 178 class Stream
185 { 179 {
186 public: 180 friend Bu::Myriad;
187 int iId; 181 private:
188 int iSize; 182 Stream( Myriad &rParent, StreamId iStream, int32_t iSize );
189 BlockArray aBlocks; 183 virtual ~Stream();
190 };
191 typedef Bu::Array<Stream *> StreamArray;
192 184
193 class Block
194 {
195 public: 185 public:
196 char *pData; 186 int32_t getSize() const;
197 bool bChanged; 187 int32_t getBlockSize() const;
198 int iBlockIndex; 188 StreamId getStreamId() const;
189 int32_t getOpenCount() const;
190
191 void setSize( int32_t iNewSize );
192 int32_t read( int32_t iStart, void *pTarget, int32_t iSize );
193 int32_t write( int32_t iStart, const void *pTarget, int32_t iSize );
194 Bu::String getLocation() const;
195 Bu::Array<int32_t> getBlockList() const;
196
197 /**
198 * Doesn't actually open, just increments the open counter.
199 * If the open counter is non-zero then at least one stream has
200 * a lock on this stream.
201 */
202 void open();
203
204 /**
205 * Doesn't actually close, just decrements the open counter.
206 *@returns true if there are still handles open, false if no
207 * streams have a lock.
208 */
209 bool close();
210
211 private:
212 mutable Bu::Mutex mAccess;
213 Myriad &rParent;
214 StreamId iStream;
215 int32_t iSize;
216 Bu::Array<int32_t> aBlocks;
217 int32_t iOpenCount;
199 }; 218 };
200 219
201 void updateHeader();
202 int findEmptyBlock();
203
204 /**
205 *@todo Change this to use a binary search, it's nicer.
206 */
207 Stream *findStream( int iId );
208
209 Block *getBlock( int iBlock );
210 void releaseBlock( Block *pBlock );
211 void syncBlock( Block *pBlock );
212
213 int streamAddBlock( Stream *pStream );
214 void setStreamSize( Stream *pStream, long iSize );
215
216 void headerChanged();
217
218 private: 220 private:
219 Bu::Stream &sStore; 221 typedef Bu::Hash<StreamId, Stream *> StreamHash;
220 int iBlockSize; 222 typedef Bu::List<int32_t> IndexList;
221 int iBlocks; 223 mutable Bu::Mutex mAccess;
222 int iUsed; 224 mutable Bu::Mutex mBacking;
223 typedef Bu::List<int> IndexList; 225 Bu::Stream &rBacking;
226 int32_t iBlockSize;
227 int32_t iBlockCount;
228 bool bIsNewStream;
229 bool bStructureChanged;
230 mutable Bu::Mutex mhStream;
231 StreamHash hStream;
224 IndexList lFreeBlocks; 232 IndexList lFreeBlocks;
225// Bu::BitString bsBlockUsed; 233 StreamId iLastUsedIndex;
226 StreamArray aStreams;
227 typedef Bu::Hash<int, Block *> BlockHash;
228 BlockHash hActiveBlocks;
229 bool bHeaderChanged;
230
231 Bu::Mutex mHeader;
232 Bu::Mutex mActiveBlocks;
233 }; 234 };
235 constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b )
236 {
237 return static_cast<Myriad::Mode>(
238 static_cast<std::underlying_type<Myriad::Mode>::type>(a) &
239 static_cast<std::underlying_type<Myriad::Mode>::type>(b)
240 );
241 }
242 constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b )
243 {
244 return static_cast<Myriad::Mode>(
245 static_cast<std::underlying_type<Myriad::Mode>::type>(a) |
246 static_cast<std::underlying_type<Myriad::Mode>::type>(b)
247 );
248 }
234}; 249};
235 250
236#endif 251#endif
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
index 50c6924..eaf91a5 100644
--- a/src/stable/myriadstream.cpp
+++ b/src/stable/myriadstream.cpp
@@ -1,248 +1,96 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriadstream.h" 1#include "bu/myriadstream.h"
9 2
10#include <string.h> 3#include "bu/mutexlocker.h"
11
12// #define MYRIAD_STREAM_DEBUG 1
13
14#ifdef MYRIAD_STREAM_DEBUG
15#include "bu/sio.h"
16
17using Bu::sio;
18using Bu::Fmt;
19#endif
20#include "bu/sio.h"
21
22#define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(rMyriad.sStore.getLocation()).arg(x)
23 4
24Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad, 5Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad,
25 Bu::Myriad::Stream *pStream ) : 6 Bu::Myriad::Stream *pStream, Bu::Myriad::Mode eMode ) :
26 rMyriad( rMyriad ), 7 rMyriad( rMyriad ),
27 pStream( pStream ), 8 pStream( pStream ),
28 pCurBlock( NULL ), 9 eMode( eMode ),
29 iPos( 0 ) 10 iPos( 0 )
30{ 11{
31#ifdef MYRIAD_STREAM_DEBUG 12 if( (eMode&Bu::Myriad::ReadWrite) == 0 )
32 sio << "MyriadStream: " << __LINE__ << ": Created, iId=" << pStream->iId << ", iSize=" 13 {
33 << pStream->iSize << sio.nl; 14 throw Bu::MyriadException( Bu::MyriadException::invalidParameter,
34#endif 15 "MyriadStream must be opened Read or Write or both.");
35 //pCurBlock = rMyriad.newBlock(); 16 }
36 //rMyriad.getBlock( uStream, pCurBlock ); 17 Bu::MutexLocker l( mAccess );
37 //uSize = pCurBlock->uBytesUsed; 18 pStream->open();
19
20 if( (eMode&Bu::Myriad::Write) != 0 )
21 {
22 // Writing mode, what other options do we deal with?
23 if( (eMode&Bu::Myriad::Truncate) != 0 )
24 {
25 // Truncate, set size to zero before starting.
26 pStream->setSize( 0 );
27 }
28 else if( (eMode&Bu::Myriad::Append) != 0 )
29 {
30 iPos = pStream->getSize();
31 }
32 }
38} 33}
39 34
40Bu::MyriadStream::~MyriadStream() 35Bu::MyriadStream::~MyriadStream()
41{ 36{
42 if( pCurBlock ) 37 close();
43 rMyriad.releaseBlock( pCurBlock );
44 //rMyriad.updateStreamSize( uStream, uSize );
45 //rMyriad.deleteBlock( pCurBlock );
46} 38}
47 39
48void Bu::MyriadStream::close() 40void Bu::MyriadStream::close()
49{ 41{
50} 42 Bu::MutexLocker l( mAccess );
51 43 if( eMode )
52Bu::size Bu::MyriadStream::read( void *pBuf, Bu::size nBytes )
53{
54#ifdef MYRIAD_STREAM_DEBUG
55 sio << "MyriadStream: read: " << __LINE__ << ": Started, asked to read " << nBytes << "b."
56 << sio.nl;
57#endif
58 if( nBytes > (Bu::size)pStream->iSize-iPos )
59 nBytes = pStream->iSize-iPos;
60 if( nBytes <= 0 )
61 return 0;
62 int iLeft = nBytes;
63#ifdef MYRIAD_STREAM_DEBUG
64 sio << "MyriadStream: read: " << __LINE__ << ": Started, going to read " << nBytes << "b."
65 << sio.nl;
66#endif
67 if( pCurBlock == NULL )
68 { 44 {
69#ifdef MYRIAD_STREAM_DEBUG 45 pStream->close();
70 sio << "MyriadStream: read: " << __LINE__ << ": No block loaded, loading initial block." 46 eMode = Bu::Myriad::None;
71 << sio.nl;
72#endif
73 pCurBlock = rMyriad.getBlock(
74 pStream->aBlocks[iPos/rMyriad.iBlockSize]
75 );
76 }
77 while( iLeft > 0 )
78 {
79 int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
80 if( pCurBlock->iBlockIndex != iCurBlock )
81 {
82#ifdef MYRIAD_STREAM_DEBUG
83 sio << "MyriadStream: read: " << __LINE__ << ": Loading new block " << iCurBlock << "."
84 << sio.nl;
85#endif
86 rMyriad.releaseBlock( pCurBlock );
87 pCurBlock = rMyriad.getBlock( iCurBlock );
88 }
89
90 int iAmnt = Bu::buMin(
91 Bu::buMin(
92 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
93 iLeft
94 ),
95 pStream->iSize-iPos
96 );
97#ifdef MYRIAD_STREAM_DEBUG
98 sio << "MyriadStream: read: " << __LINE__ << ": Copying out bytes: "
99 << iPos << "(" << (iPos%rMyriad.iBlockSize) << ")+"
100 << iAmnt
101 << ", " << iLeft << "b left." << sio.nl;
102#endif
103 memcpy(
104 pBuf,
105 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
106 iAmnt
107 );
108 iPos += iAmnt;
109 pBuf = &((char *)pBuf)[iAmnt];
110 iLeft -= iAmnt;
111 } 47 }
112 return nBytes;
113} 48}
114 49
115Bu::size Bu::MyriadStream::write( const void *pBuf, Bu::size nBytes ) 50Bu::size Bu::MyriadStream::read( void *pBuf, size iBytes )
116{ 51{
117 if( nBytes <= 0 ) 52 Bu::MutexLocker l( mAccess );
118 return 0; 53 int32_t iRead = pStream->read( iPos, pBuf, iBytes );
119 54 iPos += iRead;
120#ifdef MYRIAD_STREAM_DEBUG 55 return iRead;
121 sio << "MyriadStream: write: " << __LINE__ << ": Started, asked to write " << nBytes << "b." 56}
122 << sio.nl;
123#endif
124 if( nBytes <= 0 )
125 return 0;
126 int iLeft = nBytes;
127 /*
128 if( pCurBlock == NULL )
129 {
130#ifdef MYRIAD_STREAM_DEBUG
131 sio << "MyriadStream: write: No block loaded, loading initial block."
132 << sio.nl;
133#endif
134 pCurBlock = rMyriad.getBlock(
135 pStream->aBlocks[iPos/rMyriad.iBlockSize]
136 );
137 }*/
138
139 while( iLeft > 0 )
140 {
141 int iCurBlock;
142 if( iPos/rMyriad.iBlockSize < pStream->aBlocks.getSize() )
143 {
144 iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
145 }
146 else
147 {
148 iCurBlock = rMyriad.streamAddBlock( pStream );
149#ifdef MYRIAD_STREAM_DEBUG
150 sio << "MyriadStream: write: " << __LINE__ << ": New block allocated and appended: "
151 << iCurBlock << "." << sio.nl;
152
153#endif
154 }
155 if( !pCurBlock || pCurBlock->iBlockIndex != iCurBlock )
156 {
157#ifdef MYRIAD_STREAM_DEBUG
158 sio << "MyriadStream: write: " << __LINE__ << ": Loading new block " << iCurBlock << "."
159 << sio.nl;
160#endif
161 rMyriad.releaseBlock( pCurBlock );
162 pCurBlock = rMyriad.getBlock( iCurBlock );
163 }
164 pCurBlock->bChanged = true;
165
166 // There are two main writing modes when it comes down to it.
167 // Overwrite mode and append mode. Append is what pretty much always
168 // happens when creating a new stream.
169 if( iPos < pStream->iSize )
170 {
171 int iAmnt = Bu::buMin(
172 Bu::buMin(
173 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
174 iLeft
175 ),
176 pStream->iSize-iPos
177 );
178#ifdef MYRIAD_STREAM_DEBUG
179 sio << "MyriadStream: write (ovr): " << __LINE__ << ": Copying in bytes: "
180 << (iPos%rMyriad.iBlockSize) << "+"
181 << iAmnt
182 << ", " << iLeft << "b left." << sio.nl;
183#endif
184 memcpy(
185 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
186 pBuf,
187 iAmnt
188 );
189 iPos += iAmnt;
190 pBuf = &((char *)pBuf)[iAmnt];
191 iLeft -= iAmnt;
192 }
193 else
194 {
195 int iAmnt = Bu::buMin(
196 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
197 iLeft
198 );
199#ifdef MYRIAD_STREAM_DEBUG
200 sio << "MyriadStream: write (app): " << __LINE__ << ": Copying in bytes: "
201 << (iPos%rMyriad.iBlockSize) << "+"
202 << iAmnt
203 << ", " << iLeft << "b left." << sio.nl;
204#endif
205 memcpy(
206 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
207 pBuf,
208 iAmnt
209 );
210 iPos += iAmnt;
211 TRACE(Bu::String("Stream=%1 - pStream->iSize(%2) += iAmnt(%3)").arg(pStream->iId).arg( pStream->iSize ).arg(iAmnt).end());
212 pStream->iSize += iAmnt;
213 TRACE(Bu::String("Stream=%1 - pStream->iSize = %2").arg(pStream->iId).arg( pStream->iSize ).end());
214 rMyriad.headerChanged();
215 pBuf = &((char *)pBuf)[iAmnt];
216 iLeft -= iAmnt;
217 }
218 }
219 57
220 return nBytes; 58Bu::size Bu::MyriadStream::write( const void *pBuf, size iBytes )
59{
60 Bu::MutexLocker l( mAccess );
61 int32_t iWrite = pStream->write( iPos, pBuf, iBytes );
62 iPos += iWrite;
63 return iWrite;
221} 64}
222 65
223Bu::size Bu::MyriadStream::tell() 66Bu::size Bu::MyriadStream::tell()
224{ 67{
68 Bu::MutexLocker l( mAccess );
225 return iPos; 69 return iPos;
226} 70}
227 71
228void Bu::MyriadStream::seek( Bu::size offset ) 72void Bu::MyriadStream::seek( size offset )
229{ 73{
74 Bu::MutexLocker l( mAccess );
230 iPos += offset; 75 iPos += offset;
231} 76}
232 77
233void Bu::MyriadStream::setPos( Bu::size pos ) 78void Bu::MyriadStream::setPos( size pos )
234{ 79{
80 Bu::MutexLocker l( mAccess );
235 iPos = pos; 81 iPos = pos;
236} 82}
237 83
238void Bu::MyriadStream::setPosEnd( Bu::size pos ) 84void Bu::MyriadStream::setPosEnd( size pos )
239{ 85{
240 iPos = pStream->iSize-pos; 86 Bu::MutexLocker l( mAccess );
87 iPos = pStream->getSize()-pos;
241} 88}
242 89
243bool Bu::MyriadStream::isEos() 90bool Bu::MyriadStream::isEos()
244{ 91{
245 return iPos >= pStream->iSize; 92 Bu::MutexLocker l( mAccess );
93 return iPos == pStream->getSize();
246} 94}
247 95
248bool Bu::MyriadStream::isOpen() 96bool Bu::MyriadStream::isOpen()
@@ -252,6 +100,7 @@ bool Bu::MyriadStream::isOpen()
252 100
253void Bu::MyriadStream::flush() 101void Bu::MyriadStream::flush()
254{ 102{
103 // Does this make sense?
255} 104}
256 105
257bool Bu::MyriadStream::canRead() 106bool Bu::MyriadStream::canRead()
@@ -266,12 +115,14 @@ bool Bu::MyriadStream::canWrite()
266 115
267bool Bu::MyriadStream::isReadable() 116bool Bu::MyriadStream::isReadable()
268{ 117{
269 return true; 118 Bu::MutexLocker l( mAccess );
119 return (eMode&Bu::Myriad::Read) != 0;
270} 120}
271 121
272bool Bu::MyriadStream::isWritable() 122bool Bu::MyriadStream::isWritable()
273{ 123{
274 return true; 124 Bu::MutexLocker l( mAccess );
125 return (eMode&Bu::Myriad::Write) != 0;
275} 126}
276 127
277bool Bu::MyriadStream::isSeekable() 128bool Bu::MyriadStream::isSeekable()
@@ -286,29 +137,38 @@ bool Bu::MyriadStream::isBlocking()
286 137
287void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ ) 138void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ )
288{ 139{
140 // Dunno what this would even mean here.
289} 141}
290 142
291void Bu::MyriadStream::setSize( Bu::size iSize ) 143void Bu::MyriadStream::setSize( size iSize )
292{ 144{
293 if( iSize < 0 ) 145 Bu::MutexLocker l( mAccess );
294 iSize = 0; 146 pStream->setSize( iSize );
295 rMyriad.setStreamSize( pStream, iSize );
296 if( iPos > iSize ) 147 if( iPos > iSize )
297 iPos = iSize; 148 iPos = iSize;
298} 149}
299 150
300Bu::size Bu::MyriadStream::getSize() const 151Bu::size Bu::MyriadStream::getSize() const
301{ 152{
302 return pStream->iSize; 153 Bu::MutexLocker l( mAccess );
154 return pStream->getSize();
303} 155}
304 156
305Bu::size Bu::MyriadStream::getBlockSize() const 157Bu::size Bu::MyriadStream::getBlockSize() const
306{ 158{
307 return rMyriad.getBlockSize(); 159 Bu::MutexLocker l( mAccess );
160 return pStream->getBlockSize();
308} 161}
309 162
310Bu::String Bu::MyriadStream::getLocation() const 163Bu::String Bu::MyriadStream::getLocation() const
311{ 164{
312 return Bu::String("%1").arg( pStream->iId ); 165 Bu::MutexLocker l( mAccess );
166 return pStream->getLocation();
167}
168
169Bu::Myriad::StreamId Bu::MyriadStream::getId() const
170{
171 Bu::MutexLocker l( mAccess );
172 return pStream->getStreamId();
313} 173}
314 174
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
index a94a9a2..27a15d5 100644
--- a/src/stable/myriadstream.h
+++ b/src/stable/myriadstream.h
@@ -1,38 +1,30 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_STREAM_H 1#ifndef BU_MYRIAD_STREAM_H
9#define BU_MYRIAD_STREAM_H 2#define BU_MYRIAD_STREAM_H
10 3
11#include "bu/stream.h" 4#include "bu/stream.h"
5#include "bu/myriadstream.h"
12#include "bu/myriad.h" 6#include "bu/myriad.h"
13 7
14namespace Bu 8namespace Bu
15{ 9{
16 class MyriadStream : public Bu::Stream 10 class MyriadStream : public Bu::Stream
17 { 11 {
18 friend class Myriad; 12 friend class Myriad;
19 private: 13 private:
20 /** 14 MyriadStream( Bu::Myriad &rMyriad, Bu::Myriad::Stream *pStream,
21 * These can only be created by the Myriad class. 15 Bu::Myriad::Mode eMode );
22 */
23 MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream );
24
25 public: 16 public:
26 virtual ~MyriadStream(); 17 virtual ~MyriadStream();
27 18
19 public:
28 virtual void close(); 20 virtual void close();
29 virtual Bu::size read( void *pBuf, Bu::size nBytes ); 21 virtual size read( void *pBuf, size iBytes );
30 virtual Bu::size write( const void *pBuf, Bu::size nBytes ); 22 virtual size write( const void *pBuf, size iBytes );
31 using Stream::write; 23 using Stream::write;
32 virtual Bu::size tell(); 24 virtual size tell();
33 virtual void seek( Bu::size offset ); 25 virtual void seek( size offset );
34 virtual void setPos( Bu::size pos ); 26 virtual void setPos( size pos );
35 virtual void setPosEnd( Bu::size pos ); 27 virtual void setPosEnd( size pos );
36 virtual bool isEos(); 28 virtual bool isEos();
37 virtual bool isOpen(); 29 virtual bool isOpen();
38 virtual void flush(); 30 virtual void flush();
@@ -43,18 +35,18 @@ namespace Bu
43 virtual bool isSeekable(); 35 virtual bool isSeekable();
44 virtual bool isBlocking(); 36 virtual bool isBlocking();
45 virtual void setBlocking( bool bBlocking=true ); 37 virtual void setBlocking( bool bBlocking=true );
46 virtual void setSize( Bu::size iSize ); 38 virtual void setSize( size iSize );
47
48 virtual size getSize() const; 39 virtual size getSize() const;
49 virtual size getBlockSize() const; 40 virtual size getBlockSize() const;
50 virtual Bu::String getLocation() const; 41 virtual Bu::String getLocation() const;
42 Myriad::StreamId getId() const;
51 43
52 private: 44 private:
53 Myriad &rMyriad; 45 mutable Bu::Mutex mAccess;
54 Myriad::Stream *pStream; 46 Bu::Myriad &rMyriad;
55 Myriad::Block *pCurBlock; 47 Bu::Myriad::Stream *pStream;
56 int iBlockSize; 48 Bu::Myriad::Mode eMode;
57 int iPos; 49 int32_t iPos;
58 }; 50 };
59}; 51};
60 52
diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp
index 9af301c..73a3315 100644
--- a/src/tests/bigmyriad.cpp
+++ b/src/tests/bigmyriad.cpp
@@ -5,16 +5,17 @@
5int main() 5int main()
6{ 6{
7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create ); 7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create );
8 Bu::Myriad m( f, 2048 ); 8 Bu::Myriad m( f, 512, 12 );
9 9
10 char *buf = new char[1024*1024*10]; 10#define SSIZE (1024*8) // 1024*1024*10
11 memset( buf, 0, 1024*1024*10 ); 11 char *buf = new char[SSIZE];
12 12
13 for( int j = 0; j < 250; j++ ) 13 for( int j = 0; j < 25; j++ )
14 { 14 {
15 m.openStream( m.createStream() ).write( buf, 1024*1024*10 ); 15 memset( buf, j, SSIZE );
16// m.sync(); 16 m.create( Bu::Myriad::Write ).write( buf, SSIZE );
17 printf("\r%03d%%", (j+1)*100/250 ); 17// m.sync();
18 printf("\r%03d%%", (j+1)*100/25 );
18 fflush( stdout ); 19 fflush( stdout );
19 } 20 }
20 21
diff --git a/src/tests/cachedel.cpp b/src/tests/cachedel.cpp
index 817757c..f4cb1b5 100644
--- a/src/tests/cachedel.cpp
+++ b/src/tests/cachedel.cpp
@@ -21,6 +21,12 @@ public:
21 21
22 virtual ~Something() 22 virtual ~Something()
23 { 23 {
24 //Bu::println("Deleting %1").arg( this->toString() );
25 }
26
27 void sayHi()
28 {
29 Bu::println("Hello %1").arg( toString() );
24 } 30 }
25 31
26 virtual Bu::Uuid getKey() const 32 virtual Bu::Uuid getKey() const
@@ -39,7 +45,7 @@ public:
39 changed(); 45 changed();
40 } 46 }
41 47
42 virtual Bu::String toString() const=0; 48 virtual Bu::String toString() const=0;// { return Bu::String("ERROR"); };
43 49
44private: 50private:
45 Bu::Uuid uId; 51 Bu::Uuid uId;
@@ -60,6 +66,11 @@ public:
60 iNumber( iNumber ) 66 iNumber( iNumber )
61 { 67 {
62 } 68 }
69
70 virtual ~SubSomethingA()
71 {
72 Bu::println("Deleting-A %1").arg( this->toString() );
73 }
63 74
64 virtual Bu::String toString() const 75 virtual Bu::String toString() const
65 { 76 {
@@ -84,6 +95,10 @@ public:
84 sString( sString ) 95 sString( sString )
85 { 96 {
86 } 97 }
98
99 virtual ~SubSomethingB()
100 {
101 }
87 102
88 virtual Bu::String toString() const 103 virtual Bu::String toString() const
89 { 104 {
@@ -187,6 +202,11 @@ int main( int, char *[] )
187 Bu::MemBuf mbStore; 202 Bu::MemBuf mbStore;
188 SomethingCache c( mbStore ); 203 SomethingCache c( mbStore );
189 204
205 {
206 SubSomethingA a("Test", 1);
207 a.sayHi();
208 }
209
190 SomethingPtr ptr; 210 SomethingPtr ptr;
191 if( time(NULL)%2 ) 211 if( time(NULL)%2 )
192 ptr = c.insert( new SubSomethingA("Hello", 55) ).cast<Something>(); 212 ptr = c.insert( new SubSomethingA("Hello", 55) ).cast<Something>();
@@ -205,8 +225,13 @@ int main( int, char *[] )
205 225
206 SomethingPtr p2 = c.insert( new SubSomethingA("new test", 123) ).cast<Something>(); 226 SomethingPtr p2 = c.insert( new SubSomethingA("new test", 123) ).cast<Something>();
207 id = p2.getKey(); 227 id = p2.getKey();
228 Bu::println("p2 %1: %2").arg( id ).arg( c.has( id ) );
208 p2.unbind(); 229 p2.unbind();
230 Bu::println("p2 %1: %2").arg( id ).arg( c.has( id ) );
209 c.erase( id ); 231 c.erase( id );
232 Bu::println("p2 %1: %2").arg( id ).arg( c.has( id ) );
233
234 Bu::println("Program listing over, leaving main scope.");
210 235
211 return 0; 236 return 0;
212} 237}
diff --git a/src/tests/myriad.cpp b/src/tests/myriad.cpp
new file mode 100644
index 0000000..ee4eac4
--- /dev/null
+++ b/src/tests/myriad.cpp
@@ -0,0 +1,23 @@
1#include <bu/myriad.h>
2#include <bu/myriadstream.h>
3#include <bu/string.h>
4#include <bu/sio.h>
5#include <bu/file.h>
6
7int main( int , char *[] )
8{
9 Bu::File fMyriad("test.myr", Bu::File::WriteNew|Bu::File::Read );
10 Bu::Myriad m( fMyriad, 32 );
11
12 Bu::MyriadStream ms = m.create( Bu::Myriad::ReadWrite );
13 ms.setSize( 150 );
14 ms.setPos( 145 );
15 char stuff[10];
16 int32_t iRead = ms.read( stuff, 10 );
17
18 Bu::println("Tried to read 10, expect 5, got %1").arg( iRead );
19
20
21 return 0;
22}
23
diff --git a/src/tests/myriadfs.cpp b/src/tests/myriadfs.cpp
index 1266e4b..29ac3d9 100644
--- a/src/tests/myriadfs.cpp
+++ b/src/tests/myriadfs.cpp
@@ -9,7 +9,7 @@ using namespace Bu;
9int main() 9int main()
10{ 10{
11// Bu::MemBuf mb; 11// Bu::MemBuf mb;
12 Bu::File mb("store.myr", File::Read|File::Write|File::Create ); 12 Bu::File mb("store.mfs", File::Read|File::Write|File::Create );
13 Bu::MyriadFs mfs( mb, 512 ); 13 Bu::MyriadFs mfs( mb, 512 );
14 14
15 sio << "Creating dirs..." << sio.nl; 15 sio << "Creating dirs..." << sio.nl;
diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp
index ccf3d3b..0dd9840 100644
--- a/src/tools/myriad.cpp
+++ b/src/tools/myriad.cpp
@@ -20,6 +20,7 @@ enum Mode
20 modeCreate, 20 modeCreate,
21 modeInfo, 21 modeInfo,
22 modeStreamNew, 22 modeStreamNew,
23 modeStreamErase,
23 modeStreamDump, 24 modeStreamDump,
24 modeStreamPut, 25 modeStreamPut,
25 modeStreamGet, 26 modeStreamGet,
@@ -44,6 +45,8 @@ public:
44 "Display some info about a Myriad file." ); 45 "Display some info about a Myriad file." );
45 addOption( eMode, 'n', "new", 46 addOption( eMode, 'n', "new",
46 "Create a new sub-stream in a Myriad file."); 47 "Create a new sub-stream in a Myriad file.");
48 addOption( eMode, 'e', "erase",
49 "Erase sub-stream in a Myriad file.");
47 addOption( eMode, 'd', "dump", 50 addOption( eMode, 'd', "dump",
48 "Display a hexdump of a stream from a Myriad file."); 51 "Display a hexdump of a stream from a Myriad file.");
49 addOption( eMode, "get", 52 addOption( eMode, "get",
@@ -67,6 +70,7 @@ public:
67 setOverride( "create", modeCreate ); 70 setOverride( "create", modeCreate );
68 setOverride( "info", modeInfo ); 71 setOverride( "info", modeInfo );
69 setOverride( "new", modeStreamNew ); 72 setOverride( "new", modeStreamNew );
73 setOverride( "erase", modeStreamErase );
70 setOverride( "dump", modeStreamDump ); 74 setOverride( "dump", modeStreamDump );
71 setOverride( "put", modeStreamPut ); 75 setOverride( "put", modeStreamPut );
72 setOverride( "get", modeStreamGet ); 76 setOverride( "get", modeStreamGet );
@@ -90,6 +94,49 @@ Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
90 return f; 94 return f;
91} 95}
92 96
97void printMap( const Bu::BitString &bs )
98{
99 for( int j = 0; j < bs.getSize(); j++ )
100 {
101 if( j>0 && (j%50) == 0 )
102 Bu::println("");
103 if( bs.getBit( j ) )
104 Bu::print("#");
105 else
106 Bu::print("-");
107 }
108 Bu::println("\n");
109}
110
111void printMap( const Bu::Array<int32_t> &bm )
112{
113 int iBigest = 0;
114 for( int j = 0; j < bm.getSize(); j++ )
115 {
116 if( iBigest < bm[j] )
117 iBigest = bm[j];
118 }
119 int iWidth = Bu::String("%1").arg( iBigest ).end().getSize();
120 Bu::String sEmpty;
121 for( int j = 0; j < iWidth; j++ )
122 {
123 sEmpty += '-';
124 }
125 int iBreakAt = 60/(iWidth+1);
126 for( int j = 0; j < bm.getSize(); j++ )
127 {
128 if( j>0 && (j%iBreakAt) == 0 )
129 Bu::println("");
130
131 if( bm[j] < 0 )
132 Bu::print("%1 ").arg( sEmpty, Bu::Fmt(2).right().fill(' '));
133 else
134 Bu::print("%1 ").arg( bm[j], Bu::Fmt(2).right().fill(' '));
135
136 }
137 Bu::println("\n");
138}
139
93int main( int argc, char *argv[] ) 140int main( int argc, char *argv[] )
94{ 141{
95 Options opts( argc, argv ); 142 Options opts( argc, argv );
@@ -121,14 +168,14 @@ int main( int argc, char *argv[] )
121 Myriad m( fIn ); 168 Myriad m( fIn );
122 sio << "Myriad info:" << sio.nl 169 sio << "Myriad info:" << sio.nl
123 << " Block size: " << m.getBlockSize() << sio.nl 170 << " Block size: " << m.getBlockSize() << sio.nl
124 << " Block count: " << m.getNumBlocks() << sio.nl 171 << " Block count: " << m.getTotalBlocks() << sio.nl
125 << " Blocks used: " << m.getNumUsedBlocks() << " (" 172 << " Blocks used: " << m.getUsedBlocks() << " ("
126 << m.getNumUsedBlocks()*100/m.getNumBlocks() << "%)" 173 << m.getUsedBlocks()*100/m.getTotalBlocks() << "%)"
127 << sio.nl 174 << sio.nl
128 << " Stream count: " << m.getNumStreams() << sio.nl 175 << " Stream count: " << m.getTotalStreams() << sio.nl
129 << " Used space: " << m.getTotalUsedBytes() << sio.nl 176 << " Used space: " << m.getTotalUsedBytes() << sio.nl
130 << " Unused space: " << m.getTotalUnusedBytes() << sio.nl 177 << " Unused space: " << m.getTotalUnusedBytes() << sio.nl
131 << " % of files: " << (double)(m.getNumBlocks()*m.getBlockSize())/(double)(m.getTotalUsedBytes() + m.getTotalUnusedBytes( 4096 ))*100.0 << sio.nl; 178 << " % of files: " << (double)(m.getTotalBlocks()*m.getBlockSize())/(double)(m.getTotalUsedBytes() + m.getTotalUnusedBytes( 4096 ))*100.0 << sio.nl;
132/* Bu::Array<int> aStreams = m.getStreamIds(); 179/* Bu::Array<int> aStreams = m.getStreamIds();
133 sio << " Stream info:" << sio.nl; 180 sio << " Stream info:" << sio.nl;
134 for( Bu::Array<int>::iterator i = aStreams.begin(); i; i++ ) 181 for( Bu::Array<int>::iterator i = aStreams.begin(); i; i++ )
@@ -149,7 +196,23 @@ int main( int argc, char *argv[] )
149 { 196 {
150 File fOut( opts.sFile, File::Write|File::Read ); 197 File fOut( opts.sFile, File::Write|File::Read );
151 Myriad m( fOut ); 198 Myriad m( fOut );
152 m.createStream( opts.iPreallocate ); 199 m.create( Bu::Myriad::WriteNew, opts.iPreallocate );
200 }
201 break;
202
203 case modeStreamErase:
204 if( !opts.sFile.isSet() )
205 {
206 sio << "Please specify a file manipulate." << sio.nl;
207 return 0;
208 }
209 else
210 {
211 File fOut( opts.sFile, File::Write|File::Read );
212 Myriad m( fOut );
213 m.erase( opts.iStream );
214 printMap( m.buildBlockUseMap() );
215 printMap( m.buildBlockMap() );
153 } 216 }
154 break; 217 break;
155 218
@@ -163,7 +226,7 @@ int main( int argc, char *argv[] )
163 { 226 {
164 File fOut( opts.sFile, File::Read ); 227 File fOut( opts.sFile, File::Read );
165 Myriad m( fOut ); 228 Myriad m( fOut );
166 MyriadStream s = m.openStream( opts.iStream ); 229 MyriadStream s = m.open( opts.iStream, Bu::Myriad::Read );
167 sio << "Stream " << opts.iStream << ":" << sio.nl; 230 sio << "Stream " << opts.iStream << ":" << sio.nl;
168 char buf[8]; 231 char buf[8];
169 int iPos = 0; 232 int iPos = 0;
@@ -210,8 +273,8 @@ int main( int argc, char *argv[] )
210 { 273 {
211 File fOut( opts.sFile, File::Write|File::Read ); 274 File fOut( opts.sFile, File::Write|File::Read );
212 Myriad m( fOut ); 275 Myriad m( fOut );
213 MyriadStream sOut = m.openStream( 276 MyriadStream sOut = m.create(
214 m.createStream( opts.iPreallocate ) 277 Bu::Myriad::WriteNew, opts.iPreallocate
215 ); 278 );
216 File fIn( opts.sSrc, File::Read ); 279 File fIn( opts.sSrc, File::Read );
217 char buf[1024]; 280 char buf[1024];
@@ -236,7 +299,7 @@ int main( int argc, char *argv[] )
236 { 299 {
237 File fIn( opts.sFile, File::Write|File::Read ); 300 File fIn( opts.sFile, File::Write|File::Read );
238 Myriad m( fIn ); 301 Myriad m( fIn );
239 MyriadStream sIn = m.openStream( opts.iStream ); 302 MyriadStream sIn = m.open( opts.iStream, Bu::Myriad::Read );
240 File fOut( opts.sDst, File::Write|File::Create|File::Truncate ); 303 File fOut( opts.sDst, File::Write|File::Create|File::Truncate );
241 char buf[1024]; 304 char buf[1024];
242 while( !sIn.isEos() ) 305 while( !sIn.isEos() )
@@ -255,17 +318,8 @@ int main( int argc, char *argv[] )
255 { 318 {
256 File fIn( opts.sFile, File::Write|File::Read ); 319 File fIn( opts.sFile, File::Write|File::Read );
257 Myriad m( fIn ); 320 Myriad m( fIn );
258 Bu::BitString bs = m.getBlocksUsed(); 321 printMap( m.buildBlockUseMap() );
259 for( int j = 0; j < bs.getSize(); j++ ) 322 printMap( m.buildBlockMap() );
260 {
261 if( j>0 && (j%50) == 0 )
262 Bu::println("");
263 if( bs.getBit( j ) )
264 Bu::print("#");
265 else
266 Bu::print("-");
267 }
268 Bu::println("\n");
269 } 323 }
270 break; 324 break;
271 325
@@ -277,4 +331,3 @@ int main( int argc, char *argv[] )
277 331
278 return 0; 332 return 0;
279} 333}
280
diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit
index 24d3116..f7bea97 100644
--- a/src/unit/myriad.unit
+++ b/src/unit/myriad.unit
@@ -109,7 +109,7 @@ suite Myriad
109 File fMyriad = tempFile( sFileName ); 109 File fMyriad = tempFile( sFileName );
110 Myriad m( fMyriad, 32 ); 110 Myriad m( fMyriad, 32 );
111 111
112 MyriadStream ms = m.openStream( m.createStream() ); 112 MyriadStream ms = m.create( Myriad::ReadWrite );
113 ms.setSize( 150 ); 113 ms.setSize( 150 );
114 ms.setPos( 145 ); 114 ms.setPos( 145 );
115 char stuff[10]; 115 char stuff[10];
@@ -163,7 +163,7 @@ suite Myriad
163 if( iSize < 8 || iSize > 1024 ) 163 if( iSize < 8 || iSize > 1024 )
164 throw ExceptionBase("Read bad data, %d", iSize ); 164 throw ExceptionBase("Read bad data, %d", iSize );
165 char *buf = new char[iSize-8]; 165 char *buf = new char[iSize-8];
166 if( s.read( buf, iSize-8 ) < (size_t)iSize-8 ) 166 if( s.read( buf, iSize-8 ) < (Bu::size)iSize-8 )
167 { 167 {
168 delete[] buf; 168 delete[] buf;
169 throw ExceptionBase("Block failed verify (insuffient block data)."); 169 throw ExceptionBase("Block failed verify (insuffient block data).");
@@ -197,13 +197,12 @@ suite Myriad
197 String sFileName("myriad-XXXXXXX"); 197 String sFileName("myriad-XXXXXXX");
198 198
199 File fMyriad = tempFile( sFileName ); 199 File fMyriad = tempFile( sFileName );
200 Myriad m( fMyriad ); 200 Myriad m( fMyriad, 64 );
201 m.initialize( 64 );
202 201
203 Array<int> aStreams; 202 Array<int> aStreams;
204 for( int j = 0; j < 5; j++ ) 203 for( int j = 0; j < 5; j++ )
205 { 204 {
206 aStreams.append( m.createStream() ); 205 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
207 } 206 }
208 207
209 srandom( 512 ); 208 srandom( 512 );
@@ -213,7 +212,7 @@ suite Myriad
213 switch( random()%5 ) 212 switch( random()%5 )
214 { 213 {
215 case 0: 214 case 0:
216 aStreams.append( m.createStream() ); 215 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
217 break; 216 break;
218 217
219 case 1: 218 case 1:
@@ -221,10 +220,10 @@ suite Myriad
221 { 220 {
222 int iStream = random()%aStreams.getSize(); 221 int iStream = random()%aStreams.getSize();
223 { 222 {
224 MyriadStream ms = m.openStream( aStreams[iStream] ); 223 MyriadStream ms = m.open( aStreams[iStream], Myriad::Read );
225 verifyStream( ms ); 224 verifyStream( ms );
226 } 225 }
227 m.deleteStream( aStreams[iStream] ); 226 m.erase( aStreams[iStream] );
228 Array<int>::iterator i = aStreams.begin(); 227 Array<int>::iterator i = aStreams.begin();
229 for( int k = 0; k < iStream; k++ ) 228 for( int k = 0; k < iStream; k++ )
230 i++; 229 i++;
@@ -235,11 +234,13 @@ suite Myriad
235 default: 234 default:
236 if( aStreams.getSize() == 0 ) 235 if( aStreams.getSize() == 0 )
237 { 236 {
238 aStreams.append( m.createStream() ); 237 aStreams.append(
238 m.create( Bu::Myriad::Read ).getId()
239 );
239 } 240 }
240 { 241 {
241 int iStream = random()%aStreams.getSize(); 242 int iStream = random()%aStreams.getSize();
242 MyriadStream ms = m.openStream( aStreams[iStream] ); 243 MyriadStream ms = m.open( aStreams[iStream], Myriad::ReadWrite );
243 addBlock( ms ); 244 addBlock( ms );
244 verifyStream( ms ); 245 verifyStream( ms );
245 } 246 }
@@ -249,7 +250,7 @@ suite Myriad
249 250
250 for( Array<int>::iterator i = aStreams.begin(); i; i++ ) 251 for( Array<int>::iterator i = aStreams.begin(); i; i++ )
251 { 252 {
252 MyriadStream ms = m.openStream( *i ); 253 MyriadStream ms = m.open( *i, Myriad::Read );
253 verifyStream( ms ); 254 verifyStream( ms );
254 } 255 }
255 } 256 }
@@ -259,14 +260,13 @@ suite Myriad
259 String sFileName("myriad-XXXXXXX"); 260 String sFileName("myriad-XXXXXXX");
260 261
261 File fMyriad = tempFile( sFileName ); 262 File fMyriad = tempFile( sFileName );
262 Myriad m( fMyriad ); 263 Myriad m( fMyriad, 128 );
263 m.initialize( 128 );
264 264
265 Array<int> aStream; 265 Array<int> aStream;
266 266
267 for( int j = 0; j < 5; j++ ) 267 for( int j = 0; j < 5; j++ )
268 { 268 {
269 aStream.append( m.createStream() ); 269 aStream.append( m.create( Bu::Myriad::Read ).getId() );
270 } 270 }
271 271
272 srandom( 1024 ); 272 srandom( 1024 );
@@ -276,7 +276,7 @@ suite Myriad
276 { 276 {
277 for( Array<int>::iterator i = aStream.begin(); i; i++ ) 277 for( Array<int>::iterator i = aStream.begin(); i; i++ )
278 { 278 {
279 MyriadStream ms = m.openStream( *i ); 279 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
280 addBlock( ms, false ); 280 addBlock( ms, false );
281 ms.setSize( ms.tell() ); 281 ms.setSize( ms.tell() );
282 unitTest( ms.read( &b, 1 ) == 0 ); 282 unitTest( ms.read( &b, 1 ) == 0 );
@@ -301,7 +301,7 @@ suite Myriad
301 301
302 for( int j = 0; j < 5; j++ ) 302 for( int j = 0; j < 5; j++ )
303 { 303 {
304 aStream.append( m.createStream() ); 304 aStream.append( m.create( Bu::Myriad::Read ).getId() );
305 incProgress(); 305 incProgress();
306 } 306 }
307 } 307 }
@@ -315,7 +315,7 @@ suite Myriad
315 Myriad m( fMyriad ); 315 Myriad m( fMyriad );
316 for( Array<int>::iterator i = aStream.begin(); i; i++ ) 316 for( Array<int>::iterator i = aStream.begin(); i; i++ )
317 { 317 {
318 MyriadStream ms = m.openStream( *i ); 318 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
319 addBlock( ms, false ); 319 addBlock( ms, false );
320 ms.setSize( ms.tell() ); 320 ms.setSize( ms.tell() );
321 unitTest( ms.read( &b, 1 ) == 0 ); 321 unitTest( ms.read( &b, 1 ) == 0 );
@@ -342,17 +342,17 @@ suite Myriad
342 342
343 for( int j = 0; j < 15; j++ ) 343 for( int j = 0; j < 15; j++ )
344 { 344 {
345 int iStream = m.createStream(); 345 MyriadStream ms = m.create( Myriad::Write );
346 int iStream = ms.getId();
346 aStream.append( iStream ); 347 aStream.append( iStream );
347 VerifyObject vo( random()%1024 ); 348 VerifyObject vo( random()%1024 );
348 { 349 {
349 MyriadStream ms = m.openStream( iStream );
350 Archive ar( ms, Archive::save ); 350 Archive ar( ms, Archive::save );
351 ar << vo; 351 ar << vo;
352 unitTest( ms.tell() == vo.getBytesWritten() ); 352 unitTest( ms.tell() == vo.getBytesWritten() );
353 ms.setSize( ms.tell() ); 353 ms.setSize( ms.tell() );
354 } 354 }
355 unitTest( m.getStreamSize( iStream ) == vo.getBytesWritten() ); 355 unitTest( m.getSize( iStream ) == vo.getBytesWritten() );
356 incProgress(); 356 incProgress();
357 } 357 }
358 } 358 }
@@ -365,18 +365,18 @@ suite Myriad
365 { 365 {
366 VerifyObject vo( random()%1024 ); 366 VerifyObject vo( random()%1024 );
367 { 367 {
368 MyriadStream ms = m.openStream( *i ); 368 MyriadStream ms = m.open( *i, Myriad::Read );
369 Archive ar( ms, Archive::load ); 369 Archive ar( ms, Archive::load );
370 ar >> vo; 370 ar >> vo;
371 } 371 }
372 { 372 {
373 MyriadStream ms = m.openStream( *i ); 373 MyriadStream ms = m.open( *i, Myriad::WriteNew );
374 Archive ar( ms, Archive::save ); 374 Archive ar( ms, Archive::save );
375 ar << vo; 375 ar << vo;
376 unitTest( ms.tell() == vo.getBytesWritten() ); 376 unitTest( ms.tell() == vo.getBytesWritten() );
377 ms.setSize( ms.tell() ); 377 ms.setSize( ms.tell() );
378 } 378 }
379 unitTest( m.getStreamSize( *i ) == vo.getBytesWritten() ); 379 unitTest( m.getSize( *i ) == vo.getBytesWritten() );
380 incProgress(); 380 incProgress();
381 } 381 }
382 } 382 }
diff --git a/src/unstable/bitstring.cpp b/src/unstable/bitstring.cpp
index 21c1316..b80c073 100644
--- a/src/unstable/bitstring.cpp
+++ b/src/unstable/bitstring.cpp
@@ -209,7 +209,7 @@ void Bu::BitString::flipBit( long iBit )
209 caData[iBit/8] ^= (1<<(iBit%8)); 209 caData[iBit/8] ^= (1<<(iBit%8));
210} 210}
211 211
212bool Bu::BitString::getBit( long iBit ) 212bool Bu::BitString::getBit( long iBit ) const
213{ 213{
214 if( iBit >= iBits || iBit < 0 ) return false; 214 if( iBit >= iBits || iBit < 0 ) return false;
215 if( (caData[iBit/8] & (1<<(iBit%8))) == 0 ) 215 if( (caData[iBit/8] & (1<<(iBit%8))) == 0 )
@@ -224,7 +224,7 @@ long Bu::BitString::getBitLength()
224 return iBits; 224 return iBits;
225} 225}
226 226
227long Bu::BitString::getSize() 227long Bu::BitString::getSize() const
228{ 228{
229 return iBits; 229 return iBits;
230} 230}
@@ -311,6 +311,14 @@ void Bu::BitString::clear()
311 } 311 }
312} 312}
313 313
314void Bu::BitString::fill()
315{
316 if( caData != NULL )
317 {
318 memset( caData, 0xff, iBytes );
319 }
320}
321
314bool Bu::BitString::setBitLength( long iLength, bool bClear ) 322bool Bu::BitString::setBitLength( long iLength, bool bClear )
315{ 323{
316 return setSize( iLength, bClear ); 324 return setSize( iLength, bClear );
diff --git a/src/unstable/bitstring.h b/src/unstable/bitstring.h
index afc22fb..70ba822 100644
--- a/src/unstable/bitstring.h
+++ b/src/unstable/bitstring.h
@@ -88,7 +88,7 @@ namespace Bu
88 *@param iBit The index of the bit to test. 88 *@param iBit The index of the bit to test.
89 *@returns True for a 1, false for a 0. 89 *@returns True for a 1, false for a 0.
90 */ 90 */
91 bool getBit( long iBit ); 91 bool getBit( long iBit ) const;
92 92
93 /** 93 /**
94 * Inverts the entire BitString, in effect this calls flipBit on every 94 * Inverts the entire BitString, in effect this calls flipBit on every
@@ -106,7 +106,7 @@ namespace Bu
106 DEPRECATED 106 DEPRECATED
107 long getBitLength(); 107 long getBitLength();
108 108
109 long getSize(); 109 long getSize() const;
110 110
111 /** 111 /**
112 * Sets the entire BitString to zeros, but it does it very quickly. 112 * Sets the entire BitString to zeros, but it does it very quickly.
@@ -115,6 +115,12 @@ namespace Bu
115 void clear(); 115 void clear();
116 116
117 /** 117 /**
118 * Sets the entire BitString to ones, but it does it very quickly.
119 * This operation runs in O(N).
120 */
121 void fill();
122
123 /**
118 * Gets another BitString that is autonomous of the current one 124 * Gets another BitString that is autonomous of the current one
119 * (contains a copy of the memory, not a pointer) and contains a subset 125 * (contains a copy of the memory, not a pointer) and contains a subset
120 * of the data in the current BitString. This is an inclusive 126 * of the data in the current BitString. This is an inclusive
diff --git a/src/unstable/cachebase.h b/src/unstable/cachebase.h
index ec73ede..830f5fe 100644
--- a/src/unstable/cachebase.h
+++ b/src/unstable/cachebase.h
@@ -32,11 +32,15 @@ namespace Bu
32 bDeleted( false ), 32 bDeleted( false ),
33 pObject( pObject ) 33 pObject( pObject )
34 { 34 {
35 //Bu::println("CacheEntry::CacheEntry: registering pObject (0x%1)").
36 // arg( reinterpret_cast<ptrdiff_t>(pObject), Bu::Fmt::hex() );
35 } 37 }
36 38
37 virtual ~CacheEntry() 39 virtual ~CacheEntry()
38 { 40 {
39 mEntry.lock(); 41 mEntry.lock();
42 //Bu::println("CacheEntry::~CacheEntry: deleting pObject (0x%1)").
43 // arg( reinterpret_cast<ptrdiff_t>(pObject), Bu::Fmt::hex() );
40 delete pObject; 44 delete pObject;
41 mEntry.unlock(); 45 mEntry.unlock();
42 } 46 }
@@ -497,7 +501,7 @@ namespace Bu
497 if( pEnt->iRefCount == 0 ) 501 if( pEnt->iRefCount == 0 )
498 { 502 {
499 pEnt->mEntry.unlock(); 503 pEnt->mEntry.unlock();
500 delete pEnt->pObject; 504 //delete pEnt->pObject;
501 delete pEnt; 505 delete pEnt;
502 } 506 }
503 else 507 else
@@ -529,7 +533,7 @@ namespace Bu
529 pEnt->mEntry.unlock(); 533 pEnt->mEntry.unlock();
530 throw Bu::ExceptionBase( Bu::String("Cache entry %1 cannot be erased, there are %2 active references.").arg( key ).arg( iCount ).end().getStr() ); 534 throw Bu::ExceptionBase( Bu::String("Cache entry %1 cannot be erased, there are %2 active references.").arg( key ).arg( iCount ).end().getStr() );
531 } 535 }
532 delete pEnt->pObject; 536 //delete pEnt->pObject;
533 delete pEnt; 537 delete pEnt;
534 hCacheEntry.erase( key ); 538 hCacheEntry.erase( key );
535 } 539 }
@@ -559,7 +563,7 @@ namespace Bu
559 { 563 {
560 if( pEnt->isReadyForCleanup() ) 564 if( pEnt->isReadyForCleanup() )
561 { 565 {
562 delete pEnt->pObject; 566 //delete pEnt->pObject;
563 delete pEnt; 567 delete pEnt;
564 } 568 }
565 } 569 }
diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h
index 24002b0..3629005 100644
--- a/src/unstable/myriadcache.h
+++ b/src/unstable/myriadcache.h
@@ -28,7 +28,9 @@ namespace Bu
28 try 28 try
29 { 29 {
30 Bu::ReadWriteMutex::ReadLocker l( rwStore ); 30 Bu::ReadWriteMutex::ReadLocker l( rwStore );
31 Bu::MyriadStream ms = mStore.openStream( 1 ); 31 Bu::MyriadStream ms = mStore.open(
32 1, Bu::Myriad::Read
33 );
32 Bu::Archive ar( ms, Bu::Archive::load ); 34 Bu::Archive ar( ms, Bu::Archive::load );
33 uint8_t uVer; 35 uint8_t uVer;
34 ar >> uVer; 36 ar >> uVer;
@@ -41,10 +43,15 @@ namespace Bu
41 } 43 }
42 catch(...) 44 catch(...)
43 { 45 {
44 if( mStore.createStreamWithId( 1 ) != 1 ) 46 try
47 {
48 mStore.open( 1, Bu::Myriad::Create|Bu::Myriad::ReadWrite );
49 _sync();
50 }
51 catch(...)
52 {
45 throw Bu::ExceptionBase("Error creating index stream."); 53 throw Bu::ExceptionBase("Error creating index stream.");
46 54 }
47 _sync();
48 } 55 }
49 } 56 }
50 57
@@ -78,7 +85,10 @@ namespace Bu
78 virtual void _create( const obtype *o ) 85 virtual void _create( const obtype *o )
79 { 86 {
80 Bu::ReadWriteMutex::WriteLocker wl( rwStore ); 87 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
81 hIndex.insert( o->getKey(), mStore.createStream() ); 88 {
89 Bu::Myriad::StreamId id = mStore.allocate();
90 hIndex.insert( o->getKey(), id );
91 }
82 _save( o ); 92 _save( o );
83 93
84 bStructureChanged = true; 94 bStructureChanged = true;
@@ -87,7 +97,7 @@ namespace Bu
87 virtual void _erase( const keytype &k ) 97 virtual void _erase( const keytype &k )
88 { 98 {
89 Bu::ReadWriteMutex::WriteLocker wl( rwStore ); 99 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
90 mStore.deleteStream( hIndex.get( k ) ); 100 mStore.erase( hIndex.get( k ) );
91 hIndex.erase( k ); 101 hIndex.erase( k );
92 102
93 bStructureChanged = true; 103 bStructureChanged = true;
@@ -98,13 +108,13 @@ namespace Bu
98 const keytype &k 108 const keytype &k
99 ) 109 )
100 { 110 {
101 Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) ); 111 Bu::MyriadStream ms = mStore.open( hIndex.get( k ), Bu::Myriad::Read );
102 return _cacheObjectLoad<keytype, obtype>( initObj, k, ms ); 112 return _cacheObjectLoad<keytype, obtype>( initObj, k, ms );
103 } 113 }
104 114
105 virtual void _save( const obtype *o ) 115 virtual void _save( const obtype *o )
106 { 116 {
107 Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) ); 117 Bu::MyriadStream ms = mStore.open( hIndex.get( o->getKey() ), Bu::Myriad::WriteNew );
108 _cacheObjectSave( ms, o ); 118 _cacheObjectSave( ms, o );
109 ms.setSize( ms.tell() ); 119 ms.setSize( ms.tell() );
110 120
@@ -117,7 +127,7 @@ namespace Bu
117 if( !bStructureChanged ) 127 if( !bStructureChanged )
118 return; 128 return;
119 129
120 Bu::MyriadStream ms = mStore.openStream( 1 ); 130 Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::WriteNew );
121 Bu::Archive ar( ms, Bu::Archive::save ); 131 Bu::Archive ar( ms, Bu::Archive::save );
122 ar << (uint8_t)0 << hIndex; 132 ar << (uint8_t)0 << hIndex;
123 ar.close(); 133 ar.close();
diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp
index b24997a..f748a53 100644
--- a/src/unstable/myriadfs.cpp
+++ b/src/unstable/myriadfs.cpp
@@ -8,6 +8,7 @@
8#include "bu/config.h" 8#include "bu/config.h"
9#include "bu/myriadfs.h" 9#include "bu/myriadfs.h"
10#include "bu/myriadstream.h" 10#include "bu/myriadstream.h"
11#include "bu/mutexlocker.h"
11 12
12#include <string.h> 13#include <string.h>
13#include <unistd.h> 14#include <unistd.h>
@@ -32,10 +33,10 @@ Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
32 iGroup = getgid(); 33 iGroup = getgid();
33#endif 34#endif
34 35
35 if( mStore.hasStream( 1 ) ) 36 if( mStore.exists( 1 ) )
36 { 37 {
37 // Check to see if this is a MyriadFs stream. 38 // Check to see if this is a MyriadFs stream.
38 Bu::MyriadStream ms = mStore.openStream( 1 ); 39 Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Read );
39 char sMagic[4]; 40 char sMagic[4];
40 if( ms.read( sMagic, 4 ) < 4 ) 41 if( ms.read( sMagic, 4 ) < 4 )
41 throw MyriadFsException("The provided stream does not appear to be " 42 throw MyriadFsException("The provided stream does not appear to be "
@@ -62,8 +63,8 @@ Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
62 { 63 {
63 // Create initial header stream 64 // Create initial header stream
64 { 65 {
65 mStore.createStream( 1 ); 66 Bu::MyriadStream ms = mStore.open(
66 Bu::MyriadStream ms = mStore.openStream( 1 ); 67 1, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
67 ms.write( Myriad_Fs_MAGIC_CODE, 4 ); 68 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
68 int8_t iVer = 1; 69 int8_t iVer = 1;
69 int32_t iTmp = 1; 70 int32_t iTmp = 1;
@@ -77,8 +78,8 @@ Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
77 78
78 // Create initial inode stream, with one root node. 79 // Create initial inode stream, with one root node.
79 { 80 {
80 mStore.createStream( 2 ); 81 Bu::MyriadStream ms = mStore.open(
81 Bu::MyriadStream ms = mStore.openStream( 2 ); 82 2, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
82 RawStat rs; 83 RawStat rs;
83 rs.iNode = 0; 84 rs.iNode = 0;
84 rs.iUser = iUser; 85 rs.iUser = iUser;
@@ -92,8 +93,8 @@ Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
92 93
93 // Create inode 0's storage stream. 94 // Create inode 0's storage stream.
94 { 95 {
95 mStore.createStream( 3 ); 96 Bu::MyriadStream ms = mStore.open(
96 Bu::MyriadStream ms = mStore.openStream( 3 ); 97 3, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
97 int32_t iTmp32 = 0; 98 int32_t iTmp32 = 0;
98 ms.write( &iTmp32, 4 ); // iChildCount 99 ms.write( &iTmp32, 4 ); // iChildCount
99 } 100 }
@@ -107,15 +108,17 @@ Bu::MyriadFs::~MyriadFs()
107 108
108void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf ) 109void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf )
109{ 110{
111 Bu::MutexLocker lLock( mAccess );
110 int32_t iParent; 112 int32_t iParent;
111 int32_t iNode = lookupInode( sPath, iParent ); 113 int32_t iNode = lookupInode( sPath, iParent );
112 Bu::MyriadStream is = mStore.openStream( 2 ); 114 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read );
113 stat( iNode, rBuf, is ); 115 stat( iNode, rBuf, is );
114} 116}
115 117
116Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode, 118Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode,
117 uint16_t uPerms ) 119 uint16_t uPerms )
118{ 120{
121 Bu::MutexLocker lLock( mAccess );
119 int32_t iParent = -1; 122 int32_t iParent = -1;
120 int32_t iNode; 123 int32_t iNode;
121 try 124 try
@@ -164,6 +167,7 @@ void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
164void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms, 167void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
165 uint32_t uSpecial ) 168 uint32_t uSpecial )
166{ 169{
170 Bu::MutexLocker lLock( mAccess );
167 int32_t iParent = -1; 171 int32_t iParent = -1;
168// int32_t iNode; 172// int32_t iNode;
169 try 173 try
@@ -200,6 +204,7 @@ void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms )
200void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget, 204void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
201 const Bu::String &sPath ) 205 const Bu::String &sPath )
202{ 206{
207 Bu::MutexLocker lLock( mAccess );
203 int32_t iParent = -1; 208 int32_t iParent = -1;
204 int32_t iNode; 209 int32_t iNode;
205 try 210 try
@@ -232,6 +237,7 @@ void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
232void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget, 237void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
233 const Bu::String &sPath ) 238 const Bu::String &sPath )
234{ 239{
240 Bu::MutexLocker lLock( mAccess );
235 int32_t iParent = -1; 241 int32_t iParent = -1;
236 int32_t iNode; 242 int32_t iNode;
237 243
@@ -257,7 +263,7 @@ void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
257// sio << "End filename: " << sName << sio.nl; 263// sio << "End filename: " << sName << sio.nl;
258// sio << "Parent inode: " << iParent << sio.nl; 264// sio << "Parent inode: " << iParent << sio.nl;
259 addToDir( iParent, iNode, sName ); 265 addToDir( iParent, iNode, sName );
260 MyriadStream is = mStore.openStream( 2 ); 266 MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
261 RawStat rs; 267 RawStat rs;
262 readInode( iNode, rs, is ); 268 readInode( iNode, rs, is );
263 rs.iLinks++; 269 rs.iLinks++;
@@ -267,6 +273,7 @@ void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
267 273
268Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath ) 274Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
269{ 275{
276 Bu::MutexLocker lLock( mAccess );
270 int32_t iParent = -1; 277 int32_t iParent = -1;
271 int32_t iNode; 278 int32_t iNode;
272 iNode = lookupInode( sPath, iParent ); 279 iNode = lookupInode( sPath, iParent );
@@ -279,6 +286,7 @@ Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
279 286
280Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath ) 287Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
281{ 288{
289 Bu::MutexLocker lLock( mAccess );
282 int32_t iParent = -1; 290 int32_t iParent = -1;
283 int32_t iNode = lookupInode( sPath, iParent ); 291 int32_t iNode = lookupInode( sPath, iParent );
284 return readDir( iNode ); 292 return readDir( iNode );
@@ -287,6 +295,7 @@ Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
287void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime, 295void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
288 int64_t iMTime ) 296 int64_t iMTime )
289{ 297{
298 Bu::MutexLocker lLock( mAccess );
290 int32_t iParent = -1; 299 int32_t iParent = -1;
291 int32_t iNode; 300 int32_t iNode;
292 301
@@ -297,6 +306,7 @@ void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
297 306
298void Bu::MyriadFs::unlink( const Bu::String &sPath ) 307void Bu::MyriadFs::unlink( const Bu::String &sPath )
299{ 308{
309 Bu::MutexLocker lLock( mAccess );
300 int32_t iParent = -1; 310 int32_t iParent = -1;
301// int32_t iNode; 311// int32_t iNode;
302 312
@@ -314,7 +324,9 @@ void Bu::MyriadFs::unlink( const Bu::String &sPath )
314 readInode( (*i).iNode, rs ); 324 readInode( (*i).iNode, rs );
315 if( (rs.uPerms&typeMask) == typeDir ) 325 if( (rs.uPerms&typeMask) == typeDir )
316 { 326 {
317 MyriadStream msDir = mStore.openStream( rs.uStreamIndex ); 327 MyriadStream msDir = mStore.open(
328 rs.uStreamIndex, Bu::Myriad::Read
329 );
318 int32_t iCount; 330 int32_t iCount;
319 msDir.read( &iCount, 4 ); 331 msDir.read( &iCount, 4 );
320 if( iCount > 0 ) 332 if( iCount > 0 )
@@ -350,6 +362,7 @@ void Bu::MyriadFs::unlink( const Bu::String &sPath )
350 362
351void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize ) 363void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
352{ 364{
365 Bu::MutexLocker lLock( mAccess );
353 int32_t iParent = -1; 366 int32_t iParent = -1;
354 int32_t iNode; 367 int32_t iNode;
355 iNode = lookupInode( sPath, iParent ); 368 iNode = lookupInode( sPath, iParent );
@@ -359,6 +372,7 @@ void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
359 372
360void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo ) 373void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo )
361{ 374{
375 Bu::MutexLocker lLock( mAccess );
362 mkHardLink( sFrom, sTo ); 376 mkHardLink( sFrom, sTo );
363 unlink( sFrom ); 377 unlink( sFrom );
364} 378}
@@ -449,7 +463,7 @@ void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs )
449 463
450void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs ) 464void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs )
451{ 465{
452 MyriadStream ms = mStore.openStream( 2 ); 466 MyriadStream ms = mStore.open( 2, Bu::Myriad::Read );
453 readInode( iNode, rs, ms ); 467 readInode( iNode, rs, ms );
454} 468}
455 469
@@ -464,7 +478,7 @@ void Bu::MyriadFs::writeInode( const RawStat &rs,
464 478
465void Bu::MyriadFs::writeInode( const RawStat &rs ) 479void Bu::MyriadFs::writeInode( const RawStat &rs )
466{ 480{
467 MyriadStream ms = mStore.openStream( 2 ); 481 MyriadStream ms = mStore.open( 2, Bu::Myriad::Write );
468 writeInode( rs, ms ); 482 writeInode( rs, ms );
469} 483}
470 484
@@ -474,9 +488,9 @@ Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode )
474 int32_t iNumChildren = 0; 488 int32_t iNumChildren = 0;
475 ms.read( &iNumChildren, 4 ); 489 ms.read( &iNumChildren, 4 );
476 490
477 Bu::MyriadStream is = mStore.openStream( 2 ); 491 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read );
478 Dir lDir; 492 Dir lDir;
479 sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl; 493 // sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl;
480 for( int32_t j = 0; j < iNumChildren; j++ ) 494 for( int32_t j = 0; j < iNumChildren; j++ )
481 { 495 {
482 int32_t iChildNode = 0; 496 int32_t iChildNode = 0;
@@ -485,7 +499,6 @@ Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode )
485 throw Bu::MyriadFsException( 499 throw Bu::MyriadFsException(
486 "Failed to read iChildNode from directory."); 500 "Failed to read iChildNode from directory.");
487 } 501 }
488 Bu::println(" - iNode = %1").arg( iChildNode );
489 Stat s; 502 Stat s;
490 stat( iChildNode, s, is ); 503 stat( iChildNode, s, is );
491 uint8_t uLen; 504 uint8_t uLen;
@@ -494,14 +507,12 @@ Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode )
494 throw Bu::MyriadFsException( 507 throw Bu::MyriadFsException(
495 "Failed to read uLen from directory."); 508 "Failed to read uLen from directory.");
496 } 509 }
497 Bu::println(" - Name bytes = %1").arg( uLen );
498 s.sName.setSize( uLen ); 510 s.sName.setSize( uLen );
499 if( ms.read( s.sName.getStr(), uLen ) < uLen ) 511 if( ms.read( s.sName.getStr(), uLen ) < uLen )
500 { 512 {
501 throw Bu::MyriadFsException( 513 throw Bu::MyriadFsException(
502 "Failed to read sName from directory."); 514 "Failed to read sName from directory.");
503 } 515 }
504 Bu::println(" - Name = \"%1\"").arg( s.sName );
505 lDir.append( s ); 516 lDir.append( s );
506 517
507// sio << " " << s.sName << sio.nl; 518// sio << " " << s.sName << sio.nl;
@@ -519,9 +530,7 @@ Bu::MyriadStream Bu::MyriadFs::openByInode( int32_t iNode )
519 case typeDir: 530 case typeDir:
520 case typeSymLink: 531 case typeSymLink:
521 case typeRegFile: 532 case typeRegFile:
522 Bu::println("Opening stream by iNode=%1, myriad stream=%2") 533 return mStore.open( rs.uStreamIndex, Bu::Myriad::ReadWrite );
523 .arg( iNode ).arg( rs.uStreamIndex );
524 return mStore.openStream( rs.uStreamIndex );
525 534
526 default: 535 default:
527 throw Bu::MyriadFsException( 536 throw Bu::MyriadFsException(
@@ -575,17 +584,17 @@ int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial )
575 { 584 {
576 case typeRegFile: 585 case typeRegFile:
577 case typeSymLink: 586 case typeSymLink:
578 rs.uStreamIndex = mStore.createStream(); 587 rs.uStreamIndex = mStore.allocate();
579 break; 588 break;
580 589
581 case typeDir: 590 case typeDir:
582 rs.uStreamIndex = mStore.createStream();
583// sio << "Creating directory node, storage: " 591// sio << "Creating directory node, storage: "
584// << rs.uStreamIndex << sio.nl; 592// << rs.uStreamIndex << sio.nl;
585 { 593 {
586 Bu::MyriadStream msDir = mStore.openStream( 594 Bu::MyriadStream msDir = mStore.create(
587 rs.uStreamIndex 595 Bu::Myriad::Write
588 ); 596 );
597 rs.uStreamIndex = msDir.getId();
589 uint32_t uSize = 0; 598 uint32_t uSize = 0;
590 msDir.write( &uSize, 4 ); 599 msDir.write( &uSize, 4 );
591 } 600 }
@@ -631,7 +640,7 @@ void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs )
631 { 640 {
632 case typeRegFile: 641 case typeRegFile:
633 case typeSymLink: 642 case typeSymLink:
634 rBuf.iSize = mStore.getStreamSize( rs.uStreamIndex ); 643 rBuf.iSize = mStore.getSize( rs.uStreamIndex );
635 break; 644 break;
636 645
637 case typeChrDev: 646 case typeChrDev:
@@ -647,7 +656,7 @@ void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs )
647 656
648void Bu::MyriadFs::writeHeader() 657void Bu::MyriadFs::writeHeader()
649{ 658{
650 Bu::MyriadStream ms = mStore.openStream( 1 ); 659 Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Write );
651 ms.write( Myriad_Fs_MAGIC_CODE, 4 ); 660 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
652 int8_t iVer = 1; 661 int8_t iVer = 1;
653 int32_t iNumNodes = hNodeIndex.getSize(); 662 int32_t iNumNodes = hNodeIndex.getSize();
@@ -668,7 +677,7 @@ void Bu::MyriadFs::writeHeader()
668void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ) 677void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime )
669{ 678{
670 RawStat rs; 679 RawStat rs;
671 Bu::MyriadStream is = mStore.openStream( 2 ); 680 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
672 681
673 readInode( iNode, rs, is ); 682 readInode( iNode, rs, is );
674 rs.iATime = iATime; 683 rs.iATime = iATime;
@@ -681,18 +690,21 @@ void Bu::MyriadFs::destroyNode( int32_t iNode )
681 if( iNode == 0 ) 690 if( iNode == 0 )
682 throw Bu::MyriadFsException("You cannot destroy the root."); 691 throw Bu::MyriadFsException("You cannot destroy the root.");
683 692
684 Bu::MyriadStream is = mStore.openStream( 2 ); 693 uint32_t iPosition;
694 RawStat rsOld;
695
696 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
685 697
686 // This will be overwritten with the last node 698 // This will be overwritten with the last node
687 uint32_t iPosition = hNodeIndex.get( iNode ); 699 iPosition = hNodeIndex.get( iNode );
688 RawStat rsOld;
689 readInode( iNode, rsOld, is ); 700 readInode( iNode, rsOld, is );
701
690 switch( (rsOld.uPerms&typeMask) ) 702 switch( (rsOld.uPerms&typeMask) )
691 { 703 {
692 case typeRegFile: 704 case typeRegFile:
693 case typeDir: 705 case typeDir:
694 case typeSymLink: 706 case typeSymLink:
695 mStore.deleteStream( rsOld.uStreamIndex ); 707 mStore.erase( rsOld.uStreamIndex );
696 break; 708 break;
697 } 709 }
698 710
diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h
index ff14292..e3008bc 100644
--- a/src/unstable/myriadfs.h
+++ b/src/unstable/myriadfs.h
@@ -11,7 +11,7 @@
11#include <sys/types.h> 11#include <sys/types.h>
12 12
13#include "bu/myriad.h" 13#include "bu/myriad.h"
14#include "bu/readwritemutex.h" 14#include "bu/debugmutex.h"
15 15
16namespace Bu 16namespace Bu
17{ 17{
@@ -108,7 +108,7 @@ namespace Bu
108 Truncate = 0x08, ///< Truncate file if it does exist 108 Truncate = 0x08, ///< Truncate file if it does exist
109 Append = 0x10, ///< Always append on every write 109 Append = 0x10, ///< Always append on every write
110 NonBlock = 0x20, ///< Open file in non-blocking mode 110 NonBlock = 0x20, ///< Open file in non-blocking mode
111 Exclusive = 0x44, ///< Create file, if it exists then fail 111 Exclusive = 0x40, ///< Create file, if it exists then fail
112 112
113 // Helpful mixes 113 // Helpful mixes
114 ReadWrite = 0x03, ///< Open for reading and writing 114 ReadWrite = 0x03, ///< Open for reading and writing
@@ -172,7 +172,13 @@ namespace Bu
172 typedef Bu::Hash<int32_t, int32_t> NodeIndex; 172 typedef Bu::Hash<int32_t, int32_t> NodeIndex;
173 173
174 private: 174 private:
175 /**
176 * Lookup inode.
177 */
175 int32_t lookupInode( const Bu::String &sPath, int32_t &iParent ); 178 int32_t lookupInode( const Bu::String &sPath, int32_t &iParent );
179 /**
180 * Lookup inode.
181 */
176 int32_t lookupInode( Bu::String::const_iterator iStart, 182 int32_t lookupInode( Bu::String::const_iterator iStart,
177 int32_t iNode, int32_t &iParent ); 183 int32_t iNode, int32_t &iParent );
178 void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs ); 184 void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs );
@@ -190,12 +196,12 @@ namespace Bu
190 void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime ); 196 void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime );
191 void destroyNode( int32_t iNode ); 197 void destroyNode( int32_t iNode );
192 198
193 Bu::String filePart( const Bu::String &sPath ); 199 static Bu::String filePart( const Bu::String &sPath );
194 200
195 private: 201 private:
196 Bu::Stream &rStore; 202 Bu::Stream &rStore;
197 Bu::Myriad mStore; 203 Bu::Myriad mStore;
198 Bu::ReadWriteMutex mNodeIndex; 204 Bu::DebugMutex mAccess;
199 NodeIndex hNodeIndex; 205 NodeIndex hNodeIndex;
200 int32_t iUser; 206 int32_t iUser;
201 int32_t iGroup; 207 int32_t iGroup;
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 06:45:06 +0000