aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/stable/myriad.cpp930
-rw-r--r--src/stable/myriad.h287
-rw-r--r--src/stable/myriadstream.cpp257
-rw-r--r--src/stable/myriadstream.h49
-rw-r--r--src/tests/bigmyriad.cpp8
-rw-r--r--src/tests/cache.cpp289
-rw-r--r--src/tests/cachedel.cpp288
-rw-r--r--src/tests/mfstest.cpp88
-rw-r--r--src/tests/myriadfs.cpp62
-rw-r--r--src/tools/myriad.cpp280
-rw-r--r--src/tools/myriadfs.cpp269
-rw-r--r--src/unit/myriad.unit385
-rw-r--r--src/unstable/myriadcache.cpp8
-rw-r--r--src/unstable/myriadcache.h140
-rw-r--r--src/unstable/myriadfs.cpp722
-rw-r--r--src/unstable/myriadfs.h205
16 files changed, 376 insertions, 3891 deletions
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index c606369..5278ac5 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -1,25 +1,18 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/config.h"
9#include "bu/myriad.h" 1#include "bu/myriad.h"
10#include "bu/stream.h"
11#include "bu/myriadstream.h" 2#include "bu/myriadstream.h"
3
12#include "bu/mutexlocker.h" 4#include "bu/mutexlocker.h"
13#include <stdio.h> 5#include "bu/util.h"
14 6
15#include "bu/sio.h" 7#include "bu/sio.h"
16using Bu::sio;
17using Bu::Fmt;
18 8
19#define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84") 9#define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84")
20 10
21// #define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(sStore.getLocation()).arg(x) 11#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
22#define TRACE( x ) (void)0 12{ \
13 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
14 "Insufficent data reading myriad data from backing stream."); \
15} (void)0
23 16
24namespace Bu 17namespace Bu
25{ 18{
@@ -29,803 +22,324 @@ namespace Bu
29 } 22 }
30} 23}
31 24
32Bu::Myriad::Myriad( Bu::Stream &sStore, int iBlockSize, int iPreallocate ) : 25Bu::Myriad::Myriad( Bu::Stream &rBacking, int iBlockSize,
33 sStore( sStore ), 26 int iPreallocateBlocks ) :
27 rBacking( rBacking ),
34 iBlockSize( iBlockSize ), 28 iBlockSize( iBlockSize ),
35 iBlocks( 0 ), 29 iBlockCount( 0 ),
36 iUsed( 0 ), 30 bIsNewStream( true )
37 bHeaderChanged( false )
38{ 31{
39 try 32 if( !rBacking.isSeekable() )
40 { 33 {
41 initialize(); 34 throw Bu::MyriadException( Bu::MyriadException::invalidBackingStream,
35 "Myriad backing stream must be random access (seekable).");
42 } 36 }
43 catch( Bu::MyriadException &e ) 37 if( !loadMyriad() )
44 { 38 {
45 if( e.getErrorCode() == MyriadException::emptyStream ) 39 createMyriad( iBlockSize, iPreallocateBlocks );
46 {
47 initialize( iBlockSize, iPreallocate );
48 }
49 else
50 {
51 throw;
52 }
53 } 40 }
54} 41}
55 42
56Bu::Myriad::~Myriad() 43Bu::Myriad::~Myriad()
57{ 44{
58 mActiveBlocks.lock();
59 TRACE("mActiveBlocks locked.");
60 if( !hActiveBlocks.isEmpty() )
61 {
62 sio << "Bu::Myriad::~Myriad(): Error: There are "
63 << hActiveBlocks.getSize() << " unsynced blocks!" << sio.nl;
64 }
65 TRACE("mActiveBlocks unlocking...");
66 mActiveBlocks.unlock();
67 sync();
68
69 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
70 {
71 delete *i;
72 }
73} 45}
74 46
75void Bu::Myriad::sync() 47Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode /*eMode*/,
48 int32_t /*iPreallocateBytes*/ )
76{ 49{
77 updateHeader(); 50 return Bu::MyriadStream( *this, NULL, (Mode)0 );
78
79 mActiveBlocks.lock();
80 TRACE("mActiveBlocks locked.");
81 for( BlockHash::iterator i = hActiveBlocks.begin(); i; i++ )
82 {
83 if( (*i)->bChanged )
84 {
85 syncBlock( *i );
86 }
87 }
88 TRACE("mActiveBlocks unlocked...");
89 mActiveBlocks.unlock();
90} 51}
91 52
92void Bu::Myriad::initialize() 53Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
54 Bu::Myriad::Mode eMode )
93{ 55{
94 MutexLocker mLock( mHeader ); 56 Bu::MutexLocker l( mAccess );
95 TRACE("mHeader locked."); 57 if( !hStream.has( iStream ) )
96 lFreeBlocks.clear();
97 sStore.setPosEnd( 0 );
98 Bu::size iSize = sStore.tell();
99 sStore.setPos( 0 );
100
101 unsigned char buf[4];
102 if( sStore.read( buf, 4 ) < 4 )
103 {
104 TRACE("mHeader unlocked...");
105 throw MyriadException( MyriadException::emptyStream,
106 "Input stream appears to be empty.");
107 }
108 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
109 {
110 TRACE("mHeader unlocked...");
111 throw MyriadException( MyriadException::invalidFormat,
112 "Stream does not appear to be a valid Myriad format.");
113 }
114 sStore.read( buf, 2 );
115 if( buf[0] != 1 )
116 {
117 TRACE("mHeader unlocked...");
118 throw MyriadException( MyriadException::badVersion,
119 "We can only handle version 1 for now.");
120 }
121 if( buf[1] != 32 )
122 {
123 TRACE("mHeader unlocked...");
124 throw MyriadException( MyriadException::invalidWordSize,
125 "We can only handle 32-bit words at the moment.");
126 }
127 sStore.read( &iBlockSize, 4 );
128 int iStreams;
129 sStore.read( &iStreams, 4 );
130
131 iBlocks = iSize/iBlockSize;
132 //sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize
133 // << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl;
134
135 int iHeaderSize = 14 + 8 + 4;
136 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
137
138 while( iHeaderSize > iHeaderBlocks*iBlockSize )
139 {
140 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
141 iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
142 }
143
144 //sio << "Myriad: iHeaderSize=" << iHeaderSize
145 // << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
146
147 Stream *pFakeHdr = new Stream;
148 pFakeHdr->iId = 0;
149 pFakeHdr->setSize( iHeaderSize );
150 for( int j = 0; j < iHeaderBlocks; j++ )
151 {
152 pFakeHdr->aBlocks.append( j );
153 }
154
155// sio << "Blocks: " << iBlocks << " (size = " << iSize << "/" << iBlockSize
156// << ")" << sio.nl;
157 Bu::BitString bsBlockUsed( iBlocks, false );
158 bsBlockUsed.clear();
159
160// bool bCanSkip = false; // Can skip around, post initial header stream i/o
161 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
162 pIn->setPos( sStore.tell() );
163 for( int j = 0; j < iStreams; j++ )
164 { 58 {
165 int iSizeTmp; 59 throw Bu::MyriadException( MyriadException::noSuchStream,
166 aStreams.append( new Stream() ); 60 "No such stream.");
167 Stream &s = *aStreams[j];
168 pIn->read( &s.iId, 4 );
169 pIn->read( &iSizeTmp, 4 );
170 s.setSize( iSizeTmp );
171 int iSBlocks = blkDiv(s.getSize(), iBlockSize);
172 // sio << "Myriad: - Stream::iId=" << s.iId
173 // << ", Stream::iSize=" << s.iSize
174 // << ", Stream::aBlocks=" << iSBlocks
175 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
176 for( int k = 0; k < iSBlocks; k++ )
177 {
178 int iBId;
179 pIn->read( &iBId, 4 );
180 // sio << "Myriad: - iBId=" << iBId
181 // << ", iStartPos=" << iBId*iBlockSize
182 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
183 s.aBlocks.append( iBId );
184 bsBlockUsed.setBit( iBId );
185 iUsed++;
186 if( (j == 0 && k == iHeaderBlocks-1) )
187 {
188 // sio << "Myriad: - End of prepartition, unlocking skipping."
189 // << sio.nl;
190// bCanSkip = true;
191 MyriadStream *pTmp = new MyriadStream( *this, aStreams[0] );
192 // sio << "Myriad - Position = " << pIn->tell() << sio.nl;
193 pTmp->setPos( pIn->tell() );
194 delete pIn;
195 delete pFakeHdr;
196 pIn = pTmp;
197 }
198 }
199 } 61 }
200 delete pIn;
201
202 for( int j = 0; j < iBlocks; j++ )
203 { 62 {
204 if( bsBlockUsed.getBit( j ) == false ) 63 Bu::MutexLocker l2( mBacking );
64 if( (eMode&Write) && rBacking.isWritable() )
205 { 65 {
206// sio << "Preinitialized block " << j << " is free." << sio.nl; 66 throw Bu::MyriadException( MyriadException::badMode,
207 lFreeBlocks.append( j ); 67 "Backing stream does not support writing.");
208 } 68 }
209 } 69 }
210// sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; 70 return Bu::MyriadStream( *this, hStream.get( iStream ), eMode );
211 TRACE("mHeader unlocked...");
212} 71}
213 72
214void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) 73bool Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ )
215{ 74{
216 MutexLocker mLock( mHeader ); 75 return false;
217 TRACE("mHeader locked.");
218 lFreeBlocks.clear();
219
220 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
221 {
222 delete *i;
223 }
224 aStreams.clear();
225 iUsed = 0;
226
227 int iHeaderSize = 14 + 8 + 4;
228 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
229 char cBuf = 1;
230 int iBuf = 0;
231
232 Stream *pStr = new Stream;
233 pStr->iId = 0;
234
235 while( iHeaderSize > iHeaderBlocks*iBlockSize )
236 {
237 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
238 iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
239 }
240
241 iPreAllocate += iHeaderBlocks;
242
243 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize="
244 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
245
246// bsBlockUsed.setSize( iPreAllocate, true );
247 iUsed++;
248
249 char *pBlock = new char[iBlockSize];
250 memset( pBlock, 0, iBlockSize );
251 for( int j = 0; j < iPreAllocate; j++ )
252 {
253 sStore.write( pBlock, iBlockSize );
254 }
255 delete[] (char *)pBlock;
256
257 sStore.setPos( 0 );
258
259 // Magic number
260 sStore.write( Myriad_MAGIC_CODE, 4 );
261
262 // Version (0)
263 sStore.write( &cBuf, 1 );
264
265 // Bits per int
266 cBuf = 32;
267 sStore.write( &cBuf, 1 );
268
269 // The size of each block
270 sStore.write( &iBlockSize, 4 );
271
272 iBuf = 1;
273 // The number of streams
274 sStore.write( &iBuf, 4 );
275
276 // Stream header
277 iBuf = 0;
278 sStore.write( &iBuf, 4 );
279 sStore.write( &iHeaderSize, 4 );
280 for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ )
281 {
282 sStore.write( &iBuf, 4 );
283 }
284
285 this->iBlockSize = iBlockSize;
286 this->iBlocks = iPreAllocate;
287
288 pStr->setSize( sStore.tell() );
289// sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl;
290
291 pStr->setSize( iHeaderSize );
292 for( int j = 0; j < iHeaderBlocks; j++ )
293 {
294// sio << "Started block " << j << " is header." << sio.nl;
295 pStr->aBlocks.append( j );
296// bsBlockUsed.setBit( j );
297 iUsed++;
298 }
299 for( int j = iHeaderBlocks; j < this->iBlocks; j++ )
300 {
301// sio << "Started block " << j << " is free." << sio.nl;
302 lFreeBlocks.append( j );
303 }
304
305 aStreams.append( pStr );
306
307 //sio << bsBlockUsed.toString() << " - " << pStr->aBlocks << sio.nl;
308
309 bHeaderChanged = true;
310 //hStreams.insert( 0, BlockArray( 0 ) );
311 TRACE("mHeader unlocked...");
312}
313
314void Bu::Myriad::Stream::setSize( int iNewSize )
315{
316 MutexLocker l( mStream );
317 iSize = iNewSize;
318}
319
320void Bu::Myriad::Stream::growTo( int iNewSize )
321{
322 MutexLocker l( mStream );
323 if( iNewSize < iSize )
324 return;
325 iSize = iNewSize;
326} 76}
327 77
328int Bu::Myriad::Stream::getSize() const 78bool Bu::Myriad::setSize( Bu::Myriad::StreamId /*iStream*/,
79 int32_t /*iNewSize*/ )
329{ 80{
330 MutexLocker l( mStream ); 81 return false;
331 return iSize;
332} 82}
333 83
334void Bu::Myriad::updateHeader() 84bool Bu::Myriad::loadMyriad()
335{ 85{
336 MutexLocker mLock( mHeader ); 86 Bu::println("Load myriad!");
337 TRACE("mHeader locked."); 87 char sMagicCode[4];
338 88 rBacking.setPos( 0 );
339 if( bHeaderChanged == false ) 89 MyriadRead( sMagicCode, 4 );
90 if( memcmp( sMagicCode, Myriad_MAGIC_CODE, 4 ) )
340 { 91 {
341 TRACE("mHeader unlocked..."); 92 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
342 return; 93 "Backing stream does not seem to be a Myriad structure.");
343 } 94 }
344 if( !sStore.canWrite() ) 95 uint8_t uVer;
96 uint8_t uBitsPerInt;
97 MyriadRead( &uVer, 1 );
98 if( uVer != 1 )
345 { 99 {
346 TRACE("mHeader unlocked..."); 100 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
347 return; 101 "Only version 1 myriad structures are supported.");
348 } 102 }
349 103 MyriadRead( &uBitsPerInt, 1 );
350 char cBuf; 104 if( uBitsPerInt != 32 )
351 int iBuf;
352
353 //for( StreamArray::iterator i = aStreams.begin(); i; i++ )
354 //{
355 // sio << "Myriad: Stream " << Fmt(4) << (*i)->iId << ": " << (*i)->aBlocks << sio.nl;
356 //}
357
358 // Compute the new size of the header.
359 int iHeaderSize = 14 + 8*aStreams.getSize();
360// sio << "Myriad: updateHeader: aStreams.getSize() = " << aStreams.getSize()
361// << sio.nl;
362 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
363 { 105 {
364 iHeaderSize += 4*(*i)->aBlocks.getSize(); 106 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
365// sio << "Myriad: updateHeader: (*i)->aBlocks.getSize() = " 107 "Only 32 bits per int are supported at this time.");
366// << (*i)->aBlocks.getSize() << sio.nl;
367 } 108 }
368 int iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); 109 MyriadRead( &iBlockSize, 4 );
369 while( iNewBlocks > aStreams[0]->aBlocks.getSize() ) 110 int iStreamCount;
111 MyriadRead( &iStreamCount, 4 );
112
113 //
114 // Read stream data -- Bootstrap the zero stream
115 //
116 StreamId iStream;
117 MyriadRead( &iStream, 4 );
118 if( iStream != 0 )
370 { 119 {
371 int iBlock = findEmptyBlock(); 120 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
372// sio << "Myriad: updateHeader: Appending block " << iBlock 121 "The first stream defined must be the header/zero stream.");
373// << " to header." << sio.nl;
374 aStreams[0]->aBlocks.append( iBlock );
375// bsBlockUsed.setBit( iBlock );
376 iUsed++;
377 iHeaderSize += 4;
378 iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
379 } 122 }
380 aStreams[0]->setSize( iHeaderSize ); 123 int32_t iHeaderStreamBytes;
381// sio << "Myriad: updateHeader: iHeaderSize=" << iHeaderSize 124 MyriadRead( &iHeaderStreamBytes, 4 );
382// << ", iNewBlocks=" << iNewBlocks << ", curBlocks="
383// << aStreams[0]->aBlocks.getSize() << sio.nl;
384 125
385 MyriadStream sHdr( *this, aStreams[0] ); 126 Stream *pHeaderStream = new Stream( *this, iStream, iHeaderStreamBytes );
386 sHdr.write( Myriad_MAGIC_CODE, 4 ); 127 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
387 128
388 // Version (1) 129 while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
389 cBuf = 1; 130 > iHeaderStreamBlocks*iBlockSize )
390 sHdr.write( &cBuf, 1 );
391
392 // Bits per int
393 cBuf = 32;
394 sHdr.write( &cBuf, 1 );
395
396 // The size of each block
397 sHdr.write( &iBlockSize, 4 );
398
399 iBuf = aStreams.getSize();
400 // The number of streams
401 sHdr.write( &iBuf, 4 );
402
403 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
404 { 131 {
405 int iSizeTmp; 132 iHeaderStreamBlocks = blkDiv(
406 sHdr.write( &(*i)->iId, 4 ); 133 (iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize
407 sHdr.write( &iSizeTmp, 4 ); 134 );
408 (*i)->setSize( iSizeTmp );
409 int iUsedBlocks = blkDiv( iSizeTmp, iBlockSize );
410// for( BlockArray::iterator j = (*i)->aBlocks.begin(); j; j++ )
411 for( int j = 0; j < iUsedBlocks; j++ )
412 {
413 sHdr.write( &(*i)->aBlocks[j], 4 );
414 }
415 } 135 }
416
417 bHeaderChanged = false;
418 TRACE("mHeader unlocked...");
419}
420
421int Bu::Myriad::createStream( int iPreAllocate )
422{
423 MutexLocker mLock( mHeader );
424 TRACE("mHeader locked.");
425
426 Stream *pStr = new Stream();
427 pStr->iId = aStreams.last()->iId+1;
428 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
429 // << iPreAllocate << sio.nl;
430 pStr->setSize( 0 );
431 aStreams.append( pStr );
432 136
433 for( int j = 0; j < iPreAllocate; j++ ) 137 for( int32_t j = 0; j < iHeaderStreamBlocks; j++ )
434 { 138 {
435 int iFreeBlock = findEmptyBlock(); 139 int32_t iBlockIndex;
436// sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 140 MyriadRead( &iBlockIndex, 4 );
437 pStr->aBlocks.append( iFreeBlock ); 141 pHeaderStream->aBlocks.append( iBlockIndex );
438// bsBlockUsed.setBit( iFreeBlock );
439 iUsed++;
440 } 142 }
441 143
442 bHeaderChanged = true; 144 // Bootstrap now using the header stream to read the rest of the data.
443 145
444 TRACE("mHeader unlocked..."); 146 return true;
445 return pStr->iId;
446} 147}
447 148
448int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate ) 149void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
449{ 150{
450 MutexLocker mLock( mHeader ); 151 if( iBlockSize < 8 )
451 TRACE("mHeader locked.");
452
453 try
454 { 152 {
455 findStream( iId ); 153 throw Bu::MyriadException( Bu::MyriadException::invalidParameter,
456 TRACE("mHeader unlocked..."); 154 "iBlockSize cannot be below 8");
457 throw MyriadException( MyriadException::streamExists,
458 "There is already a stream with the given id.");
459 } 155 }
460 catch( MyriadException &e ) 156 if( rBacking.getSize() )
461 { 157 {
462 Stream *pStr = new Stream(); 158 throw Bu::MyriadException( Bu::MyriadException::invalidFormat,
463 pStr->iId = iId; 159 "Backing stream contains data, but not a myriad structure.");
464 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
465 // << iPreAllocate << sio.nl;
466 pStr->setSize( 0 );
467 if( aStreams.last()->iId < iId )
468 {
469 aStreams.append( pStr );
470 }
471 else
472 {
473 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
474 {
475 if( (*i)->iId > iId )
476 {
477 aStreams.insert( i, pStr );
478 break;
479 }
480 }
481 }
482
483 for( int j = 0; j < iPreAllocate; j++ )
484 {
485 int iFreeBlock = findEmptyBlock();
486 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
487 pStr->aBlocks.append( iFreeBlock );
488// bsBlockUsed.setBit( iFreeBlock );
489 iUsed++;
490 }
491
492 bHeaderChanged = true;
493
494 TRACE("mHeader unlocked...");
495 return pStr->iId;
496 }
497 TRACE("mHeader unlocked...");
498} 160}
499 161/*
500int Bu::Myriad::findEmptyBlock() 162 struct {
501{ 163 char sMagicCode[4];
502 bHeaderChanged = true; 164 uint8_t uVer;
503 165 uint8_t uBitsPerInt;
504 if( lFreeBlocks.isEmpty() ) 166 uint32_t uBlockSize;
505 { 167 uint32_t uStreamCount;
506 sStore.setSize( (iBlocks+1)*(Bu::size)iBlockSize ); 168 } sHeader;
507 return iBlocks++; 169
170 struct {
171 uint32_t uStreamId;
172 uint32_t uStreamSize;
173 } sStreamHeader;
174
175 Bu::println("sHeader = %1, sStreamHeader = %2").arg( sizeof(sHeader) ).arg( sizeof(sStreamHeader) );
176*/
177
178 // Start with the bytes for the file header and initial stream header
179 int iHeaderStreamBytes
180 = 14 // Base header
181 + 8; // Stream header
182
183 // Pick the block count that matches our current estimate for the header
184 // plus one block index.
185 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
186
187 Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
188 .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
189 .arg( iHeaderStreamBlocks )
190 .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
191 .arg( iHeaderStreamBlocks*iBlockSize );
192 while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
193 > iHeaderStreamBlocks*iBlockSize )
194 {
195 iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize);
196 if( iHeaderStreamBlocks > 100 )
197 break;
198 Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).")
199 .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
200 .arg( iHeaderStreamBlocks )
201 .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
202 .arg( iHeaderStreamBlocks*iBlockSize );
203 }
204
205 if( iPreallocateBlocks > iHeaderStreamBlocks )
206 {
207 rBacking.setSize( iBlockSize*iPreallocateBlocks );
508 } 208 }
509 else 209 else
510 { 210 {
511 return lFreeBlocks.dequeue(); 211 rBacking.setSize( iBlockSize*iHeaderStreamBlocks );
512 } 212 }
513}
514 213
515void Bu::Myriad::deleteStream( int iId ) 214 //
516{ 215 // Write Myriad header
517 MutexLocker mLock( mHeader ); 216 //
518 TRACE("mHeader locked."); 217 uint8_t uVer = 1;
218 uint8_t uBpi = 32;
219 int32_t iStreamCount = 1;
220 rBacking.setPos( 0 );
221 rBacking.write( Myriad_MAGIC_CODE, 4 );
222 rBacking.write( &uVer, 1 );
223 rBacking.write( &uBpi, 1 );
224 rBacking.write( &iBlockSize, 4 );
225 rBacking.write( &iStreamCount, 4 );
519 226
520 if( iId < 0 ) 227 Stream *pHeadStream = new Stream( *this, 0, Bu::Myriad::ReadWrite );
228 //
229 // Write stream header
230 //
231 uint32_t uStreamId = 0;
232 uint32_t uStreamSize = iHeaderStreamBytes+iHeaderStreamBlocks*4;
233 rBacking.write( &uStreamId, 4 );
234 rBacking.write( &uStreamSize, 4 );
235 for( int iBlockIndex = 0; iBlockIndex < iHeaderStreamBlocks; iBlockIndex++ )
521 { 236 {
522 TRACE("mHeader unlocked..."); 237 rBacking.write( &iBlockIndex, 4 );
523 throw MyriadException( MyriadException::invalidStreamId, 238 pHeadStream->aBlocks.append( iBlockIndex );
524 "Invalid stream id.");
525 } 239 }
526 if( iId == 0 ) 240 rBacking.flush();
527 {
528 TRACE("mHeader unlocked...");
529 throw MyriadException( MyriadException::protectedStream,
530 "You cannot delete stream zero, it is protected.");
531 }
532 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
533 {
534 if( (*i)->iId == iId )
535 {
536 Stream *pStream = *i;
537 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
538 {
539 lFreeBlocks.append( *j );
540// bsBlockUsed.setBit( *j, false );
541 iUsed--;
542 }
543 aStreams.erase( i );
544 bHeaderChanged = true;
545 delete pStream;
546 TRACE("mHeader unlocked...");
547 return;
548 }
549 }
550 TRACE("mHeader unlocked...");
551}
552 241
553Bu::Array<int> Bu::Myriad::getStreamIds() 242 hStream.insert( pHeadStream->iStream, pHeadStream );
554{
555 MutexLocker mLock( mHeader );
556 TRACE("mHeader locked.");
557 243
558 Bu::Array<int> aRet( aStreams.getSize() ); 244 for( int32_t j = iHeaderStreamBlocks; j < iPreallocateBlocks; j++ )
559 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
560 { 245 {
561 aRet.append( (*i)->iId ); 246 lFreeBlocks.append( j );
562 } 247 }
563
564 TRACE("mHeader unlocked...");
565 return aRet;
566}
567
568int Bu::Myriad::getStreamSize( int iId )
569{
570 MutexLocker mLock( mHeader );
571 TRACE("mHeader locked.");
572
573 TRACE("mHeader unlocked...");
574 return findStream( iId )->getSize();
575} 248}
576 249
577bool Bu::Myriad::hasStream( int iId ) 250int32_t Bu::Myriad::allocateBlock()
578{ 251{
579 MutexLocker mLock( mHeader ); 252 Bu::MutexLocker l( mAccess );
580 TRACE("mHeader locked."); 253 if( lFreeBlocks.isEmpty() )
581
582 try
583 {
584 findStream( iId );
585 TRACE("mHeader unlocked...");
586 return true;
587 }catch(...)
588 { 254 {
589 TRACE("mHeader unlocked..."); 255 // Increase the size of the backing stream
590 return false; 256 int32_t iIndex = iBlockCount++;
257 rBacking.setSize( iBlockCount*iBlockSize );
258 return iIndex;
591 } 259 }
592} 260 else
593
594Bu::MyriadStream Bu::Myriad::openStream( int iId )
595{
596 MutexLocker mLock( mHeader );
597 TRACE("mHeader locked.");
598
599 TRACE("mHeader unlocked...");
600 //sio << "Myriad: Request to open stream: " << iId << sio.nl;
601 return MyriadStream( *this, findStream( iId ) );
602}
603
604int Bu::Myriad::getNumStreams()
605{
606 MutexLocker mLock( mHeader );
607 TRACE("mHeader locked.");
608
609 TRACE("mHeader unlocked...");
610 return aStreams.getSize();
611}
612
613int Bu::Myriad::getBlockSize()
614{
615 return iBlockSize;
616}
617
618int Bu::Myriad::getNumBlocks()
619{
620 return iBlocks;
621}
622
623int Bu::Myriad::getNumUsedBlocks()
624{
625 return iUsed;
626}
627
628Bu::size Bu::Myriad::getTotalUsedBytes()
629{
630 MutexLocker mLock( mHeader );
631 TRACE("mHeader locked.");
632
633 Bu::size iTotalSize = 0;
634 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
635 { 261 {
636 iTotalSize += (*i)->getSize(); 262 // Provide an existing free block.
263 return lFreeBlocks.peekPop();
637 } 264 }
638 TRACE("mHeader unlocked...");
639 return iTotalSize;
640} 265}
641 266
642Bu::size Bu::Myriad::getTotalUnusedBytes() 267void Bu::Myriad::openStream( StreamId id )
643{ 268{
644 MutexLocker mLock( mHeader ); 269 Bu::MutexLocker l( mAccess );
645 TRACE("mHeader locked."); 270 hStream.get( id )->open();
646
647 Bu::size iTotalSize = (iBlocks-iUsed)*iBlockSize;
648 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
649 {
650 iTotalSize += iBlockSize - ((Bu::size)(*i)->getSize()%iBlockSize);
651 }
652 TRACE("mHeader unlocked...");
653 return iTotalSize;
654} 271}
655 272
656Bu::size Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize ) 273void Bu::Myriad::closeStream( StreamId id )
657{ 274{
658 MutexLocker mLock( mHeader ); 275 Bu::MutexLocker l( mAccess );
659 TRACE("mHeader locked."); 276 hStream.get( id )->close();
660
661 Bu::size iTotalSize = (iBlocks-iUsed)*iFakeBlockSize;
662 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
663 {
664 iTotalSize += iFakeBlockSize - ((*i)->getSize()%iFakeBlockSize);
665 }
666 TRACE("mHeader unlocked...");
667 return iTotalSize;
668} 277}
669 278
670Bu::Myriad::Stream *Bu::Myriad::findStream( int iId ) 279int32_t Bu::Myriad::blockRead( int32_t iStart, void *pTarget, int32_t iSize )
671{ 280{
672 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 281 int32_t iUpperSize = iBlockSize - (iStart%iBlockSize);
673 { 282 Bu::println("Max size within block: %1 vs %2 (start=%3, blocksize=%4)")
674 if( (*i)->iId == iId ) 283 .arg( iUpperSize ).arg( iSize )
675 return *i; 284 .arg( iStart ).arg( iBlockSize );
676 }
677 285
678 throw MyriadException( MyriadException::noSuchStream, 286 int32_t iAmnt = std::min( iSize, iUpperSize );
679 "The requested stream doesn't exist and cannot be opened." ); 287 Bu::MutexLocker l( mBacking );
288 rBacking.setPos( iStart );
680 289
681 return NULL; 290 return rBacking.read( pTarget, iAmnt );
682} 291}
683 292
684Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock ) 293/////////
685{ 294// Bu::Myriad::Stream
686// sio << "Myriad: Reading block " << iBlock << ", bytes " 295//
687// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl;
688 Block *pBlock = new Block;
689 pBlock->pData = new char[iBlockSize];
690 sStore.setPos( iBlockSize * (Bu::size)iBlock );
691 sStore.read( pBlock->pData, iBlockSize );
692 pBlock->bChanged = false;
693 pBlock->iBlockIndex = iBlock;
694
695 mActiveBlocks.lock();
696 TRACE("mHeader locked.");
697 hActiveBlocks.insert( iBlock, pBlock );
698 TRACE("mHeader unlocked...");
699 mActiveBlocks.unlock();
700
701 return pBlock;
702}
703 296
704void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock ) 297Bu::Myriad::Stream::Stream( Bu::Myriad &rParent, Bu::Myriad::StreamId iStream,
298 int32_t iSize ) :
299 rParent( rParent ),
300 iStream( iStream ),
301 iSize( iSize ),
302 iOpenCount( 0 ),
303 bStructureChanged( false )
705{ 304{
706 if( pBlock == NULL )
707 return;
708// sio << "Myriad: Releasing block " << pBlock->iBlockIndex << sio.nl;
709 syncBlock( pBlock );
710 mActiveBlocks.lock();
711 TRACE("mHeader locked.");
712 hActiveBlocks.erase( pBlock->iBlockIndex );
713 TRACE("mHeader unlocked...");
714 mActiveBlocks.unlock();
715
716 delete[] pBlock->pData;
717 delete pBlock;
718} 305}
719 306
720void Bu::Myriad::syncBlock( Block *pBlock ) 307Bu::Myriad::Stream::~Stream()
721{ 308{
722 if( pBlock->bChanged )
723 {
724// sio << "Myriad: - Block changed, writing back to stream." << sio.nl;
725 sStore.setPos( iBlockSize * (Bu::size)pBlock->iBlockIndex );
726 sStore.write( pBlock->pData, iBlockSize );
727 pBlock->bChanged = false;
728 }
729}
730
731int Bu::Myriad::streamAddBlock( Stream *pStream )
732{
733 MutexLocker mLock( mHeader );
734 TRACE("mHeader locked.");
735
736 int iBlock = findEmptyBlock();
737 pStream->aBlocks.append( iBlock );
738// bsBlockUsed.setBit( iBlock );
739// bHeaderChanged = true;
740 iUsed++;
741 TRACE("mHeader unlocked...");
742 return iBlock;
743} 309}
744 310
745void Bu::Myriad::setStreamSize( Stream *pStream, long iSize ) 311int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
312 int32_t iSize )
746{ 313{
747 MutexLocker mLock( mHeader ); 314 int32_t iPos = iStart;
748 TRACE("mHeader locked."); 315 int32_t iRead = 0;
749 316 Bu::MutexLocker l( mAccess );
750 if( pStream->getSize() == iSize ) 317 while( iStart > 0 )
751 {
752 TRACE("mHeader unlocked...");
753 return;
754 }
755 else if( pStream->getSize() > iSize )
756 { 318 {
757 // Shrink 319 int32_t iBlock = aBlocks[iStart/rParent.iBlockSize];
758 TRACE(Bu::String("Shrink stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->getSize()).arg(iSize).end() ); 320 int32_t iOffset = iPos % rParent.iBlockSize;
759 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; 321 int32_t iChunkRead = rParent.blockRead(
760 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize ) 322 iBlock*rParent.iBlockSize+iOffset, pTarget, iSize
761 { 323 );
762// if( bsBlockUsed.getBit( pStream->aBlocks.last() ) ) 324 if( iChunkRead == 0 )
763 iUsed--; 325 break;
764// else 326 iRead += iChunkRead;
765// sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl; 327 reinterpret_cast<ptrdiff_t &>(pTarget) += iChunkRead;
766 lFreeBlocks.enqueue( pStream->aBlocks.last() ); 328 iSize -= iChunkRead;
767// bsBlockUsed.setBit( pStream->aBlocks.last(), false );
768 pStream->aBlocks.eraseLast();
769 }
770 pStream->setSize( iSize );
771 bHeaderChanged = true;
772 } 329 }
773 else
774 {
775 // Grow
776 TRACE(Bu::String("Grow stream %1 from %2 to %3").arg(pStream->iId).arg(pStream->getSize()).arg(iSize).end() );
777 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
778 iNewSize < iSize; iNewSize += iBlockSize )
779 {
780 //streamAddBlock( pStream );
781 int iBlock = findEmptyBlock();
782 pStream->aBlocks.append( iBlock );
783// bsBlockUsed.setBit( iBlock );
784// bHeaderChanged = true;
785 iUsed++;
786 }
787 pStream->setSize( iSize );
788 bHeaderChanged = true;
789 }
790 TRACE("mHeader unlocked...");
791}
792 330
793void Bu::Myriad::headerChanged() 331 return iRead;
794{
795 bHeaderChanged = true;
796} 332}
797 333
798bool Bu::Myriad::isMyriad( Bu::Stream &sStore ) 334void Bu::Myriad::Stream::open()
799{ 335{
800 uint8_t uTmp; 336 Bu::MutexLocker l( mAccess );
801 337 iOpenCount++;
802 return isMyriad( sStore, uTmp );
803}
804
805bool Bu::Myriad::isMyriad( Bu::Stream &sStore, uint8_t &uTmp )
806{
807 sStore.setPos( 0 );
808
809 unsigned char buf[4];
810 if( sStore.read( buf, 4 ) < 4 )
811 throw MyriadException( MyriadException::emptyStream,
812 "Input stream appears to be empty.");
813 sStore.read( &uTmp, 1 );
814 sStore.setPos( 0 );
815 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
816 {
817 return false;
818 }
819 return true;
820} 338}
821 339
822const Bu::BitString Bu::Myriad::getBlocksUsed() const 340bool Bu::Myriad::Stream::close()
823{ 341{
824 Bu::BitString bs( iBlocks, false ); 342 Bu::MutexLocker l( mAccess );
825 for( int j = 0; j < iBlocks; j++ ) 343 return (bool)(--iOpenCount);
826 bs.setBit( j );
827 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
828 bs.setBit( *i, false );
829 return bs;
830} 344}
831 345
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index e63c29f..07b4a1d 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -1,24 +1,14 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_H 1#ifndef BU_MYRIAD_H
9#define BU_MYRIAD_H 2#define BU_MYRIAD_H
10 3
11#include <stdint.h> 4#include "bu/stream.h"
12#include "bu/bitstring.h"
13#include "bu/exceptionbase.h" 5#include "bu/exceptionbase.h"
6#include "bu/mutex.h"
14#include "bu/array.h" 7#include "bu/array.h"
15#include "bu/hash.h" 8#include "bu/hash.h"
16#include "bu/mutex.h"
17#include "bu/extratypes.h"
18 9
19namespace Bu 10namespace Bu
20{ 11{
21 class Stream;
22 class MyriadStream; 12 class MyriadStream;
23 13
24 subExceptionDeclBegin( MyriadException ) 14 subExceptionDeclBegin( MyriadException )
@@ -31,212 +21,127 @@ namespace Bu
31 noSuchStream, 21 noSuchStream,
32 streamExists, 22 streamExists,
33 invalidStreamId, 23 invalidStreamId,
34 protectedStream 24 protectedStream,
25 invalidParameter,
26 invalidBackingStream,
27 badMode,
35 }; 28 };
36 subExceptionDeclEnd(); 29 subExceptionDeclEnd();
37 30
38 /**
39 * Myriad block-allocated stream multiplexing system. This is a system for
40 * creating streams that contain other streams in a flexible and lightweight
41 * manner. Basically, you can create a file (or any other stream) that can
42 * store any number of flexible, growing streams. The streams within the
43 * Myriad stream are automatically numbered, not named. This works more
44 * or less like a filesystem, but without the extra layer for managing
45 * file and directory links. This would actually be very easy to add
46 * on top of Myriad, but is not required.
47 *
48 * Header format is as follows:
49 *
50 * MMMMvBssssSSSS*
51 * M = Magic number (0AD3FA84)
52 * v = version number
53 * B = Bits per int
54 * s = Blocksize in bytes
55 * S = Number of Streams
56 *
57 * The * represents the Stream headers, one per stream, as follows:
58 * IIIIssss$
59 * I = Id number of the stream
60 * s = size of stream in bytes
61 *
62 * The $ represents the Block headers, one per used block, as follows:
63 * IIII
64 * I = Index of the block
65 *
66 * The stream/block data is interleaved in the header, so all blocks stored
67 * with one stream are together. The block headers are in order, and the
68 * data in them is required to be "solid" you cannot fill partial blocks
69 * mid-way through a stream.
70 *
71 * The initial block starts with the nids header, and is both the zero block
72 * and the zero stream. For now, the minimum block size is the size needed
73 * to store the base header, the zero stream header, and the first two
74 * blocks of the zero stream, so 30 bytes. Since it's reccomended to use
75 * a size that will fit evenly into filesystem blocks, then a size of 32 is
76 * probably the smallest reccomended size because all powers of two equal
77 * to or greater than 32 are evenly divisible by 32.
78 *
79 * I have had a thought that if the block size were smaller than 42 bytes
80 * the header would consume the first N blocks where N * block size is
81 * enough space to house the initial header, the first stream header, and
82 * the first N block headers. This, of course, causes you to hit an
83 * infinite header if the block size is small enough.
84 */
85 class Myriad 31 class Myriad
86 { 32 {
87 friend class MyriadStream;
88 public: 33 public:
89 /** 34 typedef int32_t StreamId;
90 * Create a Myriad object that uses the given stream to store data. 35 enum Mode {
91 * This stream must be random access. The block size and preallocate 36 None = 0x00,
92 * values passed in are values that will be used if the given stream 37
93 * is empty. In that case the stream will be "formatted" for myriad 38 // Flags
94 * with the specified block size. If there is already a viable Myriad 39 Read = 0x01, ///< Open file for reading
95 * format present in the stream, then the blocksize and preallocate 40 Write = 0x02, ///< Open file for writing
96 * values will be ignored and the values from the stream will be used 41 Create = 0x04, ///< Create file if it doesn't exist
97 * instead. If the stream doesn't appear to be Myriad formatted an 42 Truncate = 0x08, ///< Truncate file if it does exist
98 * exception will be thrown. 43 Append = 0x10, ///< Always append on every write
99 */ 44 NonBlock = 0x20, ///< Open file in non-blocking mode
100 Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ); 45 Exclusive = 0x44, ///< Create file, if it exists then fail
101 virtual ~Myriad(); 46
102 47 // Helpful mixes
103 /** 48 ReadWrite = 0x03, ///< Open for reading and writing
104 * Destroy whatever data may be in the base stream and create a new 49 WriteNew = 0x0E ///< Create a file (or truncate) for writing.
105 * Myriad system there with the given blocksize. Use this with care, 50 /// Same as Write|Create|Truncate
106 * it will destroy anything that was already in the stream, and 51 };
107 * generally, should not ever have to be used.
108 */
109 void initialize( int iBlockSize, int iPreAllocate=1 );
110
111 /**
112 * Create a new stream within the Myriad system. The ID of the new
113 * stream is returned.
114 */
115 int createStream( int iPreAllocate=1 );
116
117 /**
118 * Create a new stream within the Myriad system with a given id. The
119 * id that you provide will be the new id of the stream unless it's
120 * already used, in which case an error is thrown. This is primarilly
121 * useful when copying an old Myriad file into a new one.
122 */
123 int createStreamWithId( int iId, int iPreAllocate=1 );
124
125 /**
126 * Delete a stream that's already within the Myriad.
127 */
128 void deleteStream( int iId );
129 52
53 public:
130 /** 54 /**
131 * Return a new Stream object assosiated with the given stream ID. 55 * Open existing Myriad stream, or initialize a new one if it doesn't
56 * exist.
57 *
58 * Myriad format V0
59 * 0 - 3: Myriad_MAGIC_CODE (0ad3fa84)
60 * 4 - 4: Version Id (1)
61 * 5 - 5: Bits per integer (32)
62 * 6 - 9: Block size in bytes.
63 * 10 - 13: Number of streams.
64 * 14 - ...: Stream Data
65 *
66 * Stream Data:
67 * 0 - 3: Stream Id
68 * 4 - 7: Size of stream in bytes
69 * 8 - ...: List of blocks in stream (4 bytes per block
132 */ 70 */
133 MyriadStream openStream( int iId ); 71 Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1, int32_t iPreallocateBlocks=-1 );
134 72 virtual ~Myriad();
135 Bu::Array<int> getStreamIds();
136 int getStreamSize( int iId );
137 bool hasStream( int iId );
138 73
139 int getNumStreams(); 74 MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
140 int getBlockSize(); 75 MyriadStream open( StreamId iStream, Mode eMode );
141 int getNumBlocks(); 76 bool erase( StreamId iStream );
142 int getNumUsedBlocks(); 77 bool setSize( StreamId iStream, int32_t iNewSize );
143 Bu::size getTotalUsedBytes();
144 Bu::size getTotalUnusedBytes();
145 Bu::size getTotalUnusedBytes( int iFakeBlockSize );
146 78
147 /** 79 private:
148 * Syncronize the header data, etc. with the storage stream. It's not 80 bool loadMyriad();
149 * a bad idea to call this periodically. 81 void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks );
150 */ 82 int32_t allocateBlock();
151 void sync();
152 83
84 void openStream( StreamId id );
85 void closeStream( StreamId id );
153 /** 86 /**
154 * Read the first few bytes from the given stream and return true/false 87 * Block restricted read, it will not read past the end of the block
155 * depending on weather or not it's a Myriad stream. This will throw 88 * that iStart places it in.
156 * an exception if the stream is empty, or is not random access.
157 */
158 static bool isMyriad( Bu::Stream &sStore, uint8_t &uVer );
159
160 /**
161 * Read the first few bytes from the given stream and return true/false
162 * depending on weather or not it's a Myriad stream. This will throw
163 * an exception if the stream is empty, or is not random access.
164 */ 89 */
165 static bool isMyriad( Bu::Stream &sStore ); 90 int32_t blockRead( int32_t iStart, void *pTarget, int32_t iSize );
166 91
167 const Bu::BitString getBlocksUsed() const; 92 public:
168
169 private:
170 /** 93 /**
171 * Initialize this object based on the data already in the assosiated 94 * Bridge/communication/tracking class for individual Myriad streams.
172 * stream. This will be called automatically for you if you forget, 95 * Not for general use, this is used by Myriad and MyriadStream to
173 * but if you want to pre-initialize for some reason, just call this 96 * control access.
174 * once before you actually start doing anything with your Myriad.
175 */ 97 */
176 void initialize();
177
178 enum
179 {
180 blockUnused = 0xFFFFFFFFUL
181 };
182
183 typedef Bu::Array<int> BlockArray;
184 class Stream 98 class Stream
185 { 99 {
186 public: 100 friend Bu::Myriad;
187 void setSize( int iNewSize );
188 void growTo( int iNewSize );
189 int getSize() const;
190
191 int iId;
192 BlockArray aBlocks;
193
194 private: 101 private:
195 int iSize; 102 Stream( Myriad &rParent, StreamId iStream, int32_t iSize );
196 mutable Bu::Mutex mStream; 103 virtual ~Stream();
197 };
198 typedef Bu::Array<Stream *> StreamArray;
199 104
200 class Block
201 {
202 public: 105 public:
203 char *pData; 106 int32_t read( int32_t iStart, void *pTarget, int32_t iSize );
204 bool bChanged; 107
205 int iBlockIndex; 108 /**
206 }; 109 * Doesn't actually open, just increments the open counter.
207 110 * If the open counter is non-zero then at least one stream has
208 void updateHeader(); 111 * a lock on this stream.
209 int findEmptyBlock(); 112 */
210 113 void open();
211 /** 114
212 *@todo Change this to use a binary search, it's nicer. 115 /**
213 */ 116 * Doesn't actually close, just decrements the open counter.
214 Stream *findStream( int iId ); 117 *@returns true if there are still handles open, false if no
118 * streams have a lock.
119 */
120 bool close();
215 121
216 Block *getBlock( int iBlock ); 122 private:
217 void releaseBlock( Block *pBlock ); 123 mutable Bu::Mutex mAccess;
218 void syncBlock( Block *pBlock ); 124 Myriad &rParent;
125 StreamId iStream;
126 int32_t iSize;
127 Bu::Array<int32_t> aBlocks;
128 int32_t iOpenCount;
129 bool bStructureChanged;
130 };
219 131
220 int streamAddBlock( Stream *pStream ); 132 private:
221 void setStreamSize( Stream *pStream, long iSize );
222 133
223 void headerChanged(); 134 typedef Bu::Hash<StreamId, Stream *> StreamHash;
224 135
225 private: 136 typedef Bu::List<int32_t> IndexList;
226 Bu::Stream &sStore; 137 Bu::Mutex mAccess;
227 int iBlockSize; 138 mutable Bu::Mutex mBacking;
228 int iBlocks; 139 Bu::Stream &rBacking;
229 int iUsed; 140 int32_t iBlockSize;
230 typedef Bu::List<int> IndexList; 141 int32_t iBlockCount;
142 bool bIsNewStream;
143 StreamHash hStream;
231 IndexList lFreeBlocks; 144 IndexList lFreeBlocks;
232// Bu::BitString bsBlockUsed;
233 StreamArray aStreams;
234 typedef Bu::Hash<int, Block *> BlockHash;
235 BlockHash hActiveBlocks;
236 bool bHeaderChanged;
237
238 Bu::Mutex mHeader;
239 Bu::Mutex mActiveBlocks;
240 }; 145 };
241}; 146};
242 147
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
index 3c78bb0..cbbd4fe 100644
--- a/src/stable/myriadstream.cpp
+++ b/src/stable/myriadstream.cpp
@@ -1,254 +1,79 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriadstream.h" 1#include "bu/myriadstream.h"
9 2
10#include <string.h> 3#include "bu/mutexlocker.h"
11
12// #define MYRIAD_STREAM_DEBUG 1
13
14#ifdef MYRIAD_STREAM_DEBUG
15#include "bu/sio.h"
16
17using Bu::sio;
18using Bu::Fmt;
19#endif
20#include "bu/sio.h"
21
22// #define TRACE( x ) Bu::println("%1:%2: %3: %4 - %5").arg(__FILE__).arg( __LINE__ ).arg(__PRETTY_FUNCTION__).arg(rMyriad.sStore.getLocation()).arg(x)
23#define TRACE( x ) (void)0
24 4
25Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad, 5Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad,
26 Bu::Myriad::Stream *pStream ) : 6 Bu::Myriad::Stream *pStream, Bu::Myriad::Mode eMode ) :
27 rMyriad( rMyriad ), 7 rMyriad( rMyriad ),
28 pStream( pStream ), 8 pStream( pStream ),
29 pCurBlock( NULL ), 9 eMode( eMode )
30 iPos( 0 )
31{ 10{
32#ifdef MYRIAD_STREAM_DEBUG 11 if( (eMode&Bu::Myriad::ReadWrite) == 0 )
33 sio << "MyriadStream: " << __LINE__ << ": Created, iId=" << pStream->iId << ", iSize=" 12 {
34 << pStream->iSize << sio.nl; 13 throw Bu::MyriadException( Bu::MyriadException::invalidParameter,
35#endif 14 "MyriadStream must be opened Read or Write or both.");
36 //pCurBlock = rMyriad.newBlock(); 15 }
37 //rMyriad.getBlock( uStream, pCurBlock ); 16 Bu::MutexLocker l( mAccess );
38 //uSize = pCurBlock->uBytesUsed; 17 pStream->open();
39} 18}
40 19
41Bu::MyriadStream::~MyriadStream() 20Bu::MyriadStream::~MyriadStream()
42{ 21{
43 if( pCurBlock ) 22 close();
44 rMyriad.releaseBlock( pCurBlock );
45 //rMyriad.updateStreamSize( uStream, uSize );
46 //rMyriad.deleteBlock( pCurBlock );
47} 23}
48 24
49void Bu::MyriadStream::close() 25void Bu::MyriadStream::close()
50{ 26{
51} 27 Bu::MutexLocker l( mAccess );
52 28 if( eMode )
53Bu::size Bu::MyriadStream::read( void *pBuf, Bu::size nBytes )
54{
55#ifdef MYRIAD_STREAM_DEBUG
56 sio << "MyriadStream: read: " << __LINE__ << ": Started, asked to read " << nBytes << "b."
57 << sio.nl;
58#endif
59 if( nBytes > (Bu::size)pStream->getSize()-iPos )
60 nBytes = pStream->getSize()-iPos;
61 if( nBytes <= 0 )
62 return 0;
63 int iLeft = nBytes;
64#ifdef MYRIAD_STREAM_DEBUG
65 sio << "MyriadStream: read: " << __LINE__ << ": Started, going to read " << nBytes << "b."
66 << sio.nl;
67#endif
68 if( pCurBlock == NULL )
69 { 29 {
70#ifdef MYRIAD_STREAM_DEBUG 30 pStream->close();
71 sio << "MyriadStream: read: " << __LINE__ << ": No block loaded, loading initial block." 31 eMode = Bu::Myriad::None;
72 << sio.nl;
73#endif
74 pCurBlock = rMyriad.getBlock(
75 pStream->aBlocks[iPos/rMyriad.iBlockSize]
76 );
77 } 32 }
78 while( iLeft > 0 ) 33}
79 {
80 int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
81 if( pCurBlock->iBlockIndex != iCurBlock )
82 {
83#ifdef MYRIAD_STREAM_DEBUG
84 sio << "MyriadStream: read: " << __LINE__ << ": Loading new block " << iCurBlock << "."
85 << sio.nl;
86#endif
87 rMyriad.releaseBlock( pCurBlock );
88 pCurBlock = rMyriad.getBlock( iCurBlock );
89 }
90 34
91 int iAmnt = Bu::buMin( 35Bu::size Bu::MyriadStream::read( void *pBuf, size iBytes )
92 Bu::buMin( 36{
93 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
94 iLeft
95 ),
96 pStream->getSize()-iPos
97 );
98#ifdef MYRIAD_STREAM_DEBUG
99 sio << "MyriadStream: read: " << __LINE__ << ": Copying out bytes: "
100 << iPos << "(" << (iPos%rMyriad.iBlockSize) << ")+"
101 << iAmnt
102 << ", " << iLeft << "b left." << sio.nl;
103#endif
104 memcpy(
105 pBuf,
106 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
107 iAmnt
108 );
109 iPos += iAmnt;
110 pBuf = &((char *)pBuf)[iAmnt];
111 iLeft -= iAmnt;
112 }
113 return nBytes;
114} 37}
115 38
116Bu::size Bu::MyriadStream::write( const void *pBuf, Bu::size nBytes ) 39Bu::String Bu::MyriadStream::readLine()
117{ 40{
118 if( nBytes <= 0 ) 41}
119 return 0;
120 42
121#ifdef MYRIAD_STREAM_DEBUG 43Bu::String Bu::MyriadStream::readAll()
122 sio << "MyriadStream: write: " << __LINE__ << ": Started, asked to write " << nBytes << "b." 44{
123 << sio.nl; 45}
124#endif
125 if( nBytes <= 0 )
126 return 0;
127 int iLeft = nBytes;
128 /*
129 if( pCurBlock == NULL )
130 {
131#ifdef MYRIAD_STREAM_DEBUG
132 sio << "MyriadStream: write: No block loaded, loading initial block."
133 << sio.nl;
134#endif
135 pCurBlock = rMyriad.getBlock(
136 pStream->aBlocks[iPos/rMyriad.iBlockSize]
137 );
138 }*/
139
140 while( iLeft > 0 )
141 {
142 int iCurBlock;
143 if( iPos/rMyriad.iBlockSize < pStream->aBlocks.getSize() )
144 {
145 iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
146 }
147 else
148 {
149 iCurBlock = rMyriad.streamAddBlock( pStream );
150#ifdef MYRIAD_STREAM_DEBUG
151 sio << "MyriadStream: write: " << __LINE__ << ": New block allocated and appended: "
152 << iCurBlock << "." << sio.nl;
153 46
154#endif 47Bu::size Bu::MyriadStream::write( const void *pBuf, size iBytes )
155 } 48{
156 if( !pCurBlock || pCurBlock->iBlockIndex != iCurBlock ) 49}
157 {
158#ifdef MYRIAD_STREAM_DEBUG
159 sio << "MyriadStream: write: " << __LINE__ << ": Loading new block " << iCurBlock << "."
160 << sio.nl;
161#endif
162 rMyriad.releaseBlock( pCurBlock );
163 pCurBlock = rMyriad.getBlock( iCurBlock );
164 }
165 pCurBlock->bChanged = true;
166
167 // There are two main writing modes when it comes down to it.
168 // Overwrite mode and append mode. Append is what pretty much always
169 // happens when creating a new stream.
170 if( iPos < pStream->getSize() )
171 {
172 int iAmnt = Bu::buMin(
173 Bu::buMin(
174 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
175 iLeft
176 ),
177 pStream->getSize()-iPos
178 );
179#ifdef MYRIAD_STREAM_DEBUG
180 sio << "MyriadStream: write (ovr): " << __LINE__ << ": Copying in bytes: "
181 << (iPos%rMyriad.iBlockSize) << "+"
182 << iAmnt
183 << ", " << iLeft << "b left." << sio.nl;
184#endif
185 memcpy(
186 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
187 pBuf,
188 iAmnt
189 );
190 iPos += iAmnt;
191 pBuf = &((char *)pBuf)[iAmnt];
192 iLeft -= iAmnt;
193 }
194 else
195 {
196 int iAmnt = Bu::buMin(
197 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
198 iLeft
199 );
200#ifdef MYRIAD_STREAM_DEBUG
201 sio << "MyriadStream: write (app): " << __LINE__ << ": Copying in bytes: "
202 << (iPos%rMyriad.iBlockSize) << "+"
203 << iAmnt
204 << ", " << iLeft << "b left." << sio.nl;
205#endif
206 memcpy(
207 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
208 pBuf,
209 iAmnt
210 );
211 iPos += iAmnt;
212 TRACE(Bu::String("Stream=%1 - pStream->iSize(%2) += iAmnt(%3)").arg(pStream->iId).arg( pStream->getSize() ).arg(iAmnt).end());
213 pStream->growTo( pStream->getSize()+iAmnt );
214 TRACE(Bu::String("Stream=%1 - pStream->iSize = %2").arg(pStream->iId).arg( pStream->getSize() ).end());
215 rMyriad.headerChanged();
216 pBuf = &((char *)pBuf)[iAmnt];
217 iLeft -= iAmnt;
218 }
219 }
220 50
221 return nBytes; 51Bu::size Bu::MyriadStream::write( const Bu::String &sBuf )
52{
222} 53}
223 54
224Bu::size Bu::MyriadStream::tell() 55Bu::size Bu::MyriadStream::tell()
225{ 56{
226 return iPos;
227} 57}
228 58
229void Bu::MyriadStream::seek( Bu::size offset ) 59void Bu::MyriadStream::seek( size offset )
230{ 60{
231 iPos += offset;
232} 61}
233 62
234void Bu::MyriadStream::setPos( Bu::size pos ) 63void Bu::MyriadStream::setPos( size pos )
235{ 64{
236 iPos = pos;
237} 65}
238 66
239void Bu::MyriadStream::setPosEnd( Bu::size pos ) 67void Bu::MyriadStream::setPosEnd( size pos )
240{ 68{
241 iPos = pStream->getSize()-pos;
242} 69}
243 70
244bool Bu::MyriadStream::isEos() 71bool Bu::MyriadStream::isEos()
245{ 72{
246 return iPos >= pStream->getSize();
247} 73}
248 74
249bool Bu::MyriadStream::isOpen() 75bool Bu::MyriadStream::isOpen()
250{ 76{
251 return true;
252} 77}
253 78
254void Bu::MyriadStream::flush() 79void Bu::MyriadStream::flush()
@@ -257,59 +82,45 @@ void Bu::MyriadStream::flush()
257 82
258bool Bu::MyriadStream::canRead() 83bool Bu::MyriadStream::canRead()
259{ 84{
260 return true;
261} 85}
262 86
263bool Bu::MyriadStream::canWrite() 87bool Bu::MyriadStream::canWrite()
264{ 88{
265 return true;
266} 89}
267 90
268bool Bu::MyriadStream::isReadable() 91bool Bu::MyriadStream::isReadable()
269{ 92{
270 return true;
271} 93}
272 94
273bool Bu::MyriadStream::isWritable() 95bool Bu::MyriadStream::isWritable()
274{ 96{
275 return true;
276} 97}
277 98
278bool Bu::MyriadStream::isSeekable() 99bool Bu::MyriadStream::isSeekable()
279{ 100{
280 return true;
281} 101}
282 102
283bool Bu::MyriadStream::isBlocking() 103bool Bu::MyriadStream::isBlocking()
284{ 104{
285 return true;
286} 105}
287 106
288void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ ) 107void Bu::MyriadStream::setBlocking( bool bBlocking )
289{ 108{
290} 109}
291 110
292void Bu::MyriadStream::setSize( Bu::size iSize ) 111void Bu::MyriadStream::setSize( size iSize )
293{ 112{
294 if( iSize < 0 )
295 iSize = 0;
296 rMyriad.setStreamSize( pStream, iSize );
297 if( iPos > iSize )
298 iPos = iSize;
299} 113}
300 114
301Bu::size Bu::MyriadStream::getSize() const 115Bu::size Bu::MyriadStream::getSize() const
302{ 116{
303 return pStream->getSize();
304} 117}
305 118
306Bu::size Bu::MyriadStream::getBlockSize() const 119Bu::size Bu::MyriadStream::getBlockSize() const
307{ 120{
308 return rMyriad.getBlockSize();
309} 121}
310 122
311Bu::String Bu::MyriadStream::getLocation() const 123Bu::String getLocation() const
312{ 124{
313 return Bu::String("%1").arg( pStream->iId );
314} 125}
315 126
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
index a94a9a2..87192a9 100644
--- a/src/stable/myriadstream.h
+++ b/src/stable/myriadstream.h
@@ -1,38 +1,31 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_STREAM_H 1#ifndef BU_MYRIAD_STREAM_H
9#define BU_MYRIAD_STREAM_H 2#define BU_MYRIAD_STREAM_H
10 3
11#include "bu/stream.h" 4#include "bu/stream.h"
5#include "bu/myriadstream.h"
12#include "bu/myriad.h" 6#include "bu/myriad.h"
13 7
14namespace Bu 8namespace Bu
15{ 9{
16 class MyriadStream : public Bu::Stream 10 class MyriadStream : public Bu::Stream
17 { 11 {
18 friend class Myriad; 12 friend class Myriad;
19 private: 13 private:
20 /** 14 MyriadStream( Bu::Myriad &rMyriad, Bu::Myriad::Stream *pStream,
21 * These can only be created by the Myriad class. 15 Bu::Myriad::Mode eMode );
22 */
23 MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream );
24
25 public:
26 virtual ~MyriadStream(); 16 virtual ~MyriadStream();
27 17
18 public:
28 virtual void close(); 19 virtual void close();
29 virtual Bu::size read( void *pBuf, Bu::size nBytes ); 20 virtual size read( void *pBuf, size iBytes );
30 virtual Bu::size write( const void *pBuf, Bu::size nBytes ); 21 virtual Bu::String readLine();
31 using Stream::write; 22 virtual Bu::String readAll();
32 virtual Bu::size tell(); 23 virtual size write( const void *pBuf, size iBytes );
33 virtual void seek( Bu::size offset ); 24 virtual size write( const Bu::String &sBuf );
34 virtual void setPos( Bu::size pos ); 25 virtual size tell();
35 virtual void setPosEnd( Bu::size pos ); 26 virtual void seek( size offset );
27 virtual void setPos( size pos );
28 virtual void setPosEnd( size pos );
36 virtual bool isEos(); 29 virtual bool isEos();
37 virtual bool isOpen(); 30 virtual bool isOpen();
38 virtual void flush(); 31 virtual void flush();
@@ -43,18 +36,16 @@ namespace Bu
43 virtual bool isSeekable(); 36 virtual bool isSeekable();
44 virtual bool isBlocking(); 37 virtual bool isBlocking();
45 virtual void setBlocking( bool bBlocking=true ); 38 virtual void setBlocking( bool bBlocking=true );
46 virtual void setSize( Bu::size iSize ); 39 virtual void setSize( size iSize );
47
48 virtual size getSize() const; 40 virtual size getSize() const;
49 virtual size getBlockSize() const; 41 virtual size getBlockSize() const;
50 virtual Bu::String getLocation() const; 42 virtual Bu::String getLocation() const;
51 43
52 private: 44 private:
53 Myriad &rMyriad; 45 mutable Bu::Mutex mAccess;
54 Myriad::Stream *pStream; 46 Bu::Myriad &rMyriad;
55 Myriad::Block *pCurBlock; 47 Bu::Myriad::Stream *pStream;
56 int iBlockSize; 48 Bu::Myriad::Mode eMode;
57 int iPos;
58 }; 49 };
59}; 50};
60 51
diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp
index 9af301c..21c5ff2 100644
--- a/src/tests/bigmyriad.cpp
+++ b/src/tests/bigmyriad.cpp
@@ -5,17 +5,17 @@
5int main() 5int main()
6{ 6{
7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create ); 7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create );
8 Bu::Myriad m( f, 2048 ); 8 Bu::Myriad m( f, 8, 12 );
9 9
10 char *buf = new char[1024*1024*10]; 10 char *buf = new char[1024*1024*10];
11 memset( buf, 0, 1024*1024*10 ); 11 memset( buf, 0, 1024*1024*10 );
12 12
13 for( int j = 0; j < 250; j++ ) 13 for( int j = 0; j < 250; j++ )
14 { 14 {
15 m.openStream( m.createStream() ).write( buf, 1024*1024*10 ); 15// m.openStream( m.createStream() ).write( buf, 1024*1024*10 );
16// m.sync(); 16// m.sync();
17 printf("\r%03d%%", (j+1)*100/250 ); 17// printf("\r%03d%%", (j+1)*100/250 );
18 fflush( stdout ); 18// fflush( stdout );
19 } 19 }
20 20
21 printf("\n\n"); 21 printf("\n\n");
diff --git a/src/tests/cache.cpp b/src/tests/cache.cpp
deleted file mode 100644
index 4751559..0000000
--- a/src/tests/cache.cpp
+++ /dev/null
@@ -1,289 +0,0 @@
1#include "bu/myriadcache.h"
2#include "bu/uuid.h"
3#include "bu/string.h"
4#include "bu/sio.h"
5
6class Something : public Bu::CacheObject<Bu::Uuid, Something>
7{
8friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s );
9friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s );
10public:
11 Something()
12 {
13 }
14
15 Something( const Bu::String &sName ) :
16 uId( Bu::Uuid::generate() ),
17 sName( sName )
18 {
19 }
20
21 virtual ~Something()
22 {
23 }
24
25 virtual Bu::Uuid getKey() const
26 {
27 return uId;
28 }
29
30 Bu::String getName() const
31 {
32 return sName;
33 }
34
35 void setName( const Bu::String &sNewName )
36 {
37 sName = sNewName;
38 changed();
39 }
40
41 virtual Bu::String toString() const=0;
42
43private:
44 Bu::Uuid uId;
45 Bu::String sName;
46};
47
48class SubSomethingA : public Something
49{
50friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s );
51friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s );
52public:
53 SubSomethingA()
54 {
55 }
56
57 SubSomethingA( const Bu::String &sName, int iNumber ) :
58 Something( sName ),
59 iNumber( iNumber )
60 {
61 }
62
63 virtual Bu::String toString() const
64 {
65 return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber );
66 }
67
68private:
69 int iNumber;
70};
71
72class SubSomethingB : public Something
73{
74friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s );
75friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s );
76public:
77 SubSomethingB()
78 {
79 }
80
81 SubSomethingB( const Bu::String &sName, const Bu::String &sString ) :
82 Something( sName ),
83 sString( sString )
84 {
85 }
86
87 virtual Bu::String toString() const
88 {
89 return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString );
90 }
91
92private:
93 Bu::String sString;
94};
95
96Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s )
97{
98 return ar >> s.uId >> s.sName;
99}
100
101Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s )
102{
103 return ar << s.uId << s.sName;
104}
105
106Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s )
107{
108 return ar >> (Something &)s >> s.iNumber;
109}
110
111Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s )
112{
113 return ar << (Something &)s << s.iNumber;
114}
115
116Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s )
117{
118 return ar >> (Something &)s >> s.sString;
119}
120
121Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s )
122{
123 return ar << (Something &)s << s.sString;
124}
125
126namespace Bu
127{
128 template<>
129 void _cacheObjectSave<const Something>( Bu::Stream &s, const Something *pObject )
130 {
131 Bu::Archive ar( s, Bu::Archive::save );
132 if( typeid(*pObject) == typeid(SubSomethingA) )
133 {
134 ar << (uint8_t)1 << (SubSomethingA &)*pObject;
135 }
136 else if( typeid(*pObject) == typeid(SubSomethingB) )
137 {
138 ar << (uint8_t)2 << (SubSomethingB &)*pObject;
139 }
140 else
141 {
142 Bu::println("Not a recognized type!");
143 throw Bu::ExceptionBase("Not recognized type!");
144 }
145 }
146
147 template<>
148 Something *_cacheObjectLoad<Bu::Uuid, Something>( Bu::CacheObject<Bu::Uuid, Something>::Initializer &initObj, const Bu::Uuid &rKey, Bu::Stream &s )
149 {
150 Bu::Archive ar( s, Bu::Archive::load );
151 uint8_t uType;
152 ar >> uType;
153 switch( uType )
154 {
155 case 1:
156 {
157 SubSomethingA *ret = initObj(new SubSomethingA());
158 ar >> *ret;
159 return ret;
160 }
161 break;
162
163 case 2:
164 {
165 SubSomethingB *ret = initObj(new SubSomethingB());
166 ar >> *ret;
167 return ret;
168 }
169 break;
170
171 default:
172 throw Bu::ExceptionBase("Flagrant error! Invalid type!");
173 }
174
175 return NULL;
176 }
177}
178
179typedef Bu::CachePtr<Bu::Uuid, Something> SomethingPtr;
180typedef Bu::CachePtr<Bu::Uuid, SubSomethingA, Something> SomethingAPtr;
181typedef Bu::CachePtr<Bu::Uuid, SubSomethingB, Something> SomethingBPtr;
182typedef Bu::MyriadCache<Bu::Uuid, Something> SomethingCache;
183
184int main( int, char *[] )
185{
186 Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite);
187 SomethingCache c( fStore );
188
189 SomethingPtr ptr;
190 if( time(NULL)%2 )
191 ptr = c.insert( new SubSomethingA("Hello", 55) ).cast<Something>();
192 else
193 ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast<Something>();
194
195 Bu::println("Something[%1]: %2").
196 arg( ptr.getKey() ).
197 arg( ptr->getName() );
198
199 SomethingCache::KeyList lKeys = c.getKeys();
200 Bu::println("Count: %1").arg( lKeys.getSize() );
201 for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ )
202 {
203 Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->toString() );
204 }
205 Bu::println("Count: %1").arg( c.getSize() );
206
207 SomethingPtr p2 = c.get( ptr.getKey() );
208 Bu::println("%1 == %2").arg( p2->getName() ).arg( ptr->getName() );
209 SomethingPtr p3 = ptr.cast<Something>();
210 Bu::println("%1: %2").arg( p3.getKey() ).arg( p3->getName() );
211
212 return 0;
213}
214
215/*
216int main( int argc, char *argv[] )
217{
218 Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite);
219 SomethingCache c( fStore );
220
221 for( int j = 1; j < argc; j++ )
222 {
223 SomethingPtr ptr = c.insert( new Something(argv[j]) );
224
225 Bu::println("Something[%1]: %2").
226 arg( ptr.getKey() ).
227 arg( ptr->getName() );
228 }
229
230 SomethingCache::KeyList lKeys = c.getKeys();
231 Bu::println("Count: %1").arg( lKeys.getSize() );
232 int j = 0;
233 for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ )
234 {
235 Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() );
236 if( ((j++)%2) )
237 c.erase( *i );
238 }
239 Bu::println("Count: %1").arg( c.getSize() );
240
241 c._debug();
242
243 SomethingPtr p2;
244 SomethingPtr p1( c.get( lKeys.first() ) );
245
246 c._debug();
247
248 {
249 SomethingPtr p2( p1 );
250 c._debug();
251 }
252
253 c._debug();
254
255 p2 = p1;
256
257 c._debug();
258
259 p1.unbind();
260
261 c._debug();
262
263 Bu::println("Name: %1").arg( p1->getName() );
264
265 p1.unbind();
266 p1.lock();
267 p1.unlock();
268
269 c._debug();
270
271 SomethingPtr p3 = c.getLazy( lKeys.first() );
272
273 c._debug();
274
275 {
276 SomethingPtr::Locker l( p3 );
277
278 Bu::println("Name again: %1").arg( p3->getName() );
279
280 p3->setName( p3->getName() + " - again" );
281 }
282
283 c.sync();
284
285 c._debug();
286
287 return 0;
288}
289*/
diff --git a/src/tests/cachedel.cpp b/src/tests/cachedel.cpp
deleted file mode 100644
index 817757c..0000000
--- a/src/tests/cachedel.cpp
+++ /dev/null
@@ -1,288 +0,0 @@
1#include "bu/myriadcache.h"
2#include "bu/uuid.h"
3#include "bu/string.h"
4#include "bu/sio.h"
5#include "bu/membuf.h"
6
7class Something : public Bu::CacheObject<Bu::Uuid, Something>
8{
9friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s );
10friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s );
11public:
12 Something()
13 {
14 }
15
16 Something( const Bu::String &sName ) :
17 uId( Bu::Uuid::generate() ),
18 sName( sName )
19 {
20 }
21
22 virtual ~Something()
23 {
24 }
25
26 virtual Bu::Uuid getKey() const
27 {
28 return uId;
29 }
30
31 Bu::String getName() const
32 {
33 return sName;
34 }
35
36 void setName( const Bu::String &sNewName )
37 {
38 sName = sNewName;
39 changed();
40 }
41
42 virtual Bu::String toString() const=0;
43
44private:
45 Bu::Uuid uId;
46 Bu::String sName;
47};
48
49class SubSomethingA : public Something
50{
51friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s );
52friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s );
53public:
54 SubSomethingA()
55 {
56 }
57
58 SubSomethingA( const Bu::String &sName, int iNumber ) :
59 Something( sName ),
60 iNumber( iNumber )
61 {
62 }
63
64 virtual Bu::String toString() const
65 {
66 return Bu::String("[typeA] %1 (%2)").arg( getName() ).arg( iNumber );
67 }
68
69private:
70 int iNumber;
71};
72
73class SubSomethingB : public Something
74{
75friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s );
76friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s );
77public:
78 SubSomethingB()
79 {
80 }
81
82 SubSomethingB( const Bu::String &sName, const Bu::String &sString ) :
83 Something( sName ),
84 sString( sString )
85 {
86 }
87
88 virtual Bu::String toString() const
89 {
90 return Bu::String("[typeB] %1 (%2)").arg( getName() ).arg( sString );
91 }
92
93private:
94 Bu::String sString;
95};
96
97Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, Something &s )
98{
99 return ar >> s.uId >> s.sName;
100}
101
102Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const Something &s )
103{
104 return ar << s.uId << s.sName;
105}
106
107Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingA &s )
108{
109 return ar >> (Something &)s >> s.iNumber;
110}
111
112Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingA &s )
113{
114 return ar << (Something &)s << s.iNumber;
115}
116
117Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, SubSomethingB &s )
118{
119 return ar >> (Something &)s >> s.sString;
120}
121
122Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const SubSomethingB &s )
123{
124 return ar << (Something &)s << s.sString;
125}
126
127namespace Bu
128{
129 template<>
130 void _cacheObjectSave<const Something>( Bu::Stream &s, const Something *pObject )
131 {
132 Bu::Archive ar( s, Bu::Archive::save );
133 if( typeid(*pObject) == typeid(SubSomethingA) )
134 {
135 ar << (uint8_t)1 << (SubSomethingA &)*pObject;
136 }
137 else if( typeid(*pObject) == typeid(SubSomethingB) )
138 {
139 ar << (uint8_t)2 << (SubSomethingB &)*pObject;
140 }
141 else
142 {
143 Bu::println("Not a recognized type!");
144 throw Bu::ExceptionBase("Not recognized type!");
145 }
146 }
147
148 template<>
149 Something *_cacheObjectLoad<Bu::Uuid, Something>( Bu::CacheObject<Bu::Uuid, Something>::Initializer &initObj, const Bu::Uuid &rKey, Bu::Stream &s )
150 {
151 Bu::Archive ar( s, Bu::Archive::load );
152 uint8_t uType;
153 ar >> uType;
154 switch( uType )
155 {
156 case 1:
157 {
158 SubSomethingA *ret = initObj(new SubSomethingA());
159 ar >> *ret;
160 return ret;
161 }
162 break;
163
164 case 2:
165 {
166 SubSomethingB *ret = initObj(new SubSomethingB());
167 ar >> *ret;
168 return ret;
169 }
170 break;
171
172 default:
173 throw Bu::ExceptionBase("Flagrant error! Invalid type!");
174 }
175
176 return NULL;
177 }
178}
179
180typedef Bu::CachePtr<Bu::Uuid, Something> SomethingPtr;
181typedef Bu::CachePtr<Bu::Uuid, SubSomethingA, Something> SomethingAPtr;
182typedef Bu::CachePtr<Bu::Uuid, SubSomethingB, Something> SomethingBPtr;
183typedef Bu::MyriadCache<Bu::Uuid, Something> SomethingCache;
184
185int main( int, char *[] )
186{
187 Bu::MemBuf mbStore;
188 SomethingCache c( mbStore );
189
190 SomethingPtr ptr;
191 if( time(NULL)%2 )
192 ptr = c.insert( new SubSomethingA("Hello", 55) ).cast<Something>();
193 else
194 ptr = c.insert( new SubSomethingB("Goodbye", "Things") ).cast<Something>();
195
196 Bu::Uuid id = ptr.getKey();
197 Bu::println("Something[%1]: %2").
198 arg( ptr.getKey() ).
199 arg( ptr->getName() );
200
201 Bu::println("has %1: %2").arg( id ).arg( c.has( id ) );
202 c.erase( ptr.getKey() );
203 Bu::println("has %1: %2").arg( id ).arg( c.has( id ) );
204 SomethingPtr b = ptr;
205
206 SomethingPtr p2 = c.insert( new SubSomethingA("new test", 123) ).cast<Something>();
207 id = p2.getKey();
208 p2.unbind();
209 c.erase( id );
210
211 return 0;
212}
213
214/*
215int main( int argc, char *argv[] )
216{
217 Bu::File fStore("test.myr", Bu::File::Create|Bu::File::ReadWrite);
218 SomethingCache c( fStore );
219
220 for( int j = 1; j < argc; j++ )
221 {
222 SomethingPtr ptr = c.insert( new Something(argv[j]) );
223
224 Bu::println("Something[%1]: %2").
225 arg( ptr.getKey() ).
226 arg( ptr->getName() );
227 }
228
229 SomethingCache::KeyList lKeys = c.getKeys();
230 Bu::println("Count: %1").arg( lKeys.getSize() );
231 int j = 0;
232 for( SomethingCache::KeyList::iterator i = lKeys.begin(); i; i++ )
233 {
234 Bu::println(" - %1: '%2'").arg( *i ).arg( c.get( *i )->getName() );
235 if( ((j++)%2) )
236 c.erase( *i );
237 }
238 Bu::println("Count: %1").arg( c.getSize() );
239
240 c._debug();
241
242 SomethingPtr p2;
243 SomethingPtr p1( c.get( lKeys.first() ) );
244
245 c._debug();
246
247 {
248 SomethingPtr p2( p1 );
249 c._debug();
250 }
251
252 c._debug();
253
254 p2 = p1;
255
256 c._debug();
257
258 p1.unbind();
259
260 c._debug();
261
262 Bu::println("Name: %1").arg( p1->getName() );
263
264 p1.unbind();
265 p1.lock();
266 p1.unlock();
267
268 c._debug();
269
270 SomethingPtr p3 = c.getLazy( lKeys.first() );
271
272 c._debug();
273
274 {
275 SomethingPtr::Locker l( p3 );
276
277 Bu::println("Name again: %1").arg( p3->getName() );
278
279 p3->setName( p3->getName() + " - again" );
280 }
281
282 c.sync();
283
284 c._debug();
285
286 return 0;
287}
288*/
diff --git a/src/tests/mfstest.cpp b/src/tests/mfstest.cpp
deleted file mode 100644
index 813ef8e..0000000
--- a/src/tests/mfstest.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
1#include <bu/file.h>
2#include <bu/myriadfs.h>
3#include <bu/myriadstream.h>
4#include <bu/sio.h>
5
6class Obstore
7{
8public:
9 Obstore() :
10 fStore("test.mfs", Bu::File::WriteNew|Bu::File::Read ),
11 fsOb( fStore, 1024 )
12 {
13 }
14
15 virtual ~Obstore()
16 {
17 }
18
19 Bu::String getObject( const Bu::String &sPath )
20 {
21 try
22 {
23 Bu::MyriadStream s = fsOb.open(
24 sPath,
25 Bu::MyriadFs::Read
26 );
27 return s.readAll();
28 }
29 catch( Bu::ExceptionBase &e )
30 {
31 Bu::println("Exception opening file: %1").arg( e.what() );
32 return Bu::String();
33 }
34 }
35
36 void storeObject( const Bu::String &sPath, const Bu::String &sData )
37 {
38 for( Bu::String::const_iterator i = sPath.begin()+1; i; i++ )
39 {
40 if( *i == '/' )
41 {
42 Bu::String sChunk( sPath.begin(), i );
43 fsOb.mkDir( sChunk, 0664 );
44 }
45 }
46 try
47 {
48 Bu::MyriadStream s = fsOb.open(
49 sPath,
50 Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate
51 );
52 s.write( sData );
53 }
54 catch(Bu::ExceptionBase &e )
55 {
56 Bu::println("Exception opening file: %1").arg( e.what() );
57 }
58 }
59
60 Bu::File fStore;
61 Bu::MyriadFs fsOb;
62};
63
64int main( int, char *[] )
65{
66 Obstore os;
67
68 os.storeObject("/killTeamState.json", "This is a json file.");
69 os.storeObject("/appCredentials/local", "App credentials");
70 os.storeObject("/appCredentials/inckybot", "App credentials");
71 os.storeObject("/appCredentials/playwith", "App credentials");
72 os.storeObject("/restreamIo/users", "User data");
73 os.storeObject("/youTube/users", "User data");
74 os.storeObject("/topTippers.json", "{\"people\": [\"sam\", \"joe\", \"bob\"]}");
75 os.storeObject("/wh40kDecks.json", "");
76
77 Bu::MyriadStream s = os.fsOb.open(
78 "/appCredentials/inckybot",
79 Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate
80 );
81 s.write("Some new credentials, this is a bigger bit of data.");
82
83 os.storeObject("/wh40kDecks.json", "{ \"op\": \"replaceDecks\", \"decks\": { \"lev\": { \"name\": \"Leviathan\", \"primary\": { \"pt\": {\"name\": \"Priority Targets\"}, \"vg\": {\"name\": \"Vital Ground\"}, \"se\": {\"name\": \"Scorched Earth\"}, \"pf\": {\"name\": \"Purge the Foe\"}, \"th\": {\"name\": \"Take and Hold\"}, \"sd\": {\"name\": \"Supply Drop\"}, \"tr\": {\"name\": \"The Ritual\"}, \"ds\": {\"name\": \"Deploy Servo-Skulls\"}, \"sp\": {\"name\": \"Sites of Power\"}, }, \"secondary\": { \"tt\": {\"name\": \"A Tempting Target\"}, \"ad\": {\"name\": \"Area Denial\"}, \"as\": {\"name\": \"Assassination\"}, \"be\": {\"name\": \"Behind Enemy Lines\"}, \"bd\": {\"name\": \"Bring It Down\"}, \"ce\": {\"name\": \"Capture Enemy Outpost\"}, \"cl\": {\"name\": \"Cleanse\"}, \"ds\": {\"name\": \"Defend Stronghold\"}, \"dt\": {\"name\": \"Deploy Teleport Homer\"}, \"ef\": {\"name\": \"Engage On All Fronts\"}, \"eb\": {\"name\": \"Extend Battle Lines\"}, \"is\": {\"name\": \"Investigate Signals\"}, \"np\": {\"name\": \"No Prisoners\"}, \"of\": {\"name\": \"Overwhelming Force\"}, \"sl\": {\"name\": \"Secure No Man’s Land\"}, \"sh\": {\"name\": \"Storm Hostile Objective\"}, }, \"gambit\": { \"dt\": {\"name\": \"Delaying Tactics\"}, \"ee\": {\"name\": \"Emergency Evacuation\"}, \"oc\": {\"name\": \"Orbital Strike Coordinates\"}, \"pp\": {\"name\": \"Proceed As Planned\"}, }, \"missionRule\": { \"cr\": {\"name\": \"Chilling Rain\"}, \"sc\": {\"name\": \"Sweep and Clear\"}, \"hs\": {\"name\": \"Hidden Supplies\"}, \"mi\": {\"name\": \"Minefields\"}, \"to\": {\"name\": \"Targets of Opportunity\"}, \"sf\": {\"name\": \"Scrambler Fields\"}, \"dr\": {\"name\": \"Delayed Reserves\"}, \"cb\": {\"name\": \"Chosen Battlefield\"}, \"mb\": {\"name\": \"Maelstrom of Battle\"}, \"sl\": {\"name\": \"Supply Lines\"}, \"si\": {\"name\": \"Secret Intel\"}, \"vs\": {\"name\": \"Vox Static\"}, }, }, } }");
84 os.storeObject("/wh40kState.json", "{\"cards\": \"hello\"}");
85
86 return 0;
87}
88
diff --git a/src/tests/myriadfs.cpp b/src/tests/myriadfs.cpp
deleted file mode 100644
index 1266e4b..0000000
--- a/src/tests/myriadfs.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
1#include "bu/file.h"
2#include "bu/membuf.h"
3#include "bu/myriadfs.h"
4#include "bu/myriadstream.h"
5#include "bu/sio.h"
6
7using namespace Bu;
8
9int main()
10{
11// Bu::MemBuf mb;
12 Bu::File mb("store.myr", File::Read|File::Write|File::Create );
13 Bu::MyriadFs mfs( mb, 512 );
14
15 sio << "Creating dirs..." << sio.nl;
16 mfs.create("/etc", Bu::MyriadFs::typeDir|0755 );
17 mfs.create("/dev", Bu::MyriadFs::typeDir|0755 );
18 mfs.create("/usr", Bu::MyriadFs::typeDir|0755 );
19
20 mfs.create("/dev/null", Bu::MyriadFs::typeChrDev|0666, 1, 3 );
21 mfs.create("/dev/zero", Bu::MyriadFs::typeChrDev|0666, 1, 5 );
22 mfs.create("/dev/sda", Bu::MyriadFs::typeBlkDev|0660, 8, 0 );
23
24 sio << "Creating files..." << sio.nl;
25 {
26 Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read );
27 ms.write("world!");
28 }
29 {
30 Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read );
31 ms.write("world, again!");
32 }
33 {
34 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write );
35 ms.write("[longer text shouldn't be seen]");
36 }
37 {
38 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate );
39 ms.write("[short text]");
40 }
41
42 sio << "Reading files..." << sio.nl;
43 {
44 Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read );
45 char buf[512];
46 buf[ms.read( buf, 512 )] = '\0';
47 sio << "read: '" << buf << "'" << sio.nl;
48 }
49 {
50 Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read );
51 char buf[512];
52 buf[ms.read( buf, 512 )] = '\0';
53 sio << "read: '" << buf << "'" << sio.nl;
54 }
55 {
56 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Read );
57 char buf[512];
58 buf[ms.read( buf, 512 )] = '\0';
59 sio << "read: '" << buf << "'" << sio.nl;
60 }
61}
62
diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp
deleted file mode 100644
index ccf3d3b..0000000
--- a/src/tools/myriad.cpp
+++ /dev/null
@@ -1,280 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/sio.h"
9#include "bu/file.h"
10#include "bu/myriad.h"
11#include "bu/myriadstream.h"
12#include "bu/optparser.h"
13
14#include <stdlib.h>
15
16using namespace Bu;
17
18enum Mode
19{
20 modeCreate,
21 modeInfo,
22 modeStreamNew,
23 modeStreamDump,
24 modeStreamPut,
25 modeStreamGet,
26 modeBlockMap,
27
28 modeNone
29};
30
31class Options : public OptParser
32{
33public:
34 Options( int argc, char *argv[] ) :
35 eMode( modeNone ),
36 iBlockSize( 64 ),
37 iPreallocate( 0 ),
38 iStream( 0 )
39 {
40 addHelpBanner("Mode of operation:");
41 addOption( eMode, 'c', "create",
42 "Create a new Myriad file." );
43 addOption( eMode, 'i', "info",
44 "Display some info about a Myriad file." );
45 addOption( eMode, 'n', "new",
46 "Create a new sub-stream in a Myriad file.");
47 addOption( eMode, 'd', "dump",
48 "Display a hexdump of a stream from a Myriad file.");
49 addOption( eMode, "get",
50 "Get a file out of a Myriad stream (use --dst).");
51 addOption( eMode, "put",
52 "Put a file into a Myriad stream (usr --src).");
53 addOption( eMode, 'm', "block-map",
54 "Visualize block usage.");
55 addHelpOption();
56
57 addHelpBanner("\nGeneral options:");
58 addOption( iBlockSize, 'b', "block-size", "Set the block size." );
59 addOption( iPreallocate, 'p', "preallocate",
60 "Number of blocks to preallocate." );
61 addOption( sFile, 'f', "file", "Set the Myriad filename." );
62 addOption( iStream, 's', "stream", "Substream to work with.");
63 addOption( sSrc, "src", "Source file for copying into a Myriad file.");
64 addOption( sDst, "dst",
65 "Destination file for copying out of a Myriad file.");
66
67 setOverride( "create", modeCreate );
68 setOverride( "info", modeInfo );
69 setOverride( "new", modeStreamNew );
70 setOverride( "dump", modeStreamDump );
71 setOverride( "put", modeStreamPut );
72 setOverride( "get", modeStreamGet );
73 setOverride( "block-map", modeBlockMap );
74
75 parse( argc, argv );
76 }
77
78 Mode eMode;
79 int iBlockSize;
80 int iPreallocate;
81 int iStream;
82 Bu::String sFile;
83 Bu::String sSrc;
84 Bu::String sDst;
85};
86
87Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
88{
89 sio << "Uh oh, the formatter was called..." << sio.nl;
90 return f;
91}
92
93int main( int argc, char *argv[] )
94{
95 Options opts( argc, argv );
96
97 switch( opts.eMode )
98 {
99 case modeCreate:
100 if( !opts.sFile.isSet() )
101 {
102 sio << "Please specify a file to create." << sio.nl;
103 return 0;
104 }
105 else
106 {
107 File fOut( opts.sFile, File::WriteNew|File::Read );
108 Myriad m( fOut, opts.iBlockSize, opts.iPreallocate );
109 }
110 break;
111
112 case modeInfo:
113 if( !opts.sFile.isSet() )
114 {
115 sio << "Please specify a file to display info about." << sio.nl;
116 return 0;
117 }
118 else
119 {
120 File fIn( opts.sFile, File::Read );
121 Myriad m( fIn );
122 sio << "Myriad info:" << sio.nl
123 << " Block size: " << m.getBlockSize() << sio.nl
124 << " Block count: " << m.getNumBlocks() << sio.nl
125 << " Blocks used: " << m.getNumUsedBlocks() << " ("
126 << m.getNumUsedBlocks()*100/m.getNumBlocks() << "%)"
127 << sio.nl
128 << " Stream count: " << m.getNumStreams() << sio.nl
129 << " Used space: " << m.getTotalUsedBytes() << sio.nl
130 << " Unused space: " << m.getTotalUnusedBytes() << sio.nl
131 << " % of files: " << (double)(m.getNumBlocks()*m.getBlockSize())/(double)(m.getTotalUsedBytes() + m.getTotalUnusedBytes( 4096 ))*100.0 << sio.nl;
132/* Bu::Array<int> aStreams = m.getStreamIds();
133 sio << " Stream info:" << sio.nl;
134 for( Bu::Array<int>::iterator i = aStreams.begin(); i; i++ )
135 {
136 sio << " " << Fmt(4) << *i << ") "
137 << m.getStreamSize( *i ) << "b" << sio.nl;
138 } */
139 }
140 break;
141
142 case modeStreamNew:
143 if( !opts.sFile.isSet() )
144 {
145 sio << "Please specify a file manipulate." << sio.nl;
146 return 0;
147 }
148 else
149 {
150 File fOut( opts.sFile, File::Write|File::Read );
151 Myriad m( fOut );
152 m.createStream( opts.iPreallocate );
153 }
154 break;
155
156 case modeStreamDump:
157 if( !opts.sFile.isSet() )
158 {
159 sio << "Please specify a file to manipulate." << sio.nl;
160 return 0;
161 }
162 else
163 {
164 File fOut( opts.sFile, File::Read );
165 Myriad m( fOut );
166 MyriadStream s = m.openStream( opts.iStream );
167 sio << "Stream " << opts.iStream << ":" << sio.nl;
168 char buf[8];
169 int iPos = 0;
170 while( !s.isEos() )
171 {
172 size_t sAmnt = s.read( buf, 8 );
173 sio << Fmt(5) << iPos << ": ";
174 iPos += sAmnt;
175 for( size_t j = 0; j < sAmnt; j++ )
176 {
177 sio << Fmt::hex(2) << (int)((unsigned char)buf[j])
178 << " ";
179 }
180 for( size_t j = sAmnt; j < 8; j++ )
181 {
182 sio << "-- ";
183 }
184 sio << "| ";
185 for( size_t j = 0; j < sAmnt; j++ )
186 {
187 if( buf[j] >= 32 && buf[j] <= 126 )
188 sio << buf[j] << " ";
189 else
190 sio << " ";
191 }
192 sio << sio.nl;
193 }
194 sio << "Position: " << s.tell() << ", isEos()=" << s.isEos()
195 << sio.nl;
196 }
197 break;
198
199 case modeStreamPut:
200 if( !opts.sFile.isSet() )
201 {
202 sio << "Please specify a file manipulate." << sio.nl;
203 return 0;
204 }
205 else if( !opts.sSrc.isSet() )
206 {
207 sio << "Please specify a source file to read." << sio.nl;
208 }
209 else
210 {
211 File fOut( opts.sFile, File::Write|File::Read );
212 Myriad m( fOut );
213 MyriadStream sOut = m.openStream(
214 m.createStream( opts.iPreallocate )
215 );
216 File fIn( opts.sSrc, File::Read );
217 char buf[1024];
218 while( !fIn.isEos() )
219 {
220 sOut.write( buf, fIn.read( buf, 1024 ) );
221 }
222 }
223 break;
224
225 case modeStreamGet:
226 if( !opts.sFile.isSet() )
227 {
228 sio << "Please specify a file manipulate." << sio.nl;
229 return 0;
230 }
231 else if( !opts.sDst.isSet() )
232 {
233 sio << "Please specify a destination file to write." << sio.nl;
234 }
235 else
236 {
237 File fIn( opts.sFile, File::Write|File::Read );
238 Myriad m( fIn );
239 MyriadStream sIn = m.openStream( opts.iStream );
240 File fOut( opts.sDst, File::Write|File::Create|File::Truncate );
241 char buf[1024];
242 while( !sIn.isEos() )
243 {
244 fOut.write( buf, sIn.read( buf, 1024 ) );
245 }
246 }
247 break;
248
249 case modeBlockMap:
250 if( !opts.sFile.isSet() )
251 {
252 sio << "Please specify a file manipulate." << sio.nl;
253 return 0;
254 }
255 {
256 File fIn( opts.sFile, File::Write|File::Read );
257 Myriad m( fIn );
258 Bu::BitString bs = m.getBlocksUsed();
259 for( int j = 0; j < bs.getSize(); j++ )
260 {
261 if( j>0 && (j%50) == 0 )
262 Bu::println("");
263 if( bs.getBit( j ) )
264 Bu::print("#");
265 else
266 Bu::print("-");
267 }
268 Bu::println("\n");
269 }
270 break;
271
272 case modeNone:
273 sio << "Please select a mode, for more info, try --help."
274 << sio.nl << sio.nl;
275 break;
276 }
277
278 return 0;
279}
280
diff --git a/src/tools/myriadfs.cpp b/src/tools/myriadfs.cpp
deleted file mode 100644
index 587bc89..0000000
--- a/src/tools/myriadfs.cpp
+++ /dev/null
@@ -1,269 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/sio.h"
9#include "bu/streamstack.h"
10#include "bu/file.h"
11#include "bu/myriadfs.h"
12#include "bu/myriadstream.h"
13#include "bu/optparser.h"
14
15enum Mode
16{
17 modeList,
18 modeCat,
19 modeCopyIn,
20 modeCopyOut,
21 modeMkdir,
22 modeInitialize,
23 modeRm,
24
25 modeNone
26};
27
28Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
29{
30 Bu::sio << "Uh oh, the formatter was called..." << Bu::sio.nl;
31 return f;
32}
33
34class Options : public Bu::OptParser
35{
36public:
37 Options( int argc, char *argv[] ) :
38 eMode( modeNone ),
39 iBlockSize( 64 )
40 {
41 addHelpBanner("Options:");
42 addOption( sFile, 'f', "file", "Myriadfs file");
43 addOption( iBlockSize, 'b', "block-size",
44 "Specify the block size when initializing a new MyriadFs");
45
46 setNonOption( Bu::slot( this, &Options::nonOption ) );
47
48 addHelpOption();
49
50 parse( argc, argv );
51 }
52
53 int nonOption( Bu::Array<Bu::String> aParams )
54 {
55 if( eMode == modeNone )
56 {
57 //Bu::println("Checking mode");
58 // First param, must be the mode
59 if( aParams[0] == "ls" )
60 eMode = modeList;
61 else if( aParams[0] == "cat" )
62 eMode = modeCat;
63 else if( aParams[0] == "cp-in" )
64 eMode = modeCopyIn;
65 else if( aParams[0] == "cp-out" )
66 eMode = modeCopyOut;
67 else if( aParams[0] == "mkdir" )
68 eMode = modeMkdir;
69 else if( aParams[0] == "initialize" )
70 eMode = modeInitialize;
71 else if( aParams[0] == "rm" )
72 eMode = modeRm;
73 else
74 Bu::println("Unknown command, try --help");
75 return 0;
76 } else {
77 lParams.append( aParams[0] );
78 }
79 //Bu::println("Got option: \"%1\"").arg( aParams[0] );
80 return 0;
81 }
82
83 Mode eMode;
84 Bu::String sFile;
85 Bu::StringList lParams;
86 int iBlockSize;
87};
88
89int main( int argc, char *argv[] )
90{
91 Options opt( argc, argv );
92
93 if( opt.sFile.isEmpty() )
94 {
95 Bu::println("You must specify a MyriadFs stream (see -f).\n");
96 return 1;
97 }
98
99 if( opt.eMode == modeNone )
100 {
101 Bu::println("Specify an operation to perfrom.\n");
102 return 1;
103 }
104
105 int iFlags = Bu::File::ReadWrite;
106
107 if( opt.eMode == modeInitialize )
108 {
109 iFlags |= Bu::File::Create|Bu::File::Truncate;
110 }
111
112 Bu::File fFs( opt.sFile, iFlags );
113 Bu::MyriadFs mFs( fFs, opt.iBlockSize );
114
115 switch( opt.eMode )
116 {
117 case modeList:
118 {
119 Bu::String sPath = "/";
120 if( opt.lParams.getSize() > 0 )
121 {
122 sPath = opt.lParams.first();
123 }
124 Bu::MyriadFs::Dir lEnt = mFs.readDir( sPath );
125 for( Bu::MyriadFs::Dir::iterator i = lEnt.begin(); i; i++ )
126 {
127 Bu::String sPerm;
128 switch( (*i).uPerms&Bu::MyriadFs::typeMask )
129 {
130 case Bu::MyriadFs::typeDir: sPerm += "d"; break;
131 case Bu::MyriadFs::typeChrDev: sPerm += "c"; break;
132 case Bu::MyriadFs::typeBlkDev: sPerm += "b"; break;
133 case Bu::MyriadFs::typeSymLink: sPerm += "l"; break;
134 case Bu::MyriadFs::typeSocket: sPerm += "s"; break;
135 default: sPerm += "-"; break;
136 }
137 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrR)?"r":"-";
138 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrW)?"w":"-";
139 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrX)?"x":"-";
140 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpR)?"r":"-";
141 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpW)?"w":"-";
142 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpX)?"x":"-";
143 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthR)?"r":"-";
144 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthW)?"w":"-";
145 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthX)?"x":"-";
146 Bu::println("%1 %2 %3:%4 %5 %6")
147 .arg( sPerm )
148 .arg( (*i).iNode )
149 .arg( (*i).iUser )
150 .arg( (*i).iGroup )
151 .arg( (*i).iSize )
152 .arg( (*i).sName );
153 }
154 }
155 break;
156
157 case modeCat:
158 {
159 if( opt.lParams.isEmpty() )
160 {
161 Bu::println("Specify at least one file.");
162 return 1;
163 }
164 int iBlockSize = 1024*1024;
165 char *pBuf = new char[iBlockSize];
166 for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ )
167 {
168 Bu::MyriadStream ms = mFs.open( *i, Bu::MyriadFs::Read );
169 int iRead = 0;
170 do
171 {
172 iRead = ms.read( pBuf, iBlockSize );
173 if( iRead > 0 )
174 {
175 Bu::sioRaw.write( pBuf, iRead );
176 }
177 } while( iRead == iBlockSize );
178 }
179 delete[] pBuf;
180 }
181 break;
182
183 case modeCopyIn:
184 {
185 if( opt.lParams.getSize() != 2 )
186 {
187 Bu::println("Specify a source file and destination file.");
188 return 1;
189 }
190 int iBlockSize = 1024*1024;
191 char *pBuf = new char[iBlockSize];
192 Bu::File fs = Bu::File(
193 opt.lParams.first(), Bu::File::Read );
194 Bu::MyriadStream ms = mFs.open(
195 opt.lParams.last(), Bu::MyriadFs::WriteNew );
196 int iRead = 0;
197 do
198 {
199 iRead = fs.read( pBuf, iBlockSize );
200 if( iRead > 0 )
201 {
202 ms.write( pBuf, iRead );
203 }
204 } while( iRead == iBlockSize );
205 delete[] pBuf;
206 }
207 break;
208
209 case modeCopyOut:
210 {
211 if( opt.lParams.getSize() != 2 )
212 {
213 Bu::println("Specify a source file and destination file.");
214 return 1;
215 }
216 int iBlockSize = 1024*1024;
217 char *pBuf = new char[iBlockSize];
218 Bu::MyriadStream ms = mFs.open(
219 opt.lParams.first(), Bu::MyriadFs::Read );
220 Bu::File fs = Bu::File(
221 opt.lParams.last(), Bu::File::WriteNew );
222 int iRead = 0;
223 do
224 {
225 iRead = ms.read( pBuf, iBlockSize );
226 if( iRead > 0 )
227 {
228 fs.write( pBuf, iRead );
229 }
230 } while( iRead == iBlockSize );
231 delete[] pBuf;
232 }
233 break;
234
235 case modeMkdir:
236 {
237 if( opt.lParams.isEmpty() )
238 {
239 Bu::println("Specify at least one directory.");
240 return 1;
241 }
242 for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ )
243 {
244 mFs.mkDir( *i, 0777 );
245 }
246 }
247 break;
248
249 case modeInitialize:
250 Bu::println("MyriadFs initialized.\n");
251 break;
252
253 case modeRm:
254 {
255 if( opt.lParams.getSize() != 1 )
256 {
257 Bu::println("Specify a file path.");
258 return 1;
259 }
260 mFs.unlink( opt.lParams.first() );
261 }
262 break;
263
264 case modeNone:
265 break;
266 }
267
268 return 0;
269}
diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit
deleted file mode 100644
index 24d3116..0000000
--- a/src/unit/myriad.unit
+++ /dev/null
@@ -1,385 +0,0 @@
1// vim: syntax=cpp
2/*
3 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
4 *
5 * This file is part of the libbu++ library and is released under the
6 * terms of the license contained in the file LICENSE.
7 */
8
9#include "bu/string.h"
10#include "bu/file.h"
11#include "bu/myriad.h"
12#include "bu/myriadstream.h"
13#include "bu/array.h"
14
15#include "bu/sio.h"
16#include "bu/archive.h"
17#include "bu/md5.h"
18#include "bu/unitsuite.h"
19
20#include <stdlib.h>
21
22using namespace Bu;
23
24class VerifyObject
25{
26friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo );
27friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo );
28public:
29 VerifyObject( int iUnits ) :
30 iUnits( iUnits ),
31 iBytesWritten( 0 )
32 {
33 }
34
35 virtual ~VerifyObject()
36 {
37 }
38
39 int getBytesWritten()
40 {
41 return iBytesWritten;
42 }
43
44private:
45 int iUnits;
46 mutable int iBytesWritten;
47};
48
49Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo )
50{
51 Md5 sum;
52 ar << vo.iUnits;
53 vo.iBytesWritten = sizeof(int);
54 sum.addData( &vo.iUnits, sizeof(int) );
55 for( int j = 0; j < vo.iUnits; j++ )
56 {
57 int iRand = random()%128;
58// ar << iRand;
59 Bu::String sDat( iRand );
60 for( int j = 0; j < iRand; j++ )
61 sDat[j] = (char)((uint8_t)(random()%256));
62 ar << sDat;
63 sum.addData( &iRand, sizeof(int) );
64 sum.addData( sDat.getStr(), iRand );
65 vo.iBytesWritten += sizeof(long) + iRand;
66 }
67 Bu::String sRes = sum.getResult();
68 ar << sRes;
69 vo.iBytesWritten += sizeof(long) + sRes.getSize();
70 return ar;
71}
72
73Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo )
74{
75 Md5 sum;
76 ar >> vo.iUnits;
77 sum.addData( &vo.iUnits, sizeof(int) );
78 for( int j = 0; j < vo.iUnits; j++ )
79 {
80 int iRand;
81// ar >> iRand;
82 Bu::String sStr;
83 ar >> sStr;
84 iRand = sStr.getSize();
85 sum.addData( &iRand, sizeof(int) );
86 sum.addData( sStr.getStr(), iRand );
87 }
88 Bu::String sSum;
89 ar >> sSum;
90 unitTest( sSum == sum.getResult() );
91 int iTooMuch;
92 try
93 {
94 ar >> iTooMuch;
95 unitFailed("should have thrown an exception.");
96 }
97 catch( Bu::ExceptionBase &e )
98 {
99 }
100 return ar;
101}
102
103suite Myriad
104{
105 test setSize
106 {
107 String sFileName("myriad-XXXXXXX");
108
109 File fMyriad = tempFile( sFileName );
110 Myriad m( fMyriad, 32 );
111
112 MyriadStream ms = m.openStream( m.createStream() );
113 ms.setSize( 150 );
114 ms.setPos( 145 );
115 char stuff[10];
116 unitTest( ms.read( stuff, 10 ) == 5 );
117
118 ms.setSize( 12 );
119 unitTest( ms.read( stuff, 10 ) == 0 );
120 unitTest( ms.write( "hello", 5 ) == 5 );
121 unitTest( ms.tell() == 17 );
122
123 ms.setSize( 500 );
124 unitTest( ms.tell() == 17 );
125 }
126
127 void addBlock( Stream &s, bool bAppend=true )
128 {
129 if( bAppend )
130 s.setPosEnd( 0 );
131 int iSize = (random()%1016)+8;
132 s.write( &iSize, 4 );
133 char *buf = new char[iSize-8];
134 for( int j = 0; j < iSize-8; j++ )
135 {
136 buf[j] = (j+iSize)%256;
137 }
138 if( random()%2 == 0 )
139 {
140 s.write( buf, iSize-8 );
141 }
142 else
143 {
144 for( int j = 0; j < iSize-8; )
145 {
146 int iAmnt = (random()%8)+1;
147 if( iAmnt+j > iSize-8 )
148 iAmnt = iSize-8-j;
149 iAmnt = s.write( buf+j, iAmnt );
150 j += iAmnt;
151 }
152 }
153 delete[] buf;
154 iSize = ~iSize;
155 s.write( &iSize, 4 );
156 }
157
158 void verifyBlock( Stream &s )
159 {
160 int iSize, iInv;
161 if( s.read( &iSize, 4 ) == 0 )
162 return;
163 if( iSize < 8 || iSize > 1024 )
164 throw ExceptionBase("Read bad data, %d", iSize );
165 char *buf = new char[iSize-8];
166 if( s.read( buf, iSize-8 ) < (size_t)iSize-8 )
167 {
168 delete[] buf;
169 throw ExceptionBase("Block failed verify (insuffient block data).");
170 }
171 for( int j = 0; j < iSize-8; j++ )
172 {
173 if( buf[j] != (char)((j+iSize)%256) )
174 {
175 char b = buf[j];
176 delete[] buf;
177 throw ExceptionBase("Block failed computed data verify "
178 "(%02X==%02X).", b, (char)((j+iSize)%256) );
179 }
180 }
181 delete[] buf;
182 if( s.read( &iInv, 4 ) < 4 )
183 throw ExceptionBase("Block failed verify (insufficient data).");
184 if( iInv != ~iSize )
185 throw ExceptionBase("Block failed inversion verify.");
186 }
187
188 void verifyStream( Stream &s )
189 {
190 s.setPos( 0 );
191 while( !s.isEos() )
192 verifyBlock( s );
193 }
194
195 test stressGrow
196 {
197 String sFileName("myriad-XXXXXXX");
198
199 File fMyriad = tempFile( sFileName );
200 Myriad m( fMyriad );
201 m.initialize( 64 );
202
203 Array<int> aStreams;
204 for( int j = 0; j < 5; j++ )
205 {
206 aStreams.append( m.createStream() );
207 }
208
209 srandom( 512 );
210
211 for( int j = 0; j < 2500; j++ )
212 {
213 switch( random()%5 )
214 {
215 case 0:
216 aStreams.append( m.createStream() );
217 break;
218
219 case 1:
220 if( aStreams.getSize() > 0 )
221 {
222 int iStream = random()%aStreams.getSize();
223 {
224 MyriadStream ms = m.openStream( aStreams[iStream] );
225 verifyStream( ms );
226 }
227 m.deleteStream( aStreams[iStream] );
228 Array<int>::iterator i = aStreams.begin();
229 for( int k = 0; k < iStream; k++ )
230 i++;
231 aStreams.erase( i );
232 }
233 break;
234
235 default:
236 if( aStreams.getSize() == 0 )
237 {
238 aStreams.append( m.createStream() );
239 }
240 {
241 int iStream = random()%aStreams.getSize();
242 MyriadStream ms = m.openStream( aStreams[iStream] );
243 addBlock( ms );
244 verifyStream( ms );
245 }
246 break;
247 }
248 }
249
250 for( Array<int>::iterator i = aStreams.begin(); i; i++ )
251 {
252 MyriadStream ms = m.openStream( *i );
253 verifyStream( ms );
254 }
255 }
256
257 test stressTruncate
258 {
259 String sFileName("myriad-XXXXXXX");
260
261 File fMyriad = tempFile( sFileName );
262 Myriad m( fMyriad );
263 m.initialize( 128 );
264
265 Array<int> aStream;
266
267 for( int j = 0; j < 5; j++ )
268 {
269 aStream.append( m.createStream() );
270 }
271
272 srandom( 1024 );
273
274 char b;
275 for( int iter = 0; iter < 2500; iter++ )
276 {
277 for( Array<int>::iterator i = aStream.begin(); i; i++ )
278 {
279 MyriadStream ms = m.openStream( *i );
280 addBlock( ms, false );
281 ms.setSize( ms.tell() );
282 unitTest( ms.read( &b, 1 ) == 0 );
283 ms.setPos( 0 );
284 verifyBlock( ms );
285 unitTest( ms.read( &b, 1 ) == 0 );
286 }
287 }
288 }
289
290 test stressTruncate2
291 {
292 String sFileName("myriad-XXXXXXX");
293
294 Array<int> aStream;
295
296 setStepCount( 5*2500 + 5 );
297
298 {
299 File fMyriad = tempFile( sFileName );
300 Myriad m( fMyriad, 128 );
301
302 for( int j = 0; j < 5; j++ )
303 {
304 aStream.append( m.createStream() );
305 incProgress();
306 }
307 }
308
309 srandom( 1024 );
310
311 char b;
312 for( int iter = 0; iter < 2500; iter++ )
313 {
314 File fMyriad( sFileName, File::ReadWrite );
315 Myriad m( fMyriad );
316 for( Array<int>::iterator i = aStream.begin(); i; i++ )
317 {
318 MyriadStream ms = m.openStream( *i );
319 addBlock( ms, false );
320 ms.setSize( ms.tell() );
321 unitTest( ms.read( &b, 1 ) == 0 );
322 ms.setPos( 0 );
323 verifyBlock( ms );
324 unitTest( ms.read( &b, 1 ) == 0 );
325 incProgress();
326 }
327 }
328 }
329
330 test stressArchive
331 {
332 String sFileName("myriad-XXXXXX");
333 Array<int> aStream;
334
335 srandom( 2096 );
336
337 setStepCount( 15*250 + 15 );
338
339 {
340 File fMyriad = tempFile( sFileName );
341 Myriad m( fMyriad, 1024 );
342
343 for( int j = 0; j < 15; j++ )
344 {
345 int iStream = m.createStream();
346 aStream.append( iStream );
347 VerifyObject vo( random()%1024 );
348 {
349 MyriadStream ms = m.openStream( iStream );
350 Archive ar( ms, Archive::save );
351 ar << vo;
352 unitTest( ms.tell() == vo.getBytesWritten() );
353 ms.setSize( ms.tell() );
354 }
355 unitTest( m.getStreamSize( iStream ) == vo.getBytesWritten() );
356 incProgress();
357 }
358 }
359
360 for( int iter = 0; iter < 250; iter++ )
361 {
362 File fMyriad( sFileName, File::ReadWrite );
363 Myriad m( fMyriad );
364 for( Array<int>::iterator i = aStream.begin(); i; i++ )
365 {
366 VerifyObject vo( random()%1024 );
367 {
368 MyriadStream ms = m.openStream( *i );
369 Archive ar( ms, Archive::load );
370 ar >> vo;
371 }
372 {
373 MyriadStream ms = m.openStream( *i );
374 Archive ar( ms, Archive::save );
375 ar << vo;
376 unitTest( ms.tell() == vo.getBytesWritten() );
377 ms.setSize( ms.tell() );
378 }
379 unitTest( m.getStreamSize( *i ) == vo.getBytesWritten() );
380 incProgress();
381 }
382 }
383 }
384}
385
diff --git a/src/unstable/myriadcache.cpp b/src/unstable/myriadcache.cpp
deleted file mode 100644
index c9eb9c4..0000000
--- a/src/unstable/myriadcache.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriadcache.h"
diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h
deleted file mode 100644
index 24002b0..0000000
--- a/src/unstable/myriadcache.h
+++ /dev/null
@@ -1,140 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_CACHE_H
9#define BU_MYRIAD_CACHE_H
10
11#include "bu/cachebase.h"
12#include "bu/myriad.h"
13#include "bu/myriadstream.h"
14#include "bu/file.h"
15#include "bu/streamstack.h"
16
17namespace Bu
18{
19 template<typename keytype, typename obtype>
20 class MyriadCache : public Bu::CacheBase<keytype, obtype>
21 {
22 public:
23 MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) :
24 sStore( sStore ),
25 mStore( sStore, iBlockSize, iPreallocate ),
26 bStructureChanged( false )
27 {
28 try
29 {
30 Bu::ReadWriteMutex::ReadLocker l( rwStore );
31 Bu::MyriadStream ms = mStore.openStream( 1 );
32 Bu::Archive ar( ms, Bu::Archive::load );
33 uint8_t uVer;
34 ar >> uVer;
35 switch( uVer )
36 {
37 case 0:
38 ar >> hIndex;
39 break;
40 }
41 }
42 catch(...)
43 {
44 if( mStore.createStreamWithId( 1 ) != 1 )
45 throw Bu::ExceptionBase("Error creating index stream.");
46
47 _sync();
48 }
49 }
50
51 virtual ~MyriadCache()
52 {
53 Bu::CacheBase<keytype,obtype>::sync();
54 }
55
56 using typename Bu::CacheBase<keytype,obtype>::KeyList;
57 using typename Bu::CacheBase<keytype,obtype>::ObjectType;
58
59 virtual typename Bu::CacheBase<keytype,obtype>::KeyList getKeys() const
60 {
61 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
62 return hIndex.getKeys();
63 }
64
65 virtual int getSize() const
66 {
67 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
68 return hIndex.getSize();
69 }
70
71 protected:
72 virtual bool _has( const keytype &key )
73 {
74 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
75 return hIndex.has( key );
76 }
77
78 virtual void _create( const obtype *o )
79 {
80 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
81 hIndex.insert( o->getKey(), mStore.createStream() );
82 _save( o );
83
84 bStructureChanged = true;
85 }
86
87 virtual void _erase( const keytype &k )
88 {
89 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
90 mStore.deleteStream( hIndex.get( k ) );
91 hIndex.erase( k );
92
93 bStructureChanged = true;
94 }
95
96 virtual obtype *_load(
97 typename Bu::CacheObject<keytype, obtype>::Initializer &initObj,
98 const keytype &k
99 )
100 {
101 Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) );
102 return _cacheObjectLoad<keytype, obtype>( initObj, k, ms );
103 }
104
105 virtual void _save( const obtype *o )
106 {
107 Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) );
108 _cacheObjectSave( ms, o );
109 ms.setSize( ms.tell() );
110
111 mStore.sync();
112 }
113
114 virtual void _sync()
115 {
116 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
117 if( !bStructureChanged )
118 return;
119
120 Bu::MyriadStream ms = mStore.openStream( 1 );
121 Bu::Archive ar( ms, Bu::Archive::save );
122 ar << (uint8_t)0 << hIndex;
123 ar.close();
124 ms.setSize( ms.tell() );
125
126 bStructureChanged = false;
127
128 mStore.sync();
129 }
130
131 private:
132 Bu::Stream &sStore;
133 Bu::Myriad mStore;
134 Bu::Hash<keytype, int> hIndex;
135 mutable Bu::ReadWriteMutex rwStore;
136 bool bStructureChanged;
137 };
138}
139
140#endif
diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp
deleted file mode 100644
index 991c21d..0000000
--- a/src/unstable/myriadfs.cpp
+++ /dev/null
@@ -1,722 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/config.h"
9#include "bu/myriadfs.h"
10#include "bu/myriadstream.h"
11
12#include <string.h>
13#include <unistd.h>
14#include <time.h>
15
16#include "bu/sio.h"
17using Bu::sio;
18using Bu::Fmt;
19
20namespace Bu { subExceptionDef( MyriadFsException ) }
21
22#define Myriad_Fs_MAGIC_CODE ((char *)"\xa7\x18\x8b\x39")
23
24Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
25 rStore( rStore ),
26 mStore( rStore, iBlockSize ),
27 iUser( 0 ),
28 iGroup( 0 )
29{
30#ifndef WIN32
31 iUser = getuid();
32 iGroup = getgid();
33#endif
34
35 if( mStore.hasStream( 1 ) )
36 {
37 // Check to see if this is a MyriadFs stream.
38 Bu::MyriadStream ms = mStore.openStream( 1 );
39 char sMagic[4];
40 if( ms.read( sMagic, 4 ) < 4 )
41 throw MyriadFsException("The provided stream does not appear to be "
42 "a MyriadFs stream.");
43 if( ::strncmp( sMagic, Myriad_Fs_MAGIC_CODE, 4 ) )
44 throw MyriadFsException("The provided stream does not appear to be "
45 "a MyriadFs stream.");
46
47 int8_t iVer;
48 ms.read( &iVer, 1 );
49
50 int32_t iNumNodes;
51 ms.read( &iNumNodes, 4 );
52 for( int32_t j = 0; j < iNumNodes; j++ )
53 {
54 int32_t iNode;
55 int32_t iPos;
56 ms.read( &iNode, 4 );
57 ms.read( &iPos, 4 );
58 hNodeIndex.insert( iNode, iPos );
59 }
60 }
61 else
62 {
63 // Create initial header stream
64 {
65 mStore.createStream( 1 );
66 Bu::MyriadStream ms = mStore.openStream( 1 );
67 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
68 int8_t iVer = 1;
69 int32_t iTmp = 1;
70 ms.write( &iVer, 1 );
71 ms.write( &iTmp, 4 ); // iNumNodes
72 iTmp = 0;
73 ms.write( &iTmp, 4 ); // iInode
74 ms.write( &iTmp, 4 ); // iPosition
75 hNodeIndex.insert( 0, 0 );
76 }
77
78 // Create initial inode stream, with one root node.
79 {
80 mStore.createStream( 2 );
81 Bu::MyriadStream ms = mStore.openStream( 2 );
82 RawStat rs;
83 rs.iNode = 0;
84 rs.iUser = iUser;
85 rs.iGroup = iGroup;
86 rs.uPerms = 0755|typeDir;
87 rs.iLinks = 1;
88 rs.uStreamIndex = 3;
89 rs.iCTime = rs.iMTime = rs.iATime = time(NULL);
90 ms.write( &rs, sizeof(RawStat) );
91 }
92
93 // Create inode 0's storage stream.
94 {
95 mStore.createStream( 3 );
96 Bu::MyriadStream ms = mStore.openStream( 3 );
97 int32_t iTmp32 = 0;
98 ms.write( &iTmp32, 4 ); // iChildCount
99 }
100 }
101}
102
103Bu::MyriadFs::~MyriadFs()
104{
105 writeHeader();
106}
107
108void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf )
109{
110 int32_t iParent;
111 int32_t iNode = lookupInode( sPath, iParent );
112 Bu::MyriadStream is = mStore.openStream( 2 );
113 stat( iNode, rBuf, is );
114}
115
116Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode,
117 uint16_t uPerms )
118{
119 int32_t iParent = -1;
120 int32_t iNode;
121 try
122 {
123 iNode = lookupInode( sPath, iParent );
124// sio << "File found." << sio.nl;
125 // The file was found
126 Bu::MyriadStream ms = openByInode( iNode );
127 if( (iMode&Truncate) )
128 {
129 ms.setSize( 0 );
130 }
131 return ms;
132 }
133 catch( Bu::MyriadFsException &e )
134 {
135 if( iParent < 0 )
136 throw;
137
138 // This means that an intermediate path component couldn't be found
139 if( e.getErrorCode() == 1 )
140 throw;
141
142 // The file wasn't found, but the path leading up to it was.
143 // first, figure out the final path element...
144 Bu::String sName = filePart( sPath );
145// sio << "End filename: " << sName << sio.nl;
146// sio << "Parent inode: " << iParent << sio.nl;
147 iNode = create( iParent, sName, (uPerms&permMask)|typeRegFile, 0 );
148// sio << "New iNode: " << iNode << sio.nl;
149 return openByInode( iNode );
150 }
151}
152
153void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms )
154{
155 create( sPath, iPerms, 0 );
156}
157
158void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
159 uint16_t iDevHi, uint16_t iDevLo )
160{
161 create( sPath, iPerms, ((uint32_t)iDevHi<<16)|(uint32_t)iDevLo );
162}
163
164void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
165 uint32_t uSpecial )
166{
167 int32_t iParent = -1;
168// int32_t iNode;
169 try
170 {
171 /*iNode =*/ lookupInode( sPath, iParent );
172// sio << "File found." << sio.nl;
173 }
174 catch( Bu::MyriadFsException &e )
175 {
176 if( iParent < 0 )
177 throw;
178
179 // This means that an intermediate path component couldn't be found
180 if( e.getErrorCode() == 1 )
181 throw;
182
183 // The file wasn't found, but the path leading up to it was.
184 // first, figure out the final path element...
185 Bu::String sName = filePart( sPath );
186// sio << "End filename: " << sName << sio.nl;
187// sio << "Parent inode: " << iParent << sio.nl;
188 /*iNode =*/ create( iParent, sName, iPerms, uSpecial );
189// sio << "New iNode: " << iNode << sio.nl;
190 }
191 // The file was found
192 //throw Bu::MyriadFsException("Path already exists.");
193}
194
195void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms )
196{
197 create( sPath, (iPerms&permMask)|typeDir, 0 );
198}
199
200void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
201 const Bu::String &sPath )
202{
203 int32_t iParent = -1;
204 int32_t iNode;
205 try
206 {
207 iNode = lookupInode( sPath, iParent );
208 }
209 catch( Bu::MyriadFsException &e )
210 {
211 if( iParent < 0 )
212 throw;
213
214 // This means that an intermediate path component couldn't be found
215 if( e.getErrorCode() == 1 )
216 throw;
217
218 // The file wasn't found, but the path leading up to it was.
219 // first, figure out the final path element...
220 Bu::String sName = filePart( sPath );
221// sio << "End filename: " << sName << sio.nl;
222// sio << "Parent inode: " << iParent << sio.nl;
223 iNode = create( iParent, sName, 0777|typeSymLink, 0 );
224// sio << "New iNode: " << iNode << sio.nl;
225 MyriadStream ms = openByInode( iNode );
226 ms.write( sTarget );
227 return;
228 }
229 throw Bu::MyriadFsException("Path already exists.");
230}
231
232void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
233 const Bu::String &sPath )
234{
235 int32_t iParent = -1;
236 int32_t iNode;
237
238 iNode = lookupInode( sTarget, iParent );
239
240 try
241 {
242 lookupInode( sPath, iParent );
243 throw Bu::MyriadFsException("Path already exists.");
244 }
245 catch( Bu::MyriadFsException &e )
246 {
247 if( iParent < 0 )
248 throw;
249
250 // This means that an intermediate path component couldn't be found
251 if( e.getErrorCode() == 1 )
252 throw;
253
254 // The file wasn't found, but the path leading up to it was.
255 // first, figure out the final path element...
256 Bu::String sName = filePart( sPath );
257// sio << "End filename: " << sName << sio.nl;
258// sio << "Parent inode: " << iParent << sio.nl;
259 addToDir( iParent, iNode, sName );
260 MyriadStream is = mStore.openStream( 2 );
261 RawStat rs;
262 readInode( iNode, rs, is );
263 rs.iLinks++;
264 writeInode( rs, is );
265 }
266}
267
268Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
269{
270 int32_t iParent = -1;
271 int32_t iNode;
272 iNode = lookupInode( sPath, iParent );
273 MyriadStream ms = openByInode( iNode );
274 Bu::String sRet;
275 sRet.setSize( ms.getSize() );
276 ms.read( sRet.getStr(), ms.getSize() );
277 return sRet;
278}
279
280Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
281{
282 int32_t iParent = -1;
283 int32_t iNode = lookupInode( sPath, iParent );
284 return readDir( iNode );
285}
286
287void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
288 int64_t iMTime )
289{
290 int32_t iParent = -1;
291 int32_t iNode;
292
293 iNode = lookupInode( sPath, iParent );
294
295 setTimes( iNode, iATime, iMTime );
296}
297
298void Bu::MyriadFs::unlink( const Bu::String &sPath )
299{
300 int32_t iParent = -1;
301// int32_t iNode;
302
303 /*iNode =*/ lookupInode( sPath, iParent );
304
305 Dir lDir = readDir( iParent );
306
307 Bu::String sName = filePart( sPath );
308
309 for( Dir::iterator i = lDir.begin(); i; i++ )
310 {
311 if( sName == (*i).sName )
312 {
313 RawStat rs;
314 readInode( (*i).iNode, rs );
315 if( (rs.uPerms&typeMask) == typeDir )
316 {
317 MyriadStream msDir = mStore.openStream( rs.uStreamIndex );
318 int32_t iCount;
319 msDir.read( &iCount, 4 );
320 if( iCount > 0 )
321 {
322 throw Bu::MyriadFsException("Directory not empty.");
323 }
324 }
325 if( --rs.iLinks == 0 )
326 {
327 destroyNode( (*i).iNode );
328 }
329 else
330 {
331 writeInode( rs );
332 }
333 lDir.erase( i );
334 break;
335 }
336 }
337
338 Bu::MyriadStream ms = openByInode( iParent );
339 int32_t iNumChildren = lDir.getSize();
340 ms.write( &iNumChildren, 4 );
341 for( Dir::iterator i = lDir.begin(); i; i++ )
342 {
343 ms.write( &(*i).iNode, 4 );
344 uint8_t iSize = (*i).sName.getSize();
345 ms.write( &iSize, 1 );
346 ms.write( (*i).sName.getStr(), iSize );
347 }
348 ms.setSize( ms.tell() );
349}
350
351void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
352{
353 int32_t iParent = -1;
354 int32_t iNode;
355 iNode = lookupInode( sPath, iParent );
356 MyriadStream ms = openByInode( iNode );
357 ms.setSize( iSize );
358}
359
360void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo )
361{
362 mkHardLink( sFrom, sTo );
363 unlink( sFrom );
364}
365
366dev_t Bu::MyriadFs::devToSys( uint32_t uDev )
367{
368 return (((uDev&0xFFFF0000)>>8)&0xFF00) | ((uDev&0xFF));
369}
370
371uint32_t Bu::MyriadFs::sysToDev( dev_t uDev )
372{
373 return (((uint32_t)uDev&0xFF00)<<8) | ((uint32_t)uDev&0xFF);
374}
375
376int32_t Bu::MyriadFs::lookupInode( const Bu::String &sPath, int32_t &iParent )
377{
378 if( sPath == "/" )
379 {
380 return 0;
381 }
382 if( sPath[0] == '/' )
383 {
384 // Absolute lookup
385 return lookupInode( sPath.begin()+1, 0, iParent );
386 }
387 else
388 {
389 // Relative lookup
390 throw Bu::ExceptionBase(
391 "Relative lookups in MyriadFs are not working yet.");
392 }
393}
394
395int32_t Bu::MyriadFs::lookupInode( Bu::String::const_iterator iStart,
396 int32_t iNode, int32_t &iParent )
397{
398 iParent = iNode;
399
400 Bu::String::const_iterator iEnd = iStart.find('/');
401 Bu::String sTok( iStart, iEnd );
402
403// sio << "Direcotry component: " << sTok << sio.nl;
404
405 Dir lDir = readDir( iNode );
406
407 for( Dir::iterator i = lDir.begin(); i; i++ )
408 {
409 if( (*i).sName == sTok )
410 {
411 // We found an item
412 if( !iEnd )
413 {
414 // It's the last one in the requested path, return it
415 return (*i).iNode;
416 }
417 else
418 {
419 // Not the last one in our path, double check it's a dir
420 if( ((*i).uPerms&typeMask) == typeDir )
421 {
422 return lookupInode( iEnd+1, (*i).iNode, iParent );
423 }
424 else
425 {
426 iParent = -1;
427 throw Bu::MyriadFsException(
428 "Element '%s' in given path is not a directory.",
429 sTok.getStr() );
430 }
431 }
432 }
433 }
434
435 if( iEnd )
436 throw Bu::MyriadFsException( 1, "Path not found");
437 else
438 throw Bu::MyriadFsException( 2, "Path not found");
439}
440
441void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs )
442{
443 rIs.setPos( hNodeIndex.get( iNode )*sizeof(RawStat) );
444 if( rIs.read( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) )
445 throw Bu::MyriadFsException("Filesystem corruption detected.");
446 if( rs.iNode != iNode )
447 throw Bu::MyriadFsException("Filesystem corruption detected.");
448}
449
450void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs )
451{
452 MyriadStream ms = mStore.openStream( 2 );
453 readInode( iNode, rs, ms );
454}
455
456void Bu::MyriadFs::writeInode( const RawStat &rs,
457 MyriadStream &rOs )
458{
459 rOs.setSize( hNodeIndex.getSize()*sizeof(RawStat) );
460 rOs.setPos( hNodeIndex.get( rs.iNode )*sizeof(RawStat) );
461 if( rOs.write( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) )
462 throw Bu::MyriadFsException("Error writing inode to header stream.");
463}
464
465void Bu::MyriadFs::writeInode( const RawStat &rs )
466{
467 MyriadStream ms = mStore.openStream( 2 );
468 writeInode( rs, ms );
469}
470
471Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode )
472{
473 Bu::MyriadStream ms = openByInode( iNode );
474 int32_t iNumChildren = 0;
475 ms.read( &iNumChildren, 4 );
476
477 Bu::MyriadStream is = mStore.openStream( 2 );
478 Dir lDir;
479 // sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl;
480 for( int32_t j = 0; j < iNumChildren; j++ )
481 {
482 int32_t iChildNode = 0;
483 if( ms.read( &iChildNode, 4 ) < 4 )
484 {
485 throw Bu::MyriadFsException(
486 "Failed to read iChildNode from directory.");
487 }
488 Stat s;
489 stat( iChildNode, s, is );
490 uint8_t uLen;
491 if( ms.read( &uLen, 1 ) < 1 )
492 {
493 throw Bu::MyriadFsException(
494 "Failed to read uLen from directory.");
495 }
496 s.sName.setSize( uLen );
497 if( ms.read( s.sName.getStr(), uLen ) < uLen )
498 {
499 throw Bu::MyriadFsException(
500 "Failed to read sName from directory.");
501 }
502 lDir.append( s );
503
504// sio << " " << s.sName << sio.nl;
505 }
506
507 return lDir;
508}
509
510Bu::MyriadStream Bu::MyriadFs::openByInode( int32_t iNode )
511{
512 RawStat rs;
513 readInode( iNode, rs );
514 switch( (rs.uPerms&typeMask) )
515 {
516 case typeDir:
517 case typeSymLink:
518 case typeRegFile:
519 return mStore.openStream( rs.uStreamIndex );
520
521 default:
522 throw Bu::MyriadFsException(
523 "inode incorrect type for low-level openByInode.");
524 }
525}
526
527void Bu::MyriadFs::addToDir( int32_t iDir, int32_t iNode,
528 const Bu::String &sName )
529{
530 if( sName.getSize() > 255 )
531 {
532 throw Bu::MyriadFsException("Filename too long, max is 255 bytes.");
533 }
534 Bu::MyriadStream ms = openByInode( iDir );
535 int32_t iNumChildren = 0;
536 ms.read( &iNumChildren, 4 );
537 iNumChildren++;
538 ms.setPos( 0 );
539 ms.write( &iNumChildren, 4 );
540 ms.setPosEnd( 0 );
541 ms.write( &iNode, 4 );
542 uint8_t uLen = sName.getSize();
543 ms.write( &uLen, 1 );
544 ms.write( sName.getStr(), uLen );
545}
546
547int32_t Bu::MyriadFs::create( int32_t iParent, const Bu::String &sName,
548 uint16_t uPerms, uint32_t uSpecial )
549{
550 int32_t iNode = allocInode( uPerms, uSpecial );
551 addToDir( iParent, iNode, sName );
552 return iNode;
553}
554
555int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial )
556{
557 int32_t iNode = 0;
558 for(; iNode < 0xfffffff; iNode++ )
559 {
560 if( !hNodeIndex.has( iNode ) )
561 {
562 hNodeIndex.insert( iNode, hNodeIndex.getSize() );
563 RawStat rs;
564 rs.iNode = iNode;
565 rs.iUser = iUser;
566 rs.iGroup = iGroup;
567 rs.uPerms = uPerms;
568 rs.iLinks = 1;
569 switch( (uPerms&typeMask) )
570 {
571 case typeRegFile:
572 case typeSymLink:
573 rs.uStreamIndex = mStore.createStream();
574 break;
575
576 case typeDir:
577 rs.uStreamIndex = mStore.createStream();
578// sio << "Creating directory node, storage: "
579// << rs.uStreamIndex << sio.nl;
580 {
581 Bu::MyriadStream msDir = mStore.openStream(
582 rs.uStreamIndex
583 );
584 uint32_t uSize = 0;
585 msDir.write( &uSize, 4 );
586 }
587 break;
588
589 case typeChrDev:
590 case typeBlkDev:
591 rs.uStreamIndex = uSpecial;
592 break;
593
594 default:
595 rs.uStreamIndex = 0;
596 break;
597 }
598 rs.iATime = time(NULL);
599 rs.iMTime = time(NULL);
600 rs.iCTime = time(NULL);
601 writeInode( rs );
602
603 return iNode;
604 }
605 }
606
607 throw Bu::MyriadFsException(
608 "No inode could be allocated. You've run out!");
609}
610
611void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs )
612{
613 RawStat rs;
614 readInode( iNode, rs, rIs );
615 rBuf.iNode = iNode;
616 rBuf.iUser = rs.iUser;
617 rBuf.iGroup = rs.iGroup;
618 rBuf.uPerms = rs.uPerms;
619 rBuf.iLinks = rs.iLinks;
620 rBuf.iATime = rs.iATime;
621 rBuf.iMTime = rs.iMTime;
622 rBuf.iCTime = rs.iCTime;
623 rBuf.uDev = 0;
624 rBuf.iSize = 0;
625 switch( (rBuf.uPerms&typeMask) )
626 {
627 case typeRegFile:
628 case typeSymLink:
629 rBuf.iSize = mStore.getStreamSize( rs.uStreamIndex );
630 break;
631
632 case typeChrDev:
633 case typeBlkDev:
634 rBuf.uDev = rs.uStreamIndex;
635 break;
636
637 default:
638 rBuf.iSize = 0;
639 break;
640 }
641}
642
643void Bu::MyriadFs::writeHeader()
644{
645 Bu::MyriadStream ms = mStore.openStream( 1 );
646 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
647 int8_t iVer = 1;
648 int32_t iNumNodes = hNodeIndex.getSize();
649 ms.write( &iVer, 1 );
650 ms.write( &iNumNodes, 4 ); // iNumNodes
651 for( NodeIndex::iterator i = hNodeIndex.begin(); i; i++ )
652 {
653 int32_t iNode = i.getKey();
654 int32_t iPosition = i.getValue();
655 ms.write( &iNode, 4 );
656 ms.write( &iPosition, 4 );
657 }
658
659 // Truncate the stream afterwards so we don't use up too much space.
660 ms.setSize( ms.tell() );
661}
662
663void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime )
664{
665 RawStat rs;
666 Bu::MyriadStream is = mStore.openStream( 2 );
667
668 readInode( iNode, rs, is );
669 rs.iATime = iATime;
670 rs.iMTime = iMTime;
671 writeInode( rs, is );
672}
673
674void Bu::MyriadFs::destroyNode( int32_t iNode )
675{
676 if( iNode == 0 )
677 throw Bu::MyriadFsException("You cannot destroy the root.");
678
679 Bu::MyriadStream is = mStore.openStream( 2 );
680
681 // This will be overwritten with the last node
682 uint32_t iPosition = hNodeIndex.get( iNode );
683 RawStat rsOld;
684 readInode( iNode, rsOld, is );
685 switch( (rsOld.uPerms&typeMask) )
686 {
687 case typeRegFile:
688 case typeDir:
689 case typeSymLink:
690 mStore.deleteStream( rsOld.uStreamIndex );
691 break;
692 }
693
694 hNodeIndex.erase( iNode );
695
696 // Read the last node, can't use the helpers, because we don't know the
697 // iNode yet.
698 if( iPosition != hNodeIndex.getSize() )
699 {
700 // If this is the last node, then we don't need to do anything, but
701 // this case handles what to do if we aren't on the last node
702 RawStat rs;
703 is.setPos( (hNodeIndex.getSize())*sizeof(RawStat) );
704 is.read( &rs, sizeof(RawStat) );
705
706 hNodeIndex.get( rs.iNode ) = iPosition;
707 writeInode( rs, is );
708 }
709
710 is.setSize( hNodeIndex.getSize() * sizeof(RawStat) );
711}
712
713Bu::String Bu::MyriadFs::filePart( const Bu::String &sPath )
714{
715 Bu::String::const_iterator iStart = sPath.begin();
716 if( *iStart == '/' )
717 iStart++;
718 for( Bu::String::const_iterator iEnd = iStart.find('/'); iEnd;
719 iStart = iEnd+1, iEnd = iStart.find('/') ) { }
720 return Bu::String( iStart, sPath.end() );
721}
722
diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h
deleted file mode 100644
index ff14292..0000000
--- a/src/unstable/myriadfs.h
+++ /dev/null
@@ -1,205 +0,0 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef MYRIAD_FS_H
9#define MYRIAD_FS_H
10
11#include <sys/types.h>
12
13#include "bu/myriad.h"
14#include "bu/readwritemutex.h"
15
16namespace Bu
17{
18 class Stream;
19
20 subExceptionDecl( MyriadFsException );
21
22 /**
23 * A POSIX compliant, node based filesystem built on top of Myriad.
24 *
25 * A header is placed into stream 1.
26 * Header format:
27 * int32_t iMagicHeader (A7188B39)
28 * int8_t iVersion (1)
29 * int32_t iNumNodes
30 * NodeLookup[iNumNodes] nNode
31 *
32 * Node lookup:
33 * int32_t iInode
34 * int32_t iPosition
35 *
36 * The node headers or inode structures have a base size of 44 bytes.
37 * The name is stored in the directory format.
38 * Basic node header format:
39 * int32_t iNode
40 * int32_t iUser
41 * int32_t iGroup
42 * uint16_t uPerms
43 * int16_t iLinks
44 * uint32_t uStreamIndex
45 * int64_t iATime
46 * int64_t iMTime
47 * int64_t iCTime
48 *
49 * Some types get special formats for their assosiated data stream, or
50 * other special considerations, here's a list:
51 *
52 * - typeFifo: No stream, uStreamIndex unused (probably)
53 * - typeChrDev: No stream, uStreamIndex is device hi/lo
54 * - typeDir: The stream contains a directory contents listing, described
55 * below
56 * - typeBlkDev: No stream, uStreamIndex is device hi/lo
57 * - typeRegFile: The stream is the file data
58 * - typeSymLink: The stream is the destination of the symlink
59 * - typeSocket: No steram, uStreamIndex unused (probably)
60 *
61 * Directory streams have this simple listing format. They contain a list
62 * of all child elements, with no particular order at the moment. The . and
63 * .. entries are not listed, they are implicit:
64 * int32_t iNumNodes
65 * NodeTable[iNumNodes] nChildren
66 *
67 * NodeTable:
68 * int32_t iInode
69 * uint8_t uNameSize
70 * char[uNameSize] sName
71 */
72 class MyriadFs
73 {
74 public:
75 MyriadFs( Bu::Stream &rStore, int iBlockSize=512 );
76 virtual ~MyriadFs();
77
78 enum
79 {
80 permOthX = 0000001,
81 permOthW = 0000002,
82 permOthR = 0000004,
83 permGrpX = 0000010,
84 permGrpW = 0000020,
85 permGrpR = 0000040,
86 permUsrX = 0000100,
87 permUsrW = 0000200,
88 permUsrR = 0000400,
89 permSticky = 0001000,
90 permSetGid = 0002000,
91 permSetUid = 0004000,
92 permMask = 0007777,
93 typeFifo = 0010000,
94 typeChrDev = 0020000,
95 typeDir = 0040000,
96 typeBlkDev = 0060000,
97 typeRegFile = 0100000,
98 typeSymLink = 0120000,
99 typeSocket = 0140000,
100 typeMask = 0170000
101 };
102
103 enum
104 {
105 Read = 0x01, ///< Open file for reading
106 Write = 0x02, ///< Open file for writing
107 Create = 0x04, ///< Create file if it doesn't exist
108 Truncate = 0x08, ///< Truncate file if it does exist
109 Append = 0x10, ///< Always append on every write
110 NonBlock = 0x20, ///< Open file in non-blocking mode
111 Exclusive = 0x44, ///< Create file, if it exists then fail
112
113 // Helpful mixes
114 ReadWrite = 0x03, ///< Open for reading and writing
115 WriteNew = 0x0E ///< Create a file (or truncate) for writing.
116 /// Same as Write|Create|Truncate
117 };
118
119 class Stat
120 {
121 public:
122 int32_t iNode;
123 int32_t iUser;
124 int32_t iGroup;
125 uint16_t uPerms;
126 int16_t iLinks;
127 int64_t iATime;
128 int64_t iMTime;
129 int64_t iCTime;
130 int32_t iSize;
131 uint32_t uDev;
132 Bu::String sName;
133 };
134 typedef Bu::List<Stat> Dir;
135
136 void stat( const Bu::String &sPath, Stat &rBuf );
137 MyriadStream open( const Bu::String &sPath, int iMode,
138 uint16_t uPerms=0664 );
139 void create( const Bu::String &sPath, uint16_t iPerms );
140 void create( const Bu::String &sPath, uint16_t iPerms,
141 uint16_t iDevHi, uint16_t iDevLo );
142 void create( const Bu::String &sPath, uint16_t iPerms,
143 uint32_t uSpecial );
144 void mkDir( const Bu::String &sPath, uint16_t iPerms );
145 void mkSymLink( const Bu::String &sTarget, const Bu::String &sPath );
146 void mkHardLink( const Bu::String &sTarget, const Bu::String &sPath );
147 Bu::String readSymLink( const Bu::String &sPath );
148 Dir readDir( const Bu::String &sPath );
149 void setTimes( const Bu::String &sPath, int64_t iATime,
150 int64_t iMTime );
151 void unlink( const Bu::String &sPath );
152 void setFileSize( const Bu::String &sPath, int32_t iSize );
153 void rename( const Bu::String &sFrom, const Bu::String &sTo );
154
155 static dev_t devToSys( uint32_t uDev );
156 static uint32_t sysToDev( dev_t uDev );
157
158 private:
159 class RawStat
160 {
161 public:
162 int32_t iNode;
163 int32_t iUser;
164 int32_t iGroup;
165 uint16_t uPerms;
166 int16_t iLinks;
167 uint32_t uStreamIndex;
168 int64_t iATime;
169 int64_t iMTime;
170 int64_t iCTime;
171 };
172 typedef Bu::Hash<int32_t, int32_t> NodeIndex;
173
174 private:
175 int32_t lookupInode( const Bu::String &sPath, int32_t &iParent );
176 int32_t lookupInode( Bu::String::const_iterator iStart,
177 int32_t iNode, int32_t &iParent );
178 void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs );
179 void readInode( int32_t iNode, RawStat &rs );
180 void writeInode( const RawStat &rs );
181 void writeInode( const RawStat &rs, MyriadStream &rOs );
182 Dir readDir( int32_t iNode );
183 MyriadStream openByInode( int32_t iNode );
184 void addToDir( int32_t iDir, int32_t iNode, const Bu::String &sName );
185 int32_t create( int32_t iParent, const Bu::String &sName,
186 uint16_t uPerms, uint32_t uSpecial );
187 int32_t allocInode( uint16_t uPerms, uint32_t uSpecial );
188 void stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs );
189 void writeHeader();
190 void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime );
191 void destroyNode( int32_t iNode );
192
193 Bu::String filePart( const Bu::String &sPath );
194
195 private:
196 Bu::Stream &rStore;
197 Bu::Myriad mStore;
198 Bu::ReadWriteMutex mNodeIndex;
199 NodeIndex hNodeIndex;
200 int32_t iUser;
201 int32_t iGroup;
202 };
203};
204
205#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 05:31:19 +0000