aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/experimental/debugmutex.cpp78
-rw-r--r--src/experimental/debugmutex.h102
-rw-r--r--src/stable/mutex.h8
-rw-r--r--src/stable/myriad.cpp167
-rw-r--r--src/stable/myriad.h81
-rw-r--r--src/stable/myriadstream.cpp6
-rw-r--r--src/stable/myriadstream.h1
-rw-r--r--src/tests/bigmyriad.cpp9
-rw-r--r--src/tests/myriad.cpp23
-rw-r--r--src/tests/myriadfs.cpp62
-rw-r--r--src/tools/myriad.cpp77
-rw-r--r--src/tools/myriadfs.cpp269
-rw-r--r--src/unit/myriad.unit385
-rw-r--r--src/unstable/bitstring.cpp12
-rw-r--r--src/unstable/bitstring.h10
-rw-r--r--src/unstable/myriadcache.cpp8
-rw-r--r--src/unstable/myriadcache.h150
-rw-r--r--src/unstable/myriadfs.cpp739
-rw-r--r--src/unstable/myriadfs.h211
19 files changed, 2329 insertions, 69 deletions
diff --git a/src/experimental/debugmutex.cpp b/src/experimental/debugmutex.cpp
new file mode 100644
index 0000000..2b61ae2
--- /dev/null
+++ b/src/experimental/debugmutex.cpp
@@ -0,0 +1,78 @@
1#include "bu/debugmutex.h"
2
3#include "bu/exceptionbase.h"
4
5Bu::DebugMutex::DebugMutex()
6{
7}
8
9Bu::DebugMutex::~DebugMutex()
10{
11}
12
13int Bu::DebugMutex::lock()
14{
15 pthread_t self = pthread_self();
16 mState.lock();
17 bool bFound = false;
18 for( ThreadList::iterator i = lThreads.begin(); i; i++ )
19 {
20 if( (*i) == self )
21 {
22 bFound = true;
23 if( (*i).bLocked == true )
24 {
25 throw Bu::ExceptionBase( Bu::String("Double lock in thread: %1").arg( (*i).sName ).end().getStr() );
26
27 }
28 else
29 {
30 (*i).bLocked = true;
31 }
32 break;
33 }
34 }
35 if( bFound == false )
36 {
37 lThreads.append( ThreadInfo( true ) );
38 }
39 mState.unlock();
40 return Bu::Mutex::lock();
41}
42
43int Bu::DebugMutex::unlock()
44{
45 pthread_t self = pthread_self();
46 mState.lock();
47 bool bFound = false;
48 for( ThreadList::iterator i = lThreads.begin(); i; i++ )
49 {
50 if( (*i) == self )
51 {
52 bFound = true;
53 if( (*i).bLocked == false )
54 {
55 throw Bu::ExceptionBase( Bu::String("Unlock in thread that did not lock: %1").arg( (*i).sName ).end().getStr() );
56
57 }
58 else
59 {
60 (*i).bLocked = false;
61 }
62 break;
63 }
64 }
65 if( bFound == false )
66 {
67 ThreadInfo info( false );
68 throw Bu::ExceptionBase( Bu::String("Unlock in thread that never locked mutex: %1").arg( info.sName ).end().getStr() );
69 }
70 mState.unlock();
71 return Bu::Mutex::unlock();
72}
73
74int Bu::DebugMutex::trylock()
75{
76 return Bu::Mutex::trylock();
77}
78
diff --git a/src/experimental/debugmutex.h b/src/experimental/debugmutex.h
new file mode 100644
index 0000000..ca8ef9f
--- /dev/null
+++ b/src/experimental/debugmutex.h
@@ -0,0 +1,102 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_DEBUG_MUTEX_H
9#define BU_DEBUG_MUTEX_H
10
11#include "bu/mutex.h"
12#include "bu/list.h"
13#include "bu/string.h"
14
15namespace Bu
16{
17 /**
18 * Simple mutex wrapper. Currently this doesn't do anything extra for you
19 * except keep all of the functionality together in an OO sorta' way and
20 * keep you from having to worry about cleaning up your mutexes properly,
21 * or initing them.
22 *@ingroup Threading
23 */
24 class DebugMutex : public Mutex
25 {
26 public:
27 /**
28 * Create an unlocked mutex.
29 */
30 DebugMutex();
31
32 /**
33 * Destroy a mutex. This can only be done when a mutex is unlocked.
34 * Failure to unlock before destroying a mutex object could cause it to
35 * wait for the mutex to unlock, the odds of which are usually farily
36 * low at deconstruction time.
37 */
38 virtual ~DebugMutex();
39
40 /**
41 * Lock the mutex. This causes all future calls to lock on this
42 * instance of mutex to block until the first thread that called mutex
43 * unlocks it. At that point the next thread that called lock will get
44 * a chance to go to work. Because of the nature of a mutex lock it is
45 * a very bad idea to do any kind of serious or rather time consuming
46 * computation within a locked section. This can cause thread-deadlock
47 * and your program may hang.
48 */
49 virtual int lock();
50
51 /**
52 * Unlock the mutex. This allows the next thread that asked for a lock
53 * to lock the mutex and continue with execution.
54 */
55 virtual int unlock();
56
57 /**
58 * Try to lock the mutex. This is the option to go with if you cannot
59 * avoid putting lengthy operations within a locked section. trylock
60 * will attempt to lock the mutex, if the mutex is already locked this
61 * function returns immediately with an error code.
62 */
63 virtual int trylock();
64
65 private:
66 Bu::Mutex mState;
67
68 class ThreadInfo
69 {
70 public:
71 ThreadInfo( bool bLocked=false ) :
72 idThread( pthread_self() ),
73 bLocked( bLocked )
74 {
75 char buf[64];
76 if( pthread_getname_np( idThread, buf, 64 ) == 0 )
77 sName = buf;
78 }
79 ~ThreadInfo() {}
80
81 bool operator==( const ThreadInfo &rhs )
82 {
83 return pthread_equal( idThread, rhs.idThread );
84 }
85
86 bool operator==( const pthread_t &rhs )
87 {
88 return pthread_equal( idThread, rhs );
89 }
90
91 pthread_t idThread;
92 Bu::String sName;
93 bool bLocked;
94 };
95 typedef Bu::List<ThreadInfo> ThreadList;
96 ThreadList lThreads;
97 };
98}
99
100#endif
101
102
diff --git a/src/stable/mutex.h b/src/stable/mutex.h
index d9e8910..8034974 100644
--- a/src/stable/mutex.h
+++ b/src/stable/mutex.h
@@ -33,7 +33,7 @@ namespace Bu
33 * wait for the mutex to unlock, the odds of which are usually farily 33 * wait for the mutex to unlock, the odds of which are usually farily
34 * low at deconstruction time. 34 * low at deconstruction time.
35 */ 35 */
36 ~Mutex(); 36 virtual ~Mutex();
37 37
38 /** 38 /**
39 * Lock the mutex. This causes all future calls to lock on this 39 * Lock the mutex. This causes all future calls to lock on this
@@ -44,13 +44,13 @@ namespace Bu
44 * computation within a locked section. This can cause thread-deadlock 44 * computation within a locked section. This can cause thread-deadlock
45 * and your program may hang. 45 * and your program may hang.
46 */ 46 */
47 int lock(); 47 virtual int lock();
48 48
49 /** 49 /**
50 * Unlock the mutex. This allows the next thread that asked for a lock 50 * Unlock the mutex. This allows the next thread that asked for a lock
51 * to lock the mutex and continue with execution. 51 * to lock the mutex and continue with execution.
52 */ 52 */
53 int unlock(); 53 virtual int unlock();
54 54
55 /** 55 /**
56 * Try to lock the mutex. This is the option to go with if you cannot 56 * Try to lock the mutex. This is the option to go with if you cannot
@@ -58,7 +58,7 @@ namespace Bu
58 * will attempt to lock the mutex, if the mutex is already locked this 58 * will attempt to lock the mutex, if the mutex is already locked this
59 * function returns immediately with an error code. 59 * function returns immediately with an error code.
60 */ 60 */
61 int trylock(); 61 virtual int trylock();
62 62
63 protected: 63 protected:
64 pthread_mutex_t mutex; /**< The internal mutex reference. */ 64 pthread_mutex_t mutex; /**< The internal mutex reference. */
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 1bf2301..492676e 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -12,13 +12,13 @@
12#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \ 12#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
13{ \ 13{ \
14 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ 14 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
15 "Insufficent data reading myriad data from backing stream."); \ 15 "Insufficient data reading myriad data from backing stream."); \
16} (void)0 16} (void)0
17 17
18#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \ 18#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
19{ \ 19{ \
20 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ 20 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
21 "Insufficent data reading from myriad stream."); \ 21 "Insufficient data reading from myriad stream."); \
22} (void)0 22} (void)0
23 23
24namespace Bu 24namespace Bu
@@ -74,7 +74,7 @@ Bu::MyriadStream Bu::Myriad::create( Bu::Myriad::Mode eMode,
74 mhStream.unlock(); 74 mhStream.unlock();
75 bStructureChanged = true; 75 bStructureChanged = true;
76 76
77 return Bu::MyriadStream( *this, pStream, (Mode)(eMode&ReadWrite) ); 77 return Bu::MyriadStream( *this, pStream, eMode&ReadWrite );
78} 78}
79 79
80Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream, 80Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
@@ -84,7 +84,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
84 Bu::MutexLocker l( mhStream ); 84 Bu::MutexLocker l( mhStream );
85 if( (eMode&Create) ) 85 if( (eMode&Create) )
86 { 86 {
87 if( !hStream.has( iStream ) ) 87 if( hStream.has( iStream ) )
88 { 88 {
89 if( (eMode&Exclusive) ) 89 if( (eMode&Exclusive) )
90 { 90 {
@@ -112,7 +112,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
112 } 112 }
113 { 113 {
114 Bu::MutexLocker l2( mBacking ); 114 Bu::MutexLocker l2( mBacking );
115 if( (eMode&Write) && rBacking.isWritable() ) 115 if( (eMode&Write) && !rBacking.isWritable() )
116 { 116 {
117 throw Bu::MyriadException( MyriadException::badMode, 117 throw Bu::MyriadException( MyriadException::badMode,
118 "Backing stream does not support writing."); 118 "Backing stream does not support writing.");
@@ -129,8 +129,43 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
129 return Bu::MyriadStream( *this, pStream, eMode ); 129 return Bu::MyriadStream( *this, pStream, eMode );
130} 130}
131 131
132void Bu::Myriad::erase( Bu::Myriad::StreamId /*iStream*/ ) 132Bu::Myriad::StreamId Bu::Myriad::allocate()
133{ 133{
134 Bu::MutexLocker l( mAccess );
135
136 Stream *pStream = new Stream( *this, ++iLastUsedIndex, 0 );
137 mhStream.lock();
138 hStream.insert( pStream->iStream, pStream );
139 mhStream.unlock();
140 bStructureChanged = true;
141
142 return pStream->iStream;
143}
144
145void Bu::Myriad::erase( Bu::Myriad::StreamId iStream )
146{
147 // For now, let's prevent you from erasing a stream if it's open.
148 Bu::MutexLocker l( mhStream );
149 if( !hStream.has( iStream ) )
150 {
151 throw Bu::MyriadException( Bu::MyriadException::noSuchStream,
152 "No such stream exists.");
153 }
154 Stream *pStream = hStream.get( iStream );
155 Bu::MutexLocker sl( pStream->mAccess );
156 if( pStream->iOpenCount > 0 )
157 {
158 throw Bu::MyriadException( Bu::MyriadException::streamOpen,
159 "Cannot currently erase a stream while it is open.");
160 }
161
162 for( Bu::Array<int32_t>::iterator i = pStream->aBlocks.begin(); i; i++ )
163 {
164 releaseBlock( *i, false );
165 }
166 pStream->aBlocks.clear();
167 hStream.erase( iStream );
168 delete pStream;
134} 169}
135 170
136void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream, 171void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
@@ -144,6 +179,18 @@ void Bu::Myriad::setSize( Bu::Myriad::StreamId iStream,
144 pStream->setSize( iNewSize ); 179 pStream->setSize( iNewSize );
145} 180}
146 181
182int32_t Bu::Myriad::getSize( StreamId iStream ) const
183{
184 Bu::MutexLocker l( mhStream );
185 return hStream.get( iStream )->getSize();
186}
187
188bool Bu::Myriad::exists( StreamId iStream ) const
189{
190 Bu::MutexLocker l( mhStream );
191 return hStream.has( iStream );
192}
193
147Bu::String Bu::Myriad::getLocation() const 194Bu::String Bu::Myriad::getLocation() const
148{ 195{
149 Bu::MutexLocker l( mAccess ); 196 Bu::MutexLocker l( mAccess );
@@ -218,9 +265,62 @@ int32_t Bu::Myriad::getTotalUnusedBytes(int32_t iAssumeBlockSize ) const
218 return iTotal; 265 return iTotal;
219} 266}
220 267
268Bu::Myriad::StreamIdList Bu::Myriad::getStreamList() const
269{
270 mhStream.lock();
271 StreamIdList lIds = hStream.getKeys();
272 mhStream.unlock();
273 lIds.sort();
274 if( lIds.first() == 0 )
275 {
276 lIds.eraseFirst();
277 }
278 return lIds;
279}
280
281Bu::BitString Bu::Myriad::buildBlockUseMap() const
282{
283 Bu::MutexLocker l( mAccess );
284 Bu::BitString bsMap( iBlockCount );
285 bsMap.fill();
286 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
287 {
288 bsMap.setBit( *i, false );
289 }
290 return bsMap;
291}
292
293Bu::Myriad::StreamIdArray Bu::Myriad::buildBlockMap() const
294{
295 Bu::MutexLocker l( mAccess );
296 StreamIdArray bm( iBlockCount );
297 for( int j = 0; j < iBlockCount; j++ )
298 {
299 bm.append( -1 );
300 }
301 Bu::MutexLocker l2( mhStream );
302 for( StreamHash::const_iterator iStream = hStream.begin();
303 iStream; iStream++ )
304 {
305 int32_t iId = iStream.getKey();
306 Stream *pStream = iStream.getValue();
307 for( Bu::Array<int32_t>::const_iterator iBlock =
308 pStream->aBlocks.begin(); iBlock; iBlock++ )
309 {
310 bm[*iBlock] = iId;
311 }
312 }
313 return bm;
314}
315
316void Bu::Myriad::sync()
317{
318 writeHeader();
319}
320
221bool Bu::Myriad::loadMyriad() 321bool Bu::Myriad::loadMyriad()
222{ 322{
223 Bu::println("Load myriad!"); 323 //Bu::println("Load myriad!");
224 char sMagicCode[4]; 324 char sMagicCode[4];
225 rBacking.setPos( 0 ); 325 rBacking.setPos( 0 );
226 MyriadRead( sMagicCode, 4 ); 326 MyriadRead( sMagicCode, 4 );
@@ -323,7 +423,9 @@ bool Bu::Myriad::loadMyriad()
323 } 423 }
324 424
325 lFreeBlocks = hUnusedBlocks.getKeys(); 425 lFreeBlocks = hUnusedBlocks.getKeys();
326 Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); 426 //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
427
428 bIsNewStream = false;
327 429
328 return true; 430 return true;
329} 431}
@@ -350,32 +452,21 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
350 // plus one block index. 452 // plus one block index.
351 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize ); 453 int iHeaderStreamBlocks = blkDiv(iHeaderStreamBytes+4, iBlockSize );
352 454
353 Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") 455 //Bu::println("Initial estimate: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
354 .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
355 .arg( iHeaderStreamBlocks )
356 .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
357 .arg( iHeaderStreamBlocks*iBlockSize );
358 while( iHeaderStreamBytes+(iHeaderStreamBlocks*4) 456 while( iHeaderStreamBytes+(iHeaderStreamBlocks*4)
359 > iHeaderStreamBlocks*iBlockSize ) 457 > iHeaderStreamBlocks*iBlockSize )
360 { 458 {
361 iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize); 459 iHeaderStreamBlocks = blkDiv((iHeaderStreamBytes+((iHeaderStreamBlocks+1)*4)), iBlockSize);
362 if( iHeaderStreamBlocks > 100 ) 460 if( iHeaderStreamBlocks > 100 )
363 break; 461 break;
364 Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).") 462 //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
365 .arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) )
366 .arg( iHeaderStreamBlocks )
367 .arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) )
368 .arg( iHeaderStreamBlocks*iBlockSize );
369 } 463 }
370 464
371 if( iPreallocateBlocks > iHeaderStreamBlocks ) 465 if( iPreallocateBlocks < iHeaderStreamBlocks )
372 { 466 {
373 rBacking.setSize( iBlockSize*iPreallocateBlocks ); 467 iPreallocateBlocks = iHeaderStreamBlocks;
374 }
375 else
376 {
377 rBacking.setSize( iBlockSize*iHeaderStreamBlocks );
378 } 468 }
469 rBacking.setSize( iBlockSize*iPreallocateBlocks );
379 470
380 // 471 //
381 // Write Myriad header 472 // Write Myriad header
@@ -417,12 +508,12 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
417 508
418void Bu::Myriad::writeHeader() 509void Bu::Myriad::writeHeader()
419{ 510{
511 Bu::MutexLocker l( mAccess );
420 if( !rBacking.isWritable() ) 512 if( !rBacking.isWritable() )
421 return; 513 return;
422 Bu::println("Writing stream breakdown:"); 514 //Bu::println("Writing stream breakdown:");
423 Bu::MemBuf mbHeader; 515 Bu::MemBuf mbHeader;
424 { 516 {
425 Bu::MutexLocker l( mAccess );
426 Bu::MutexLocker l2( mhStream ); 517 Bu::MutexLocker l2( mhStream );
427 518
428 int32_t iHdrStreamSize = __calcHeaderSize(); 519 int32_t iHdrStreamSize = __calcHeaderSize();
@@ -449,7 +540,7 @@ void Bu::Myriad::writeHeader()
449 psHeader->iSize = iHdrStreamSize; 540 psHeader->iSize = iHdrStreamSize;
450 } 541 }
451 542
452 Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize ); 543 //Bu::println("Computed header size: %1 bytes. Ver=%2, Bpi=%3, BlockSize=%4").arg( iHdrStreamSize ).arg( 1 ).arg( 32 ).arg( iBlockSize );
453 544
454 uint8_t uVer = 1; 545 uint8_t uVer = 1;
455 uint8_t uBpi = 32; 546 uint8_t uBpi = 32;
@@ -474,17 +565,15 @@ void Bu::Myriad::writeHeader()
474 uint32_t uStreamSize = pStream->getSize(); 565 uint32_t uStreamSize = pStream->getSize();
475 mbHeader.write( &uStreamId, 4 ); 566 mbHeader.write( &uStreamId, 4 );
476 mbHeader.write( &uStreamSize, 4 ); 567 mbHeader.write( &uStreamSize, 4 );
568 int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize );
477 Bu::Array<int32_t> aBlocks = pStream->getBlockList(); 569 Bu::Array<int32_t> aBlocks = pStream->getBlockList();
478 570
479 Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)") 571 //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
480 .arg( *i ).arg( uStreamSize )
481 .arg( aBlocks.getSize() )
482 .arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
483 572
484 for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ ) 573// for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
574 for( int j = 0; j < iBlocks; j++ )
485 { 575 {
486 int32_t iIdx = *i; 576 mbHeader.write( &aBlocks[j], 4 );
487 mbHeader.write( &iIdx, 4 );
488 } 577 }
489 } 578 }
490 579
@@ -517,15 +606,15 @@ int32_t Bu::Myriad::__calcHeaderSize()
517 } 606 }
518 } 607 }
519 608
520 Bu::println("HeaderCalc:"); 609 //Bu::println("HeaderCalc:");
521 Bu::println(" Base (no header stream): %1").arg( iHdrSize ); 610 //Bu::println(" Base (no header stream): %1").arg( iHdrSize );
522 int32_t iNewSize = iHdrSize; 611 int32_t iNewSize = iHdrSize;
523 int32_t iOldSize; 612 int32_t iOldSize;
524 613
525 do { 614 do {
526 iOldSize = iNewSize; 615 iOldSize = iNewSize;
527 iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4; 616 iNewSize = iHdrSize + Bu::blkDiv(iNewSize, iBlockSize)*4;
528 Bu::println(" Recomp: %1").arg( iNewSize ); 617 //Bu::println(" Recomp: %1").arg( iNewSize );
529 } while( iOldSize != iNewSize ); 618 } while( iOldSize != iNewSize );
530 619
531 return iNewSize; 620 return iNewSize;
@@ -705,9 +794,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
705 794
706 if( iStart+iSize >= this->iSize ) 795 if( iStart+iSize >= this->iSize )
707 { 796 {
708 int32_t iDiff = this->iSize-(iStart+iSize); 797 iSize = this->iSize-iStart;
709 iSize -= iDiff;
710 iStart += iDiff;
711 } 798 }
712 799
713 while( iSize > 0 ) 800 while( iSize > 0 )
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 58f700a..5accd1e 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -7,6 +7,8 @@
7#include "bu/array.h" 7#include "bu/array.h"
8#include "bu/hash.h" 8#include "bu/hash.h"
9 9
10#include "bu/bitstring.h"
11
10namespace Bu 12namespace Bu
11{ 13{
12 class MyriadStream; 14 class MyriadStream;
@@ -25,14 +27,25 @@ namespace Bu
25 invalidParameter, 27 invalidParameter,
26 invalidBackingStream, 28 invalidBackingStream,
27 badMode, 29 badMode,
30 streamOpen,
28 }; 31 };
29 subExceptionDeclEnd(); 32 subExceptionDeclEnd();
30 33
34 /**
35 * Myriad Stream Multiplexer. This is a system that allows you to store
36 * many streams within a single backing stream. This is great for databases,
37 * caching, etc. It's fairly lightweight, and allows all streams to grow
38 * dynamically using a block-allocation scheme. This is used extensively
39 * by the caching system and MyriadFs as well as other systems within
40 * libbu++.
41 */
31 class Myriad 42 class Myriad
32 { 43 {
33 public: 44 public:
34 typedef int32_t StreamId; 45 typedef int32_t StreamId;
35 enum Mode { 46 typedef Bu::Array<StreamId> StreamIdArray;
47 typedef Bu::List<StreamId> StreamIdList;
48 enum Mode : int32_t {
36 None = 0x00, 49 None = 0x00,
37 50
38 // Flags 51 // Flags
@@ -42,7 +55,7 @@ namespace Bu
42 Truncate = 0x08, ///< Truncate file if it does exist 55 Truncate = 0x08, ///< Truncate file if it does exist
43 Append = 0x10, ///< Start writing at end of file 56 Append = 0x10, ///< Start writing at end of file
44 //NonBlock = 0x20, ///< Open file in non-blocking mode 57 //NonBlock = 0x20, ///< Open file in non-blocking mode
45 Exclusive = 0x44, ///< Create file, if it exists then fail 58 Exclusive = 0x40, ///< Create file, if it exists then fail
46 59
47 // Helpful mixes 60 // Helpful mixes
48 ReadWrite = 0x03, ///< Open for reading and writing 61 ReadWrite = 0x03, ///< Open for reading and writing
@@ -52,8 +65,9 @@ namespace Bu
52 65
53 public: 66 public:
54 /** 67 /**
55 * Open existing Myriad stream, or initialize a new one if it doesn't 68 * Open existing Myriad container, or initialize a new one if the
56 * exist. 69 * backing stream is empty. If other data is already in the provided
70 * backing stream an error is thrown.
57 * 71 *
58 * Myriad format V0 72 * Myriad format V0
59 * 0 - 3: Myriad_MAGIC_CODE (0ad3fa84) 73 * 0 - 3: Myriad_MAGIC_CODE (0ad3fa84)
@@ -68,13 +82,40 @@ namespace Bu
68 * 4 - 7: Size of stream in bytes 82 * 4 - 7: Size of stream in bytes
69 * 8 - ...: List of blocks in stream (4 bytes per block 83 * 8 - ...: List of blocks in stream (4 bytes per block
70 */ 84 */
71 Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1, int32_t iPreallocateBlocks=-1 ); 85 Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1,
86 int32_t iPreallocateBlocks=-1 );
72 virtual ~Myriad(); 87 virtual ~Myriad();
73 88
89 /**
90 * Creates a new stream open in the specified eMode and, optionally,
91 * preallocates the specificed amount of space. The stream is zero
92 * bytes even if space is preallocated. The open stream is returned,
93 * ready for use. Use this if you don't care what the id is of the
94 * newly created stream.
95 */
74 MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 ); 96 MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
97
98 /**
99 * Open an existing stream or create a new stream with the specified
100 * id (iStream) with the specified eMode. This respects the normal file
101 * modes, see Bu::Myriad::Mode for details.
102 */
75 MyriadStream open( StreamId iStream, Mode eMode ); 103 MyriadStream open( StreamId iStream, Mode eMode );
104
105 /**
106 * Allocate a new stream but do not open it, just ensure it exists and
107 * return the id of the newly allocated stream.
108 */
109 StreamId allocate();
110
111 /**
112 * Erase the stream specified by iStream. This only can work when the
113 * stream is not open at the moment.
114 */
76 void erase( StreamId iStream ); 115 void erase( StreamId iStream );
77 void setSize( StreamId iStream, int32_t iNewSize ); 116 void setSize( StreamId iStream, int32_t iNewSize );
117 int32_t getSize( StreamId iStream ) const;
118 bool exists( StreamId iStream ) const;
78 Bu::String getLocation() const; 119 Bu::String getLocation() const;
79 int32_t getBlockSize() const; 120 int32_t getBlockSize() const;
80 int32_t getTotalBlocks() const; 121 int32_t getTotalBlocks() const;
@@ -83,6 +124,22 @@ namespace Bu
83 int32_t getTotalStreams() const; 124 int32_t getTotalStreams() const;
84 int32_t getTotalUsedBytes() const; 125 int32_t getTotalUsedBytes() const;
85 int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const; 126 int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
127 Bu::BitString buildBlockUseMap() const;
128 StreamIdArray buildBlockMap() const;
129
130 /**
131 * Lists all stream ids that you are allowed to open. Technically there
132 * is always a zero stream, but it is used by Myriad for stream/block
133 * accounting. It works like a normal stream but you should not open
134 * it.
135 */
136 StreamIdList getStreamList() const;
137
138 /**
139 * Flush all caches to the backing stream, write all structural and
140 * header changes.
141 */
142 void sync();
86 143
87 private: 144 private:
88 bool loadMyriad(); 145 bool loadMyriad();
@@ -175,6 +232,20 @@ namespace Bu
175 IndexList lFreeBlocks; 232 IndexList lFreeBlocks;
176 StreamId iLastUsedIndex; 233 StreamId iLastUsedIndex;
177 }; 234 };
235 constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b )
236 {
237 return static_cast<Myriad::Mode>(
238 static_cast<std::underlying_type<Myriad::Mode>::type>(a) &
239 static_cast<std::underlying_type<Myriad::Mode>::type>(b)
240 );
241 }
242 constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b )
243 {
244 return static_cast<Myriad::Mode>(
245 static_cast<std::underlying_type<Myriad::Mode>::type>(a) |
246 static_cast<std::underlying_type<Myriad::Mode>::type>(b)
247 );
248 }
178}; 249};
179 250
180#endif 251#endif
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
index 9ea2e17..eaf91a5 100644
--- a/src/stable/myriadstream.cpp
+++ b/src/stable/myriadstream.cpp
@@ -166,3 +166,9 @@ Bu::String Bu::MyriadStream::getLocation() const
166 return pStream->getLocation(); 166 return pStream->getLocation();
167} 167}
168 168
169Bu::Myriad::StreamId Bu::MyriadStream::getId() const
170{
171 Bu::MutexLocker l( mAccess );
172 return pStream->getStreamId();
173}
174
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
index b86dbd7..27a15d5 100644
--- a/src/stable/myriadstream.h
+++ b/src/stable/myriadstream.h
@@ -39,6 +39,7 @@ namespace Bu
39 virtual size getSize() const; 39 virtual size getSize() const;
40 virtual size getBlockSize() const; 40 virtual size getBlockSize() const;
41 virtual Bu::String getLocation() const; 41 virtual Bu::String getLocation() const;
42 Myriad::StreamId getId() const;
42 43
43 private: 44 private:
44 mutable Bu::Mutex mAccess; 45 mutable Bu::Mutex mAccess;
diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp
index 9d24741..73a3315 100644
--- a/src/tests/bigmyriad.cpp
+++ b/src/tests/bigmyriad.cpp
@@ -5,14 +5,15 @@
5int main() 5int main()
6{ 6{
7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create ); 7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create );
8 Bu::Myriad m( f, 256, 12 ); 8 Bu::Myriad m( f, 512, 12 );
9 9
10 char *buf = new char[1024*1024*10]; 10#define SSIZE (1024*8) // 1024*1024*10
11 char *buf = new char[SSIZE];
11 12
12 for( int j = 0; j < 25; j++ ) 13 for( int j = 0; j < 25; j++ )
13 { 14 {
14 memset( buf, j, 1024*1024*10 ); 15 memset( buf, j, SSIZE );
15 m.create( Bu::Myriad::Write ).write( buf, 1024*1024*10 ); 16 m.create( Bu::Myriad::Write ).write( buf, SSIZE );
16// m.sync(); 17// m.sync();
17 printf("\r%03d%%", (j+1)*100/25 ); 18 printf("\r%03d%%", (j+1)*100/25 );
18 fflush( stdout ); 19 fflush( stdout );
diff --git a/src/tests/myriad.cpp b/src/tests/myriad.cpp
new file mode 100644
index 0000000..ee4eac4
--- /dev/null
+++ b/src/tests/myriad.cpp
@@ -0,0 +1,23 @@
1#include <bu/myriad.h>
2#include <bu/myriadstream.h>
3#include <bu/string.h>
4#include <bu/sio.h>
5#include <bu/file.h>
6
7int main( int , char *[] )
8{
9 Bu::File fMyriad("test.myr", Bu::File::WriteNew|Bu::File::Read );
10 Bu::Myriad m( fMyriad, 32 );
11
12 Bu::MyriadStream ms = m.create( Bu::Myriad::ReadWrite );
13 ms.setSize( 150 );
14 ms.setPos( 145 );
15 char stuff[10];
16 int32_t iRead = ms.read( stuff, 10 );
17
18 Bu::println("Tried to read 10, expect 5, got %1").arg( iRead );
19
20
21 return 0;
22}
23
diff --git a/src/tests/myriadfs.cpp b/src/tests/myriadfs.cpp
new file mode 100644
index 0000000..29ac3d9
--- /dev/null
+++ b/src/tests/myriadfs.cpp
@@ -0,0 +1,62 @@
1#include "bu/file.h"
2#include "bu/membuf.h"
3#include "bu/myriadfs.h"
4#include "bu/myriadstream.h"
5#include "bu/sio.h"
6
7using namespace Bu;
8
9int main()
10{
11// Bu::MemBuf mb;
12 Bu::File mb("store.mfs", File::Read|File::Write|File::Create );
13 Bu::MyriadFs mfs( mb, 512 );
14
15 sio << "Creating dirs..." << sio.nl;
16 mfs.create("/etc", Bu::MyriadFs::typeDir|0755 );
17 mfs.create("/dev", Bu::MyriadFs::typeDir|0755 );
18 mfs.create("/usr", Bu::MyriadFs::typeDir|0755 );
19
20 mfs.create("/dev/null", Bu::MyriadFs::typeChrDev|0666, 1, 3 );
21 mfs.create("/dev/zero", Bu::MyriadFs::typeChrDev|0666, 1, 5 );
22 mfs.create("/dev/sda", Bu::MyriadFs::typeBlkDev|0660, 8, 0 );
23
24 sio << "Creating files..." << sio.nl;
25 {
26 Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read );
27 ms.write("world!");
28 }
29 {
30 Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read );
31 ms.write("world, again!");
32 }
33 {
34 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write );
35 ms.write("[longer text shouldn't be seen]");
36 }
37 {
38 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Write|Bu::MyriadFs::Create|Bu::MyriadFs::Truncate );
39 ms.write("[short text]");
40 }
41
42 sio << "Reading files..." << sio.nl;
43 {
44 Bu::MyriadStream ms = mfs.open("/hello", Bu::MyriadFs::Read );
45 char buf[512];
46 buf[ms.read( buf, 512 )] = '\0';
47 sio << "read: '" << buf << "'" << sio.nl;
48 }
49 {
50 Bu::MyriadStream ms = mfs.open("/etc/hello", Bu::MyriadFs::Read );
51 char buf[512];
52 buf[ms.read( buf, 512 )] = '\0';
53 sio << "read: '" << buf << "'" << sio.nl;
54 }
55 {
56 Bu::MyriadStream ms = mfs.open("/etc/trunc", Bu::MyriadFs::Read );
57 char buf[512];
58 buf[ms.read( buf, 512 )] = '\0';
59 sio << "read: '" << buf << "'" << sio.nl;
60 }
61}
62
diff --git a/src/tools/myriad.cpp b/src/tools/myriad.cpp
index 1dc73ec..0dd9840 100644
--- a/src/tools/myriad.cpp
+++ b/src/tools/myriad.cpp
@@ -20,6 +20,7 @@ enum Mode
20 modeCreate, 20 modeCreate,
21 modeInfo, 21 modeInfo,
22 modeStreamNew, 22 modeStreamNew,
23 modeStreamErase,
23 modeStreamDump, 24 modeStreamDump,
24 modeStreamPut, 25 modeStreamPut,
25 modeStreamGet, 26 modeStreamGet,
@@ -44,6 +45,8 @@ public:
44 "Display some info about a Myriad file." ); 45 "Display some info about a Myriad file." );
45 addOption( eMode, 'n', "new", 46 addOption( eMode, 'n', "new",
46 "Create a new sub-stream in a Myriad file."); 47 "Create a new sub-stream in a Myriad file.");
48 addOption( eMode, 'e', "erase",
49 "Erase sub-stream in a Myriad file.");
47 addOption( eMode, 'd', "dump", 50 addOption( eMode, 'd', "dump",
48 "Display a hexdump of a stream from a Myriad file."); 51 "Display a hexdump of a stream from a Myriad file.");
49 addOption( eMode, "get", 52 addOption( eMode, "get",
@@ -67,6 +70,7 @@ public:
67 setOverride( "create", modeCreate ); 70 setOverride( "create", modeCreate );
68 setOverride( "info", modeInfo ); 71 setOverride( "info", modeInfo );
69 setOverride( "new", modeStreamNew ); 72 setOverride( "new", modeStreamNew );
73 setOverride( "erase", modeStreamErase );
70 setOverride( "dump", modeStreamDump ); 74 setOverride( "dump", modeStreamDump );
71 setOverride( "put", modeStreamPut ); 75 setOverride( "put", modeStreamPut );
72 setOverride( "get", modeStreamGet ); 76 setOverride( "get", modeStreamGet );
@@ -90,6 +94,49 @@ Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
90 return f; 94 return f;
91} 95}
92 96
97void printMap( const Bu::BitString &bs )
98{
99 for( int j = 0; j < bs.getSize(); j++ )
100 {
101 if( j>0 && (j%50) == 0 )
102 Bu::println("");
103 if( bs.getBit( j ) )
104 Bu::print("#");
105 else
106 Bu::print("-");
107 }
108 Bu::println("\n");
109}
110
111void printMap( const Bu::Array<int32_t> &bm )
112{
113 int iBigest = 0;
114 for( int j = 0; j < bm.getSize(); j++ )
115 {
116 if( iBigest < bm[j] )
117 iBigest = bm[j];
118 }
119 int iWidth = Bu::String("%1").arg( iBigest ).end().getSize();
120 Bu::String sEmpty;
121 for( int j = 0; j < iWidth; j++ )
122 {
123 sEmpty += '-';
124 }
125 int iBreakAt = 60/(iWidth+1);
126 for( int j = 0; j < bm.getSize(); j++ )
127 {
128 if( j>0 && (j%iBreakAt) == 0 )
129 Bu::println("");
130
131 if( bm[j] < 0 )
132 Bu::print("%1 ").arg( sEmpty, Bu::Fmt(2).right().fill(' '));
133 else
134 Bu::print("%1 ").arg( bm[j], Bu::Fmt(2).right().fill(' '));
135
136 }
137 Bu::println("\n");
138}
139
93int main( int argc, char *argv[] ) 140int main( int argc, char *argv[] )
94{ 141{
95 Options opts( argc, argv ); 142 Options opts( argc, argv );
@@ -153,6 +200,22 @@ int main( int argc, char *argv[] )
153 } 200 }
154 break; 201 break;
155 202
203 case modeStreamErase:
204 if( !opts.sFile.isSet() )
205 {
206 sio << "Please specify a file manipulate." << sio.nl;
207 return 0;
208 }
209 else
210 {
211 File fOut( opts.sFile, File::Write|File::Read );
212 Myriad m( fOut );
213 m.erase( opts.iStream );
214 printMap( m.buildBlockUseMap() );
215 printMap( m.buildBlockMap() );
216 }
217 break;
218
156 case modeStreamDump: 219 case modeStreamDump:
157 if( !opts.sFile.isSet() ) 220 if( !opts.sFile.isSet() )
158 { 221 {
@@ -255,17 +318,8 @@ int main( int argc, char *argv[] )
255 { 318 {
256 File fIn( opts.sFile, File::Write|File::Read ); 319 File fIn( opts.sFile, File::Write|File::Read );
257 Myriad m( fIn ); 320 Myriad m( fIn );
258/* Bu::BitString bs = m.getBlocksUsed(); 321 printMap( m.buildBlockUseMap() );
259 for( int j = 0; j < bs.getSize(); j++ ) 322 printMap( m.buildBlockMap() );
260 {
261 if( j>0 && (j%50) == 0 )
262 Bu::println("");
263 if( bs.getBit( j ) )
264 Bu::print("#");
265 else
266 Bu::print("-");
267 }*/
268 Bu::println("\n");
269 } 323 }
270 break; 324 break;
271 325
@@ -277,4 +331,3 @@ int main( int argc, char *argv[] )
277 331
278 return 0; 332 return 0;
279} 333}
280
diff --git a/src/tools/myriadfs.cpp b/src/tools/myriadfs.cpp
new file mode 100644
index 0000000..587bc89
--- /dev/null
+++ b/src/tools/myriadfs.cpp
@@ -0,0 +1,269 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/sio.h"
9#include "bu/streamstack.h"
10#include "bu/file.h"
11#include "bu/myriadfs.h"
12#include "bu/myriadstream.h"
13#include "bu/optparser.h"
14
15enum Mode
16{
17 modeList,
18 modeCat,
19 modeCopyIn,
20 modeCopyOut,
21 modeMkdir,
22 modeInitialize,
23 modeRm,
24
25 modeNone
26};
27
28Bu::Formatter &operator>>( Bu::Formatter &f, Mode & /*e*/ )
29{
30 Bu::sio << "Uh oh, the formatter was called..." << Bu::sio.nl;
31 return f;
32}
33
34class Options : public Bu::OptParser
35{
36public:
37 Options( int argc, char *argv[] ) :
38 eMode( modeNone ),
39 iBlockSize( 64 )
40 {
41 addHelpBanner("Options:");
42 addOption( sFile, 'f', "file", "Myriadfs file");
43 addOption( iBlockSize, 'b', "block-size",
44 "Specify the block size when initializing a new MyriadFs");
45
46 setNonOption( Bu::slot( this, &Options::nonOption ) );
47
48 addHelpOption();
49
50 parse( argc, argv );
51 }
52
53 int nonOption( Bu::Array<Bu::String> aParams )
54 {
55 if( eMode == modeNone )
56 {
57 //Bu::println("Checking mode");
58 // First param, must be the mode
59 if( aParams[0] == "ls" )
60 eMode = modeList;
61 else if( aParams[0] == "cat" )
62 eMode = modeCat;
63 else if( aParams[0] == "cp-in" )
64 eMode = modeCopyIn;
65 else if( aParams[0] == "cp-out" )
66 eMode = modeCopyOut;
67 else if( aParams[0] == "mkdir" )
68 eMode = modeMkdir;
69 else if( aParams[0] == "initialize" )
70 eMode = modeInitialize;
71 else if( aParams[0] == "rm" )
72 eMode = modeRm;
73 else
74 Bu::println("Unknown command, try --help");
75 return 0;
76 } else {
77 lParams.append( aParams[0] );
78 }
79 //Bu::println("Got option: \"%1\"").arg( aParams[0] );
80 return 0;
81 }
82
83 Mode eMode;
84 Bu::String sFile;
85 Bu::StringList lParams;
86 int iBlockSize;
87};
88
89int main( int argc, char *argv[] )
90{
91 Options opt( argc, argv );
92
93 if( opt.sFile.isEmpty() )
94 {
95 Bu::println("You must specify a MyriadFs stream (see -f).\n");
96 return 1;
97 }
98
99 if( opt.eMode == modeNone )
100 {
101 Bu::println("Specify an operation to perfrom.\n");
102 return 1;
103 }
104
105 int iFlags = Bu::File::ReadWrite;
106
107 if( opt.eMode == modeInitialize )
108 {
109 iFlags |= Bu::File::Create|Bu::File::Truncate;
110 }
111
112 Bu::File fFs( opt.sFile, iFlags );
113 Bu::MyriadFs mFs( fFs, opt.iBlockSize );
114
115 switch( opt.eMode )
116 {
117 case modeList:
118 {
119 Bu::String sPath = "/";
120 if( opt.lParams.getSize() > 0 )
121 {
122 sPath = opt.lParams.first();
123 }
124 Bu::MyriadFs::Dir lEnt = mFs.readDir( sPath );
125 for( Bu::MyriadFs::Dir::iterator i = lEnt.begin(); i; i++ )
126 {
127 Bu::String sPerm;
128 switch( (*i).uPerms&Bu::MyriadFs::typeMask )
129 {
130 case Bu::MyriadFs::typeDir: sPerm += "d"; break;
131 case Bu::MyriadFs::typeChrDev: sPerm += "c"; break;
132 case Bu::MyriadFs::typeBlkDev: sPerm += "b"; break;
133 case Bu::MyriadFs::typeSymLink: sPerm += "l"; break;
134 case Bu::MyriadFs::typeSocket: sPerm += "s"; break;
135 default: sPerm += "-"; break;
136 }
137 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrR)?"r":"-";
138 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrW)?"w":"-";
139 sPerm += ((*i).uPerms&Bu::MyriadFs::permUsrX)?"x":"-";
140 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpR)?"r":"-";
141 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpW)?"w":"-";
142 sPerm += ((*i).uPerms&Bu::MyriadFs::permGrpX)?"x":"-";
143 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthR)?"r":"-";
144 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthW)?"w":"-";
145 sPerm += ((*i).uPerms&Bu::MyriadFs::permOthX)?"x":"-";
146 Bu::println("%1 %2 %3:%4 %5 %6")
147 .arg( sPerm )
148 .arg( (*i).iNode )
149 .arg( (*i).iUser )
150 .arg( (*i).iGroup )
151 .arg( (*i).iSize )
152 .arg( (*i).sName );
153 }
154 }
155 break;
156
157 case modeCat:
158 {
159 if( opt.lParams.isEmpty() )
160 {
161 Bu::println("Specify at least one file.");
162 return 1;
163 }
164 int iBlockSize = 1024*1024;
165 char *pBuf = new char[iBlockSize];
166 for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ )
167 {
168 Bu::MyriadStream ms = mFs.open( *i, Bu::MyriadFs::Read );
169 int iRead = 0;
170 do
171 {
172 iRead = ms.read( pBuf, iBlockSize );
173 if( iRead > 0 )
174 {
175 Bu::sioRaw.write( pBuf, iRead );
176 }
177 } while( iRead == iBlockSize );
178 }
179 delete[] pBuf;
180 }
181 break;
182
183 case modeCopyIn:
184 {
185 if( opt.lParams.getSize() != 2 )
186 {
187 Bu::println("Specify a source file and destination file.");
188 return 1;
189 }
190 int iBlockSize = 1024*1024;
191 char *pBuf = new char[iBlockSize];
192 Bu::File fs = Bu::File(
193 opt.lParams.first(), Bu::File::Read );
194 Bu::MyriadStream ms = mFs.open(
195 opt.lParams.last(), Bu::MyriadFs::WriteNew );
196 int iRead = 0;
197 do
198 {
199 iRead = fs.read( pBuf, iBlockSize );
200 if( iRead > 0 )
201 {
202 ms.write( pBuf, iRead );
203 }
204 } while( iRead == iBlockSize );
205 delete[] pBuf;
206 }
207 break;
208
209 case modeCopyOut:
210 {
211 if( opt.lParams.getSize() != 2 )
212 {
213 Bu::println("Specify a source file and destination file.");
214 return 1;
215 }
216 int iBlockSize = 1024*1024;
217 char *pBuf = new char[iBlockSize];
218 Bu::MyriadStream ms = mFs.open(
219 opt.lParams.first(), Bu::MyriadFs::Read );
220 Bu::File fs = Bu::File(
221 opt.lParams.last(), Bu::File::WriteNew );
222 int iRead = 0;
223 do
224 {
225 iRead = ms.read( pBuf, iBlockSize );
226 if( iRead > 0 )
227 {
228 fs.write( pBuf, iRead );
229 }
230 } while( iRead == iBlockSize );
231 delete[] pBuf;
232 }
233 break;
234
235 case modeMkdir:
236 {
237 if( opt.lParams.isEmpty() )
238 {
239 Bu::println("Specify at least one directory.");
240 return 1;
241 }
242 for( Bu::StringList::iterator i = opt.lParams.begin(); i; i++ )
243 {
244 mFs.mkDir( *i, 0777 );
245 }
246 }
247 break;
248
249 case modeInitialize:
250 Bu::println("MyriadFs initialized.\n");
251 break;
252
253 case modeRm:
254 {
255 if( opt.lParams.getSize() != 1 )
256 {
257 Bu::println("Specify a file path.");
258 return 1;
259 }
260 mFs.unlink( opt.lParams.first() );
261 }
262 break;
263
264 case modeNone:
265 break;
266 }
267
268 return 0;
269}
diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit
new file mode 100644
index 0000000..f7bea97
--- /dev/null
+++ b/src/unit/myriad.unit
@@ -0,0 +1,385 @@
1// vim: syntax=cpp
2/*
3 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
4 *
5 * This file is part of the libbu++ library and is released under the
6 * terms of the license contained in the file LICENSE.
7 */
8
9#include "bu/string.h"
10#include "bu/file.h"
11#include "bu/myriad.h"
12#include "bu/myriadstream.h"
13#include "bu/array.h"
14
15#include "bu/sio.h"
16#include "bu/archive.h"
17#include "bu/md5.h"
18#include "bu/unitsuite.h"
19
20#include <stdlib.h>
21
22using namespace Bu;
23
24class VerifyObject
25{
26friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo );
27friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo );
28public:
29 VerifyObject( int iUnits ) :
30 iUnits( iUnits ),
31 iBytesWritten( 0 )
32 {
33 }
34
35 virtual ~VerifyObject()
36 {
37 }
38
39 int getBytesWritten()
40 {
41 return iBytesWritten;
42 }
43
44private:
45 int iUnits;
46 mutable int iBytesWritten;
47};
48
49Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo )
50{
51 Md5 sum;
52 ar << vo.iUnits;
53 vo.iBytesWritten = sizeof(int);
54 sum.addData( &vo.iUnits, sizeof(int) );
55 for( int j = 0; j < vo.iUnits; j++ )
56 {
57 int iRand = random()%128;
58// ar << iRand;
59 Bu::String sDat( iRand );
60 for( int j = 0; j < iRand; j++ )
61 sDat[j] = (char)((uint8_t)(random()%256));
62 ar << sDat;
63 sum.addData( &iRand, sizeof(int) );
64 sum.addData( sDat.getStr(), iRand );
65 vo.iBytesWritten += sizeof(long) + iRand;
66 }
67 Bu::String sRes = sum.getResult();
68 ar << sRes;
69 vo.iBytesWritten += sizeof(long) + sRes.getSize();
70 return ar;
71}
72
73Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo )
74{
75 Md5 sum;
76 ar >> vo.iUnits;
77 sum.addData( &vo.iUnits, sizeof(int) );
78 for( int j = 0; j < vo.iUnits; j++ )
79 {
80 int iRand;
81// ar >> iRand;
82 Bu::String sStr;
83 ar >> sStr;
84 iRand = sStr.getSize();
85 sum.addData( &iRand, sizeof(int) );
86 sum.addData( sStr.getStr(), iRand );
87 }
88 Bu::String sSum;
89 ar >> sSum;
90 unitTest( sSum == sum.getResult() );
91 int iTooMuch;
92 try
93 {
94 ar >> iTooMuch;
95 unitFailed("should have thrown an exception.");
96 }
97 catch( Bu::ExceptionBase &e )
98 {
99 }
100 return ar;
101}
102
103suite Myriad
104{
105 test setSize
106 {
107 String sFileName("myriad-XXXXXXX");
108
109 File fMyriad = tempFile( sFileName );
110 Myriad m( fMyriad, 32 );
111
112 MyriadStream ms = m.create( Myriad::ReadWrite );
113 ms.setSize( 150 );
114 ms.setPos( 145 );
115 char stuff[10];
116 unitTest( ms.read( stuff, 10 ) == 5 );
117
118 ms.setSize( 12 );
119 unitTest( ms.read( stuff, 10 ) == 0 );
120 unitTest( ms.write( "hello", 5 ) == 5 );
121 unitTest( ms.tell() == 17 );
122
123 ms.setSize( 500 );
124 unitTest( ms.tell() == 17 );
125 }
126
127 void addBlock( Stream &s, bool bAppend=true )
128 {
129 if( bAppend )
130 s.setPosEnd( 0 );
131 int iSize = (random()%1016)+8;
132 s.write( &iSize, 4 );
133 char *buf = new char[iSize-8];
134 for( int j = 0; j < iSize-8; j++ )
135 {
136 buf[j] = (j+iSize)%256;
137 }
138 if( random()%2 == 0 )
139 {
140 s.write( buf, iSize-8 );
141 }
142 else
143 {
144 for( int j = 0; j < iSize-8; )
145 {
146 int iAmnt = (random()%8)+1;
147 if( iAmnt+j > iSize-8 )
148 iAmnt = iSize-8-j;
149 iAmnt = s.write( buf+j, iAmnt );
150 j += iAmnt;
151 }
152 }
153 delete[] buf;
154 iSize = ~iSize;
155 s.write( &iSize, 4 );
156 }
157
158 void verifyBlock( Stream &s )
159 {
160 int iSize, iInv;
161 if( s.read( &iSize, 4 ) == 0 )
162 return;
163 if( iSize < 8 || iSize > 1024 )
164 throw ExceptionBase("Read bad data, %d", iSize );
165 char *buf = new char[iSize-8];
166 if( s.read( buf, iSize-8 ) < (Bu::size)iSize-8 )
167 {
168 delete[] buf;
169 throw ExceptionBase("Block failed verify (insuffient block data).");
170 }
171 for( int j = 0; j < iSize-8; j++ )
172 {
173 if( buf[j] != (char)((j+iSize)%256) )
174 {
175 char b = buf[j];
176 delete[] buf;
177 throw ExceptionBase("Block failed computed data verify "
178 "(%02X==%02X).", b, (char)((j+iSize)%256) );
179 }
180 }
181 delete[] buf;
182 if( s.read( &iInv, 4 ) < 4 )
183 throw ExceptionBase("Block failed verify (insufficient data).");
184 if( iInv != ~iSize )
185 throw ExceptionBase("Block failed inversion verify.");
186 }
187
188 void verifyStream( Stream &s )
189 {
190 s.setPos( 0 );
191 while( !s.isEos() )
192 verifyBlock( s );
193 }
194
195 test stressGrow
196 {
197 String sFileName("myriad-XXXXXXX");
198
199 File fMyriad = tempFile( sFileName );
200 Myriad m( fMyriad, 64 );
201
202 Array<int> aStreams;
203 for( int j = 0; j < 5; j++ )
204 {
205 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
206 }
207
208 srandom( 512 );
209
210 for( int j = 0; j < 2500; j++ )
211 {
212 switch( random()%5 )
213 {
214 case 0:
215 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
216 break;
217
218 case 1:
219 if( aStreams.getSize() > 0 )
220 {
221 int iStream = random()%aStreams.getSize();
222 {
223 MyriadStream ms = m.open( aStreams[iStream], Myriad::Read );
224 verifyStream( ms );
225 }
226 m.erase( aStreams[iStream] );
227 Array<int>::iterator i = aStreams.begin();
228 for( int k = 0; k < iStream; k++ )
229 i++;
230 aStreams.erase( i );
231 }
232 break;
233
234 default:
235 if( aStreams.getSize() == 0 )
236 {
237 aStreams.append(
238 m.create( Bu::Myriad::Read ).getId()
239 );
240 }
241 {
242 int iStream = random()%aStreams.getSize();
243 MyriadStream ms = m.open( aStreams[iStream], Myriad::ReadWrite );
244 addBlock( ms );
245 verifyStream( ms );
246 }
247 break;
248 }
249 }
250
251 for( Array<int>::iterator i = aStreams.begin(); i; i++ )
252 {
253 MyriadStream ms = m.open( *i, Myriad::Read );
254 verifyStream( ms );
255 }
256 }
257
258 test stressTruncate
259 {
260 String sFileName("myriad-XXXXXXX");
261
262 File fMyriad = tempFile( sFileName );
263 Myriad m( fMyriad, 128 );
264
265 Array<int> aStream;
266
267 for( int j = 0; j < 5; j++ )
268 {
269 aStream.append( m.create( Bu::Myriad::Read ).getId() );
270 }
271
272 srandom( 1024 );
273
274 char b;
275 for( int iter = 0; iter < 2500; iter++ )
276 {
277 for( Array<int>::iterator i = aStream.begin(); i; i++ )
278 {
279 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
280 addBlock( ms, false );
281 ms.setSize( ms.tell() );
282 unitTest( ms.read( &b, 1 ) == 0 );
283 ms.setPos( 0 );
284 verifyBlock( ms );
285 unitTest( ms.read( &b, 1 ) == 0 );
286 }
287 }
288 }
289
290 test stressTruncate2
291 {
292 String sFileName("myriad-XXXXXXX");
293
294 Array<int> aStream;
295
296 setStepCount( 5*2500 + 5 );
297
298 {
299 File fMyriad = tempFile( sFileName );
300 Myriad m( fMyriad, 128 );
301
302 for( int j = 0; j < 5; j++ )
303 {
304 aStream.append( m.create( Bu::Myriad::Read ).getId() );
305 incProgress();
306 }
307 }
308
309 srandom( 1024 );
310
311 char b;
312 for( int iter = 0; iter < 2500; iter++ )
313 {
314 File fMyriad( sFileName, File::ReadWrite );
315 Myriad m( fMyriad );
316 for( Array<int>::iterator i = aStream.begin(); i; i++ )
317 {
318 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
319 addBlock( ms, false );
320 ms.setSize( ms.tell() );
321 unitTest( ms.read( &b, 1 ) == 0 );
322 ms.setPos( 0 );
323 verifyBlock( ms );
324 unitTest( ms.read( &b, 1 ) == 0 );
325 incProgress();
326 }
327 }
328 }
329
330 test stressArchive
331 {
332 String sFileName("myriad-XXXXXX");
333 Array<int> aStream;
334
335 srandom( 2096 );
336
337 setStepCount( 15*250 + 15 );
338
339 {
340 File fMyriad = tempFile( sFileName );
341 Myriad m( fMyriad, 1024 );
342
343 for( int j = 0; j < 15; j++ )
344 {
345 MyriadStream ms = m.create( Myriad::Write );
346 int iStream = ms.getId();
347 aStream.append( iStream );
348 VerifyObject vo( random()%1024 );
349 {
350 Archive ar( ms, Archive::save );
351 ar << vo;
352 unitTest( ms.tell() == vo.getBytesWritten() );
353 ms.setSize( ms.tell() );
354 }
355 unitTest( m.getSize( iStream ) == vo.getBytesWritten() );
356 incProgress();
357 }
358 }
359
360 for( int iter = 0; iter < 250; iter++ )
361 {
362 File fMyriad( sFileName, File::ReadWrite );
363 Myriad m( fMyriad );
364 for( Array<int>::iterator i = aStream.begin(); i; i++ )
365 {
366 VerifyObject vo( random()%1024 );
367 {
368 MyriadStream ms = m.open( *i, Myriad::Read );
369 Archive ar( ms, Archive::load );
370 ar >> vo;
371 }
372 {
373 MyriadStream ms = m.open( *i, Myriad::WriteNew );
374 Archive ar( ms, Archive::save );
375 ar << vo;
376 unitTest( ms.tell() == vo.getBytesWritten() );
377 ms.setSize( ms.tell() );
378 }
379 unitTest( m.getSize( *i ) == vo.getBytesWritten() );
380 incProgress();
381 }
382 }
383 }
384}
385
diff --git a/src/unstable/bitstring.cpp b/src/unstable/bitstring.cpp
index 21c1316..b80c073 100644
--- a/src/unstable/bitstring.cpp
+++ b/src/unstable/bitstring.cpp
@@ -209,7 +209,7 @@ void Bu::BitString::flipBit( long iBit )
209 caData[iBit/8] ^= (1<<(iBit%8)); 209 caData[iBit/8] ^= (1<<(iBit%8));
210} 210}
211 211
212bool Bu::BitString::getBit( long iBit ) 212bool Bu::BitString::getBit( long iBit ) const
213{ 213{
214 if( iBit >= iBits || iBit < 0 ) return false; 214 if( iBit >= iBits || iBit < 0 ) return false;
215 if( (caData[iBit/8] & (1<<(iBit%8))) == 0 ) 215 if( (caData[iBit/8] & (1<<(iBit%8))) == 0 )
@@ -224,7 +224,7 @@ long Bu::BitString::getBitLength()
224 return iBits; 224 return iBits;
225} 225}
226 226
227long Bu::BitString::getSize() 227long Bu::BitString::getSize() const
228{ 228{
229 return iBits; 229 return iBits;
230} 230}
@@ -311,6 +311,14 @@ void Bu::BitString::clear()
311 } 311 }
312} 312}
313 313
314void Bu::BitString::fill()
315{
316 if( caData != NULL )
317 {
318 memset( caData, 0xff, iBytes );
319 }
320}
321
314bool Bu::BitString::setBitLength( long iLength, bool bClear ) 322bool Bu::BitString::setBitLength( long iLength, bool bClear )
315{ 323{
316 return setSize( iLength, bClear ); 324 return setSize( iLength, bClear );
diff --git a/src/unstable/bitstring.h b/src/unstable/bitstring.h
index afc22fb..70ba822 100644
--- a/src/unstable/bitstring.h
+++ b/src/unstable/bitstring.h
@@ -88,7 +88,7 @@ namespace Bu
88 *@param iBit The index of the bit to test. 88 *@param iBit The index of the bit to test.
89 *@returns True for a 1, false for a 0. 89 *@returns True for a 1, false for a 0.
90 */ 90 */
91 bool getBit( long iBit ); 91 bool getBit( long iBit ) const;
92 92
93 /** 93 /**
94 * Inverts the entire BitString, in effect this calls flipBit on every 94 * Inverts the entire BitString, in effect this calls flipBit on every
@@ -106,7 +106,7 @@ namespace Bu
106 DEPRECATED 106 DEPRECATED
107 long getBitLength(); 107 long getBitLength();
108 108
109 long getSize(); 109 long getSize() const;
110 110
111 /** 111 /**
112 * Sets the entire BitString to zeros, but it does it very quickly. 112 * Sets the entire BitString to zeros, but it does it very quickly.
@@ -115,6 +115,12 @@ namespace Bu
115 void clear(); 115 void clear();
116 116
117 /** 117 /**
118 * Sets the entire BitString to ones, but it does it very quickly.
119 * This operation runs in O(N).
120 */
121 void fill();
122
123 /**
118 * Gets another BitString that is autonomous of the current one 124 * Gets another BitString that is autonomous of the current one
119 * (contains a copy of the memory, not a pointer) and contains a subset 125 * (contains a copy of the memory, not a pointer) and contains a subset
120 * of the data in the current BitString. This is an inclusive 126 * of the data in the current BitString. This is an inclusive
diff --git a/src/unstable/myriadcache.cpp b/src/unstable/myriadcache.cpp
new file mode 100644
index 0000000..c9eb9c4
--- /dev/null
+++ b/src/unstable/myriadcache.cpp
@@ -0,0 +1,8 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriadcache.h"
diff --git a/src/unstable/myriadcache.h b/src/unstable/myriadcache.h
new file mode 100644
index 0000000..f71f9b5
--- /dev/null
+++ b/src/unstable/myriadcache.h
@@ -0,0 +1,150 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_CACHE_H
9#define BU_MYRIAD_CACHE_H
10
11#include "bu/cachebase.h"
12#include "bu/myriad.h"
13#include "bu/myriadstream.h"
14#include "bu/file.h"
15#include "bu/streamstack.h"
16
17namespace Bu
18{
19 template<typename keytype, typename obtype>
20 class MyriadCache : public Bu::CacheBase<keytype, obtype>
21 {
22 public:
23 MyriadCache( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 ) :
24 sStore( sStore ),
25 mStore( sStore, iBlockSize, iPreallocate ),
26 bStructureChanged( false )
27 {
28 try
29 {
30 Bu::ReadWriteMutex::ReadLocker l( rwStore );
31 Bu::MyriadStream ms = mStore.open(
32 1, Bu::Myriad::WriteNew|Bu::Myriad::Read
33 );
34 Bu::Archive ar( ms, Bu::Archive::load );
35 uint8_t uVer;
36 ar >> uVer;
37 switch( uVer )
38 {
39 case 0:
40 ar >> hIndex;
41 break;
42 }
43 }
44 catch(...)
45 {
46 try
47 {
48 mStore.open( 1, Bu::Myriad::Create|Bu::Myriad::ReadWrite );
49 _sync();
50 }
51 catch(...)
52 {
53 throw Bu::ExceptionBase("Error creating index stream.");
54 }
55 }
56 }
57
58 virtual ~MyriadCache()
59 {
60 Bu::CacheBase<keytype,obtype>::sync();
61 }
62
63 using typename Bu::CacheBase<keytype,obtype>::KeyList;
64 using typename Bu::CacheBase<keytype,obtype>::ObjectType;
65
66 virtual typename Bu::CacheBase<keytype,obtype>::KeyList getKeys() const
67 {
68 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
69 return hIndex.getKeys();
70 }
71
72 virtual int getSize() const
73 {
74 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
75 return hIndex.getSize();
76 }
77
78 protected:
79 virtual bool _has( const keytype &key )
80 {
81 Bu::ReadWriteMutex::ReadLocker rl( rwStore );
82 return hIndex.has( key );
83 }
84
85 virtual void _create( const obtype *o )
86 {
87 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
88 {
89 Bu::Myriad::StreamId id = mStore.allocate();
90 hIndex.insert( o->getKey(), id );
91 }
92 _save( o );
93
94 bStructureChanged = true;
95 }
96
97 virtual void _erase( const keytype &k )
98 {
99 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
100 mStore.erase( hIndex.get( k ) );
101 hIndex.erase( k );
102
103 bStructureChanged = true;
104 }
105
106 virtual obtype *_load(
107 typename Bu::CacheObject<keytype, obtype>::Initializer &initObj,
108 const keytype &k
109 )
110 {
111 Bu::MyriadStream ms = mStore.openStream( hIndex.get( k ) );
112 return _cacheObjectLoad<keytype, obtype>( initObj, k, ms );
113 }
114
115 virtual void _save( const obtype *o )
116 {
117 Bu::MyriadStream ms = mStore.openStream( hIndex.get( o->getKey() ) );
118 _cacheObjectSave( ms, o );
119 ms.setSize( ms.tell() );
120
121 mStore.sync();
122 }
123
124 virtual void _sync()
125 {
126 Bu::ReadWriteMutex::WriteLocker wl( rwStore );
127 if( !bStructureChanged )
128 return;
129
130 Bu::MyriadStream ms = mStore.openStream( 1 );
131 Bu::Archive ar( ms, Bu::Archive::save );
132 ar << (uint8_t)0 << hIndex;
133 ar.close();
134 ms.setSize( ms.tell() );
135
136 bStructureChanged = false;
137
138 mStore.sync();
139 }
140
141 private:
142 Bu::Stream &sStore;
143 Bu::Myriad mStore;
144 Bu::Hash<keytype, int> hIndex;
145 mutable Bu::ReadWriteMutex rwStore;
146 bool bStructureChanged;
147 };
148}
149
150#endif
diff --git a/src/unstable/myriadfs.cpp b/src/unstable/myriadfs.cpp
new file mode 100644
index 0000000..f748a53
--- /dev/null
+++ b/src/unstable/myriadfs.cpp
@@ -0,0 +1,739 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/config.h"
9#include "bu/myriadfs.h"
10#include "bu/myriadstream.h"
11#include "bu/mutexlocker.h"
12
13#include <string.h>
14#include <unistd.h>
15#include <time.h>
16
17#include "bu/sio.h"
18using Bu::sio;
19using Bu::Fmt;
20
21namespace Bu { subExceptionDef( MyriadFsException ) }
22
23#define Myriad_Fs_MAGIC_CODE ((char *)"\xa7\x18\x8b\x39")
24
25Bu::MyriadFs::MyriadFs( Bu::Stream &rStore, int iBlockSize ) :
26 rStore( rStore ),
27 mStore( rStore, iBlockSize ),
28 iUser( 0 ),
29 iGroup( 0 )
30{
31#ifndef WIN32
32 iUser = getuid();
33 iGroup = getgid();
34#endif
35
36 if( mStore.exists( 1 ) )
37 {
38 // Check to see if this is a MyriadFs stream.
39 Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Read );
40 char sMagic[4];
41 if( ms.read( sMagic, 4 ) < 4 )
42 throw MyriadFsException("The provided stream does not appear to be "
43 "a MyriadFs stream.");
44 if( ::strncmp( sMagic, Myriad_Fs_MAGIC_CODE, 4 ) )
45 throw MyriadFsException("The provided stream does not appear to be "
46 "a MyriadFs stream.");
47
48 int8_t iVer;
49 ms.read( &iVer, 1 );
50
51 int32_t iNumNodes;
52 ms.read( &iNumNodes, 4 );
53 for( int32_t j = 0; j < iNumNodes; j++ )
54 {
55 int32_t iNode;
56 int32_t iPos;
57 ms.read( &iNode, 4 );
58 ms.read( &iPos, 4 );
59 hNodeIndex.insert( iNode, iPos );
60 }
61 }
62 else
63 {
64 // Create initial header stream
65 {
66 Bu::MyriadStream ms = mStore.open(
67 1, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
68 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
69 int8_t iVer = 1;
70 int32_t iTmp = 1;
71 ms.write( &iVer, 1 );
72 ms.write( &iTmp, 4 ); // iNumNodes
73 iTmp = 0;
74 ms.write( &iTmp, 4 ); // iInode
75 ms.write( &iTmp, 4 ); // iPosition
76 hNodeIndex.insert( 0, 0 );
77 }
78
79 // Create initial inode stream, with one root node.
80 {
81 Bu::MyriadStream ms = mStore.open(
82 2, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
83 RawStat rs;
84 rs.iNode = 0;
85 rs.iUser = iUser;
86 rs.iGroup = iGroup;
87 rs.uPerms = 0755|typeDir;
88 rs.iLinks = 1;
89 rs.uStreamIndex = 3;
90 rs.iCTime = rs.iMTime = rs.iATime = time(NULL);
91 ms.write( &rs, sizeof(RawStat) );
92 }
93
94 // Create inode 0's storage stream.
95 {
96 Bu::MyriadStream ms = mStore.open(
97 3, Bu::Myriad::WriteNew|Bu::Myriad::Exclusive );
98 int32_t iTmp32 = 0;
99 ms.write( &iTmp32, 4 ); // iChildCount
100 }
101 }
102}
103
104Bu::MyriadFs::~MyriadFs()
105{
106 writeHeader();
107}
108
109void Bu::MyriadFs::stat( const Bu::String &sPath, Bu::MyriadFs::Stat &rBuf )
110{
111 Bu::MutexLocker lLock( mAccess );
112 int32_t iParent;
113 int32_t iNode = lookupInode( sPath, iParent );
114 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read );
115 stat( iNode, rBuf, is );
116}
117
118Bu::MyriadStream Bu::MyriadFs::open( const Bu::String &sPath, int iMode,
119 uint16_t uPerms )
120{
121 Bu::MutexLocker lLock( mAccess );
122 int32_t iParent = -1;
123 int32_t iNode;
124 try
125 {
126 iNode = lookupInode( sPath, iParent );
127// sio << "File found." << sio.nl;
128 // The file was found
129 Bu::MyriadStream ms = openByInode( iNode );
130 if( (iMode&Truncate) )
131 {
132 ms.setSize( 0 );
133 }
134 return ms;
135 }
136 catch( Bu::MyriadFsException &e )
137 {
138 if( iParent < 0 )
139 throw;
140
141 // This means that an intermediate path component couldn't be found
142 if( e.getErrorCode() == 1 )
143 throw;
144
145 // The file wasn't found, but the path leading up to it was.
146 // first, figure out the final path element...
147 Bu::String sName = filePart( sPath );
148// sio << "End filename: " << sName << sio.nl;
149// sio << "Parent inode: " << iParent << sio.nl;
150 iNode = create( iParent, sName, (uPerms&permMask)|typeRegFile, 0 );
151// sio << "New iNode: " << iNode << sio.nl;
152 return openByInode( iNode );
153 }
154}
155
156void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms )
157{
158 create( sPath, iPerms, 0 );
159}
160
161void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
162 uint16_t iDevHi, uint16_t iDevLo )
163{
164 create( sPath, iPerms, ((uint32_t)iDevHi<<16)|(uint32_t)iDevLo );
165}
166
167void Bu::MyriadFs::create( const Bu::String &sPath, uint16_t iPerms,
168 uint32_t uSpecial )
169{
170 Bu::MutexLocker lLock( mAccess );
171 int32_t iParent = -1;
172// int32_t iNode;
173 try
174 {
175 /*iNode =*/ lookupInode( sPath, iParent );
176// sio << "File found." << sio.nl;
177 }
178 catch( Bu::MyriadFsException &e )
179 {
180 if( iParent < 0 )
181 throw;
182
183 // This means that an intermediate path component couldn't be found
184 if( e.getErrorCode() == 1 )
185 throw;
186
187 // The file wasn't found, but the path leading up to it was.
188 // first, figure out the final path element...
189 Bu::String sName = filePart( sPath );
190// sio << "End filename: " << sName << sio.nl;
191// sio << "Parent inode: " << iParent << sio.nl;
192 /*iNode =*/ create( iParent, sName, iPerms, uSpecial );
193// sio << "New iNode: " << iNode << sio.nl;
194 }
195 // The file was found
196 //throw Bu::MyriadFsException("Path already exists.");
197}
198
199void Bu::MyriadFs::mkDir( const Bu::String &sPath, uint16_t iPerms )
200{
201 create( sPath, (iPerms&permMask)|typeDir, 0 );
202}
203
204void Bu::MyriadFs::mkSymLink( const Bu::String &sTarget,
205 const Bu::String &sPath )
206{
207 Bu::MutexLocker lLock( mAccess );
208 int32_t iParent = -1;
209 int32_t iNode;
210 try
211 {
212 iNode = lookupInode( sPath, iParent );
213 }
214 catch( Bu::MyriadFsException &e )
215 {
216 if( iParent < 0 )
217 throw;
218
219 // This means that an intermediate path component couldn't be found
220 if( e.getErrorCode() == 1 )
221 throw;
222
223 // The file wasn't found, but the path leading up to it was.
224 // first, figure out the final path element...
225 Bu::String sName = filePart( sPath );
226// sio << "End filename: " << sName << sio.nl;
227// sio << "Parent inode: " << iParent << sio.nl;
228 iNode = create( iParent, sName, 0777|typeSymLink, 0 );
229// sio << "New iNode: " << iNode << sio.nl;
230 MyriadStream ms = openByInode( iNode );
231 ms.write( sTarget );
232 return;
233 }
234 throw Bu::MyriadFsException("Path already exists.");
235}
236
237void Bu::MyriadFs::mkHardLink( const Bu::String &sTarget,
238 const Bu::String &sPath )
239{
240 Bu::MutexLocker lLock( mAccess );
241 int32_t iParent = -1;
242 int32_t iNode;
243
244 iNode = lookupInode( sTarget, iParent );
245
246 try
247 {
248 lookupInode( sPath, iParent );
249 throw Bu::MyriadFsException("Path already exists.");
250 }
251 catch( Bu::MyriadFsException &e )
252 {
253 if( iParent < 0 )
254 throw;
255
256 // This means that an intermediate path component couldn't be found
257 if( e.getErrorCode() == 1 )
258 throw;
259
260 // The file wasn't found, but the path leading up to it was.
261 // first, figure out the final path element...
262 Bu::String sName = filePart( sPath );
263// sio << "End filename: " << sName << sio.nl;
264// sio << "Parent inode: " << iParent << sio.nl;
265 addToDir( iParent, iNode, sName );
266 MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
267 RawStat rs;
268 readInode( iNode, rs, is );
269 rs.iLinks++;
270 writeInode( rs, is );
271 }
272}
273
274Bu::String Bu::MyriadFs::readSymLink( const Bu::String &sPath )
275{
276 Bu::MutexLocker lLock( mAccess );
277 int32_t iParent = -1;
278 int32_t iNode;
279 iNode = lookupInode( sPath, iParent );
280 MyriadStream ms = openByInode( iNode );
281 Bu::String sRet;
282 sRet.setSize( ms.getSize() );
283 ms.read( sRet.getStr(), ms.getSize() );
284 return sRet;
285}
286
287Bu::MyriadFs::Dir Bu::MyriadFs::readDir( const Bu::String &sPath )
288{
289 Bu::MutexLocker lLock( mAccess );
290 int32_t iParent = -1;
291 int32_t iNode = lookupInode( sPath, iParent );
292 return readDir( iNode );
293}
294
295void Bu::MyriadFs::setTimes( const Bu::String &sPath, int64_t iATime,
296 int64_t iMTime )
297{
298 Bu::MutexLocker lLock( mAccess );
299 int32_t iParent = -1;
300 int32_t iNode;
301
302 iNode = lookupInode( sPath, iParent );
303
304 setTimes( iNode, iATime, iMTime );
305}
306
307void Bu::MyriadFs::unlink( const Bu::String &sPath )
308{
309 Bu::MutexLocker lLock( mAccess );
310 int32_t iParent = -1;
311// int32_t iNode;
312
313 /*iNode =*/ lookupInode( sPath, iParent );
314
315 Dir lDir = readDir( iParent );
316
317 Bu::String sName = filePart( sPath );
318
319 for( Dir::iterator i = lDir.begin(); i; i++ )
320 {
321 if( sName == (*i).sName )
322 {
323 RawStat rs;
324 readInode( (*i).iNode, rs );
325 if( (rs.uPerms&typeMask) == typeDir )
326 {
327 MyriadStream msDir = mStore.open(
328 rs.uStreamIndex, Bu::Myriad::Read
329 );
330 int32_t iCount;
331 msDir.read( &iCount, 4 );
332 if( iCount > 0 )
333 {
334 throw Bu::MyriadFsException("Directory not empty.");
335 }
336 }
337 if( --rs.iLinks == 0 )
338 {
339 destroyNode( (*i).iNode );
340 }
341 else
342 {
343 writeInode( rs );
344 }
345 lDir.erase( i );
346 break;
347 }
348 }
349
350 Bu::MyriadStream ms = openByInode( iParent );
351 int32_t iNumChildren = lDir.getSize();
352 ms.write( &iNumChildren, 4 );
353 for( Dir::iterator i = lDir.begin(); i; i++ )
354 {
355 ms.write( &(*i).iNode, 4 );
356 uint8_t iSize = (*i).sName.getSize();
357 ms.write( &iSize, 1 );
358 ms.write( (*i).sName.getStr(), iSize );
359 }
360 ms.setSize( ms.tell() );
361}
362
363void Bu::MyriadFs::setFileSize( const Bu::String &sPath, int32_t iSize )
364{
365 Bu::MutexLocker lLock( mAccess );
366 int32_t iParent = -1;
367 int32_t iNode;
368 iNode = lookupInode( sPath, iParent );
369 MyriadStream ms = openByInode( iNode );
370 ms.setSize( iSize );
371}
372
373void Bu::MyriadFs::rename( const Bu::String &sFrom, const Bu::String &sTo )
374{
375 Bu::MutexLocker lLock( mAccess );
376 mkHardLink( sFrom, sTo );
377 unlink( sFrom );
378}
379
380dev_t Bu::MyriadFs::devToSys( uint32_t uDev )
381{
382 return (((uDev&0xFFFF0000)>>8)&0xFF00) | ((uDev&0xFF));
383}
384
385uint32_t Bu::MyriadFs::sysToDev( dev_t uDev )
386{
387 return (((uint32_t)uDev&0xFF00)<<8) | ((uint32_t)uDev&0xFF);
388}
389
390int32_t Bu::MyriadFs::lookupInode( const Bu::String &sPath, int32_t &iParent )
391{
392 if( sPath == "/" )
393 {
394 return 0;
395 }
396 if( sPath[0] == '/' )
397 {
398 // Absolute lookup
399 return lookupInode( sPath.begin()+1, 0, iParent );
400 }
401 else
402 {
403 // Relative lookup
404 throw Bu::ExceptionBase(
405 "Relative lookups in MyriadFs are not working yet.");
406 }
407}
408
409int32_t Bu::MyriadFs::lookupInode( Bu::String::const_iterator iStart,
410 int32_t iNode, int32_t &iParent )
411{
412 iParent = iNode;
413
414 Bu::String::const_iterator iEnd = iStart.find('/');
415 Bu::String sTok( iStart, iEnd );
416
417// sio << "Direcotry component: " << sTok << sio.nl;
418
419 Dir lDir = readDir( iNode );
420
421 for( Dir::iterator i = lDir.begin(); i; i++ )
422 {
423 if( (*i).sName == sTok )
424 {
425 // We found an item
426 if( !iEnd )
427 {
428 // It's the last one in the requested path, return it
429 return (*i).iNode;
430 }
431 else
432 {
433 // Not the last one in our path, double check it's a dir
434 if( ((*i).uPerms&typeMask) == typeDir )
435 {
436 return lookupInode( iEnd+1, (*i).iNode, iParent );
437 }
438 else
439 {
440 iParent = -1;
441 throw Bu::MyriadFsException(
442 "Element '%s' in given path is not a directory.",
443 sTok.getStr() );
444 }
445 }
446 }
447 }
448
449 if( iEnd )
450 throw Bu::MyriadFsException( 1, "Path not found");
451 else
452 throw Bu::MyriadFsException( 2, "Path not found");
453}
454
455void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs )
456{
457 rIs.setPos( hNodeIndex.get( iNode )*sizeof(RawStat) );
458 if( rIs.read( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) )
459 throw Bu::MyriadFsException("Filesystem corruption detected.");
460 if( rs.iNode != iNode )
461 throw Bu::MyriadFsException("Filesystem corruption detected.");
462}
463
464void Bu::MyriadFs::readInode( int32_t iNode, RawStat &rs )
465{
466 MyriadStream ms = mStore.open( 2, Bu::Myriad::Read );
467 readInode( iNode, rs, ms );
468}
469
470void Bu::MyriadFs::writeInode( const RawStat &rs,
471 MyriadStream &rOs )
472{
473 rOs.setSize( hNodeIndex.getSize()*sizeof(RawStat) );
474 rOs.setPos( hNodeIndex.get( rs.iNode )*sizeof(RawStat) );
475 if( rOs.write( &rs, sizeof(RawStat) ) < (int)sizeof(RawStat) )
476 throw Bu::MyriadFsException("Error writing inode to header stream.");
477}
478
479void Bu::MyriadFs::writeInode( const RawStat &rs )
480{
481 MyriadStream ms = mStore.open( 2, Bu::Myriad::Write );
482 writeInode( rs, ms );
483}
484
485Bu::MyriadFs::Dir Bu::MyriadFs::readDir( int32_t iNode )
486{
487 Bu::MyriadStream ms = openByInode( iNode );
488 int32_t iNumChildren = 0;
489 ms.read( &iNumChildren, 4 );
490
491 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::Read );
492 Dir lDir;
493 // sio << "Reading dir " << iNode << ", " << iNumChildren << " entries:" << sio.nl;
494 for( int32_t j = 0; j < iNumChildren; j++ )
495 {
496 int32_t iChildNode = 0;
497 if( ms.read( &iChildNode, 4 ) < 4 )
498 {
499 throw Bu::MyriadFsException(
500 "Failed to read iChildNode from directory.");
501 }
502 Stat s;
503 stat( iChildNode, s, is );
504 uint8_t uLen;
505 if( ms.read( &uLen, 1 ) < 1 )
506 {
507 throw Bu::MyriadFsException(
508 "Failed to read uLen from directory.");
509 }
510 s.sName.setSize( uLen );
511 if( ms.read( s.sName.getStr(), uLen ) < uLen )
512 {
513 throw Bu::MyriadFsException(
514 "Failed to read sName from directory.");
515 }
516 lDir.append( s );
517
518// sio << " " << s.sName << sio.nl;
519 }
520
521 return lDir;
522}
523
524Bu::MyriadStream Bu::MyriadFs::openByInode( int32_t iNode )
525{
526 RawStat rs;
527 readInode( iNode, rs );
528 switch( (rs.uPerms&typeMask) )
529 {
530 case typeDir:
531 case typeSymLink:
532 case typeRegFile:
533 return mStore.open( rs.uStreamIndex, Bu::Myriad::ReadWrite );
534
535 default:
536 throw Bu::MyriadFsException(
537 "inode incorrect type for low-level openByInode.");
538 }
539}
540
541void Bu::MyriadFs::addToDir( int32_t iDir, int32_t iNode,
542 const Bu::String &sName )
543{
544 if( sName.getSize() > 255 )
545 {
546 throw Bu::MyriadFsException("Filename too long, max is 255 bytes.");
547 }
548 Bu::MyriadStream ms = openByInode( iDir );
549 int32_t iNumChildren = 0;
550 ms.read( &iNumChildren, 4 );
551 iNumChildren++;
552 ms.setPos( 0 );
553 ms.write( &iNumChildren, 4 );
554 ms.setPosEnd( 0 );
555 ms.write( &iNode, 4 );
556 uint8_t uLen = sName.getSize();
557 ms.write( &uLen, 1 );
558 ms.write( sName.getStr(), uLen );
559}
560
561int32_t Bu::MyriadFs::create( int32_t iParent, const Bu::String &sName,
562 uint16_t uPerms, uint32_t uSpecial )
563{
564 int32_t iNode = allocInode( uPerms, uSpecial );
565 addToDir( iParent, iNode, sName );
566 return iNode;
567}
568
569int32_t Bu::MyriadFs::allocInode( uint16_t uPerms, uint32_t uSpecial )
570{
571 int32_t iNode = 0;
572 for(; iNode < 0xfffffff; iNode++ )
573 {
574 if( !hNodeIndex.has( iNode ) )
575 {
576 hNodeIndex.insert( iNode, hNodeIndex.getSize() );
577 RawStat rs;
578 rs.iNode = iNode;
579 rs.iUser = iUser;
580 rs.iGroup = iGroup;
581 rs.uPerms = uPerms;
582 rs.iLinks = 1;
583 switch( (uPerms&typeMask) )
584 {
585 case typeRegFile:
586 case typeSymLink:
587 rs.uStreamIndex = mStore.allocate();
588 break;
589
590 case typeDir:
591// sio << "Creating directory node, storage: "
592// << rs.uStreamIndex << sio.nl;
593 {
594 Bu::MyriadStream msDir = mStore.create(
595 Bu::Myriad::Write
596 );
597 rs.uStreamIndex = msDir.getId();
598 uint32_t uSize = 0;
599 msDir.write( &uSize, 4 );
600 }
601 break;
602
603 case typeChrDev:
604 case typeBlkDev:
605 rs.uStreamIndex = uSpecial;
606 break;
607
608 default:
609 rs.uStreamIndex = 0;
610 break;
611 }
612 rs.iATime = time(NULL);
613 rs.iMTime = time(NULL);
614 rs.iCTime = time(NULL);
615 writeInode( rs );
616
617 return iNode;
618 }
619 }
620
621 throw Bu::MyriadFsException(
622 "No inode could be allocated. You've run out!");
623}
624
625void Bu::MyriadFs::stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs )
626{
627 RawStat rs;
628 readInode( iNode, rs, rIs );
629 rBuf.iNode = iNode;
630 rBuf.iUser = rs.iUser;
631 rBuf.iGroup = rs.iGroup;
632 rBuf.uPerms = rs.uPerms;
633 rBuf.iLinks = rs.iLinks;
634 rBuf.iATime = rs.iATime;
635 rBuf.iMTime = rs.iMTime;
636 rBuf.iCTime = rs.iCTime;
637 rBuf.uDev = 0;
638 rBuf.iSize = 0;
639 switch( (rBuf.uPerms&typeMask) )
640 {
641 case typeRegFile:
642 case typeSymLink:
643 rBuf.iSize = mStore.getSize( rs.uStreamIndex );
644 break;
645
646 case typeChrDev:
647 case typeBlkDev:
648 rBuf.uDev = rs.uStreamIndex;
649 break;
650
651 default:
652 rBuf.iSize = 0;
653 break;
654 }
655}
656
657void Bu::MyriadFs::writeHeader()
658{
659 Bu::MyriadStream ms = mStore.open( 1, Bu::Myriad::Write );
660 ms.write( Myriad_Fs_MAGIC_CODE, 4 );
661 int8_t iVer = 1;
662 int32_t iNumNodes = hNodeIndex.getSize();
663 ms.write( &iVer, 1 );
664 ms.write( &iNumNodes, 4 ); // iNumNodes
665 for( NodeIndex::iterator i = hNodeIndex.begin(); i; i++ )
666 {
667 int32_t iNode = i.getKey();
668 int32_t iPosition = i.getValue();
669 ms.write( &iNode, 4 );
670 ms.write( &iPosition, 4 );
671 }
672
673 // Truncate the stream afterwards so we don't use up too much space.
674 ms.setSize( ms.tell() );
675}
676
677void Bu::MyriadFs::setTimes( int32_t iNode, int64_t iATime, int64_t iMTime )
678{
679 RawStat rs;
680 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
681
682 readInode( iNode, rs, is );
683 rs.iATime = iATime;
684 rs.iMTime = iMTime;
685 writeInode( rs, is );
686}
687
688void Bu::MyriadFs::destroyNode( int32_t iNode )
689{
690 if( iNode == 0 )
691 throw Bu::MyriadFsException("You cannot destroy the root.");
692
693 uint32_t iPosition;
694 RawStat rsOld;
695
696 Bu::MyriadStream is = mStore.open( 2, Bu::Myriad::ReadWrite );
697
698 // This will be overwritten with the last node
699 iPosition = hNodeIndex.get( iNode );
700 readInode( iNode, rsOld, is );
701
702 switch( (rsOld.uPerms&typeMask) )
703 {
704 case typeRegFile:
705 case typeDir:
706 case typeSymLink:
707 mStore.erase( rsOld.uStreamIndex );
708 break;
709 }
710
711 hNodeIndex.erase( iNode );
712
713 // Read the last node, can't use the helpers, because we don't know the
714 // iNode yet.
715 if( iPosition != hNodeIndex.getSize() )
716 {
717 // If this is the last node, then we don't need to do anything, but
718 // this case handles what to do if we aren't on the last node
719 RawStat rs;
720 is.setPos( (hNodeIndex.getSize())*sizeof(RawStat) );
721 is.read( &rs, sizeof(RawStat) );
722
723 hNodeIndex.get( rs.iNode ) = iPosition;
724 writeInode( rs, is );
725 }
726
727 is.setSize( hNodeIndex.getSize() * sizeof(RawStat) );
728}
729
730Bu::String Bu::MyriadFs::filePart( const Bu::String &sPath )
731{
732 Bu::String::const_iterator iStart = sPath.begin();
733 if( *iStart == '/' )
734 iStart++;
735 for( Bu::String::const_iterator iEnd = iStart.find('/'); iEnd;
736 iStart = iEnd+1, iEnd = iStart.find('/') ) { }
737 return Bu::String( iStart, sPath.end() );
738}
739
diff --git a/src/unstable/myriadfs.h b/src/unstable/myriadfs.h
new file mode 100644
index 0000000..e3008bc
--- /dev/null
+++ b/src/unstable/myriadfs.h
@@ -0,0 +1,211 @@
1/*
2 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef MYRIAD_FS_H
9#define MYRIAD_FS_H
10
11#include <sys/types.h>
12
13#include "bu/myriad.h"
14#include "bu/debugmutex.h"
15
16namespace Bu
17{
18 class Stream;
19
20 subExceptionDecl( MyriadFsException );
21
22 /**
23 * A POSIX compliant, node based filesystem built on top of Myriad.
24 *
25 * A header is placed into stream 1.
26 * Header format:
27 * int32_t iMagicHeader (A7188B39)
28 * int8_t iVersion (1)
29 * int32_t iNumNodes
30 * NodeLookup[iNumNodes] nNode
31 *
32 * Node lookup:
33 * int32_t iInode
34 * int32_t iPosition
35 *
36 * The node headers or inode structures have a base size of 44 bytes.
37 * The name is stored in the directory format.
38 * Basic node header format:
39 * int32_t iNode
40 * int32_t iUser
41 * int32_t iGroup
42 * uint16_t uPerms
43 * int16_t iLinks
44 * uint32_t uStreamIndex
45 * int64_t iATime
46 * int64_t iMTime
47 * int64_t iCTime
48 *
49 * Some types get special formats for their assosiated data stream, or
50 * other special considerations, here's a list:
51 *
52 * - typeFifo: No stream, uStreamIndex unused (probably)
53 * - typeChrDev: No stream, uStreamIndex is device hi/lo
54 * - typeDir: The stream contains a directory contents listing, described
55 * below
56 * - typeBlkDev: No stream, uStreamIndex is device hi/lo
57 * - typeRegFile: The stream is the file data
58 * - typeSymLink: The stream is the destination of the symlink
59 * - typeSocket: No steram, uStreamIndex unused (probably)
60 *
61 * Directory streams have this simple listing format. They contain a list
62 * of all child elements, with no particular order at the moment. The . and
63 * .. entries are not listed, they are implicit:
64 * int32_t iNumNodes
65 * NodeTable[iNumNodes] nChildren
66 *
67 * NodeTable:
68 * int32_t iInode
69 * uint8_t uNameSize
70 * char[uNameSize] sName
71 */
72 class MyriadFs
73 {
74 public:
75 MyriadFs( Bu::Stream &rStore, int iBlockSize=512 );
76 virtual ~MyriadFs();
77
78 enum
79 {
80 permOthX = 0000001,
81 permOthW = 0000002,
82 permOthR = 0000004,
83 permGrpX = 0000010,
84 permGrpW = 0000020,
85 permGrpR = 0000040,
86 permUsrX = 0000100,
87 permUsrW = 0000200,
88 permUsrR = 0000400,
89 permSticky = 0001000,
90 permSetGid = 0002000,
91 permSetUid = 0004000,
92 permMask = 0007777,
93 typeFifo = 0010000,
94 typeChrDev = 0020000,
95 typeDir = 0040000,
96 typeBlkDev = 0060000,
97 typeRegFile = 0100000,
98 typeSymLink = 0120000,
99 typeSocket = 0140000,
100 typeMask = 0170000
101 };
102
103 enum
104 {
105 Read = 0x01, ///< Open file for reading
106 Write = 0x02, ///< Open file for writing
107 Create = 0x04, ///< Create file if it doesn't exist
108 Truncate = 0x08, ///< Truncate file if it does exist
109 Append = 0x10, ///< Always append on every write
110 NonBlock = 0x20, ///< Open file in non-blocking mode
111 Exclusive = 0x40, ///< Create file, if it exists then fail
112
113 // Helpful mixes
114 ReadWrite = 0x03, ///< Open for reading and writing
115 WriteNew = 0x0E ///< Create a file (or truncate) for writing.
116 /// Same as Write|Create|Truncate
117 };
118
119 class Stat
120 {
121 public:
122 int32_t iNode;
123 int32_t iUser;
124 int32_t iGroup;
125 uint16_t uPerms;
126 int16_t iLinks;
127 int64_t iATime;
128 int64_t iMTime;
129 int64_t iCTime;
130 int32_t iSize;
131 uint32_t uDev;
132 Bu::String sName;
133 };
134 typedef Bu::List<Stat> Dir;
135
136 void stat( const Bu::String &sPath, Stat &rBuf );
137 MyriadStream open( const Bu::String &sPath, int iMode,
138 uint16_t uPerms=0664 );
139 void create( const Bu::String &sPath, uint16_t iPerms );
140 void create( const Bu::String &sPath, uint16_t iPerms,
141 uint16_t iDevHi, uint16_t iDevLo );
142 void create( const Bu::String &sPath, uint16_t iPerms,
143 uint32_t uSpecial );
144 void mkDir( const Bu::String &sPath, uint16_t iPerms );
145 void mkSymLink( const Bu::String &sTarget, const Bu::String &sPath );
146 void mkHardLink( const Bu::String &sTarget, const Bu::String &sPath );
147 Bu::String readSymLink( const Bu::String &sPath );
148 Dir readDir( const Bu::String &sPath );
149 void setTimes( const Bu::String &sPath, int64_t iATime,
150 int64_t iMTime );
151 void unlink( const Bu::String &sPath );
152 void setFileSize( const Bu::String &sPath, int32_t iSize );
153 void rename( const Bu::String &sFrom, const Bu::String &sTo );
154
155 static dev_t devToSys( uint32_t uDev );
156 static uint32_t sysToDev( dev_t uDev );
157
158 private:
159 class RawStat
160 {
161 public:
162 int32_t iNode;
163 int32_t iUser;
164 int32_t iGroup;
165 uint16_t uPerms;
166 int16_t iLinks;
167 uint32_t uStreamIndex;
168 int64_t iATime;
169 int64_t iMTime;
170 int64_t iCTime;
171 };
172 typedef Bu::Hash<int32_t, int32_t> NodeIndex;
173
174 private:
175 /**
176 * Lookup inode.
177 */
178 int32_t lookupInode( const Bu::String &sPath, int32_t &iParent );
179 /**
180 * Lookup inode.
181 */
182 int32_t lookupInode( Bu::String::const_iterator iStart,
183 int32_t iNode, int32_t &iParent );
184 void readInode( int32_t iNode, RawStat &rs, MyriadStream &rIs );
185 void readInode( int32_t iNode, RawStat &rs );
186 void writeInode( const RawStat &rs );
187 void writeInode( const RawStat &rs, MyriadStream &rOs );
188 Dir readDir( int32_t iNode );
189 MyriadStream openByInode( int32_t iNode );
190 void addToDir( int32_t iDir, int32_t iNode, const Bu::String &sName );
191 int32_t create( int32_t iParent, const Bu::String &sName,
192 uint16_t uPerms, uint32_t uSpecial );
193 int32_t allocInode( uint16_t uPerms, uint32_t uSpecial );
194 void stat( int32_t iNode, Stat &rBuf, MyriadStream &rIs );
195 void writeHeader();
196 void setTimes( int32_t iNode, int64_t iATime, int64_t iMTime );
197 void destroyNode( int32_t iNode );
198
199 static Bu::String filePart( const Bu::String &sPath );
200
201 private:
202 Bu::Stream &rStore;
203 Bu::Myriad mStore;
204 Bu::DebugMutex mAccess;
205 NodeIndex hNodeIndex;
206 int32_t iUser;
207 int32_t iGroup;
208 };
209};
210
211#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-20 19:33:51 +0000