aboutsummaryrefslogtreecommitdiff
path: root/src/stable
diff options
context:
space:
mode:
Diffstat (limited to 'src/stable')
-rw-r--r--src/stable/myriad.cpp663
-rw-r--r--src/stable/myriad.h222
-rw-r--r--src/stable/myriadstream.cpp309
-rw-r--r--src/stable/myriadstream.h61
4 files changed, 1255 insertions, 0 deletions
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
new file mode 100644
index 0000000..de44930
--- /dev/null
+++ b/src/stable/myriad.cpp
@@ -0,0 +1,663 @@
1/*
2 * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriad.h"
9#include "bu/stream.h"
10#include "bu/myriadstream.h"
11#include <stdio.h>
12
13#include "bu/sio.h"
14using Bu::sio;
15using Bu::Fmt;
16
17#define Myriad_MAGIC_CODE ((unsigned char *)"\x0a\xd3\xfa\x84")
18
19namespace Bu
20{
21 subExceptionDef( MyriadException )
22 template<typename t> t blkDiv( t total, t block ) {
23 return (total/block)+((total%block==0)?(0):(1));
24 }
25}
26
27Bu::Myriad::Myriad( Bu::Stream &sStore, int iBlockSize, int iPreallocate ) :
28 sStore( sStore ),
29 iBlockSize( iBlockSize ),
30 iBlocks( 0 ),
31 iUsed( 0 ),
32 bHeaderChanged( false )
33{
34 try
35 {
36 initialize();
37 }
38 catch( Bu::MyriadException &e )
39 {
40 if( e.getErrorCode() == MyriadException::emptyStream )
41 {
42 initialize( iBlockSize, iPreallocate );
43 }
44 else
45 {
46 throw;
47 }
48 }
49}
50
51Bu::Myriad::~Myriad()
52{
53 if( !hActiveBlocks.isEmpty() )
54 {
55 sio << "Bu::Myriad::~Myriad(): Error: There are "
56 << hActiveBlocks.getSize() << " unsynced blocks!" << sio.nl;
57 }
58 sync();
59
60 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
61 {
62 delete *i;
63 }
64}
65
66void Bu::Myriad::sync()
67{
68 updateHeader();
69
70 for( BlockHash::iterator i = hActiveBlocks.begin(); i; i++ )
71 {
72 if( (*i)->bChanged )
73 {
74 syncBlock( *i );
75 }
76 }
77}
78
79void Bu::Myriad::initialize()
80{
81 sStore.setPosEnd( 0 );
82 int iSize = sStore.tell();
83 sStore.setPos( 0 );
84
85 unsigned char buf[4];
86 if( sStore.read( buf, 4 ) < 4 )
87 throw MyriadException( MyriadException::emptyStream,
88 "Input stream appears to be empty.");
89 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
90 {
91 throw MyriadException( MyriadException::invalidFormat,
92 "Stream does not appear to be a valid Myriad format.");
93 }
94 sStore.read( buf, 2 );
95 if( buf[0] != 1 )
96 throw MyriadException( MyriadException::badVersion,
97 "We can only handle version 1 for now.");
98 if( buf[1] != 32 )
99 throw MyriadException( MyriadException::invalidWordSize,
100 "We can only handle 32-bit words at the moment.");
101 sStore.read( &iBlockSize, 4 );
102 int iStreams;
103 sStore.read( &iStreams, 4 );
104
105 iBlocks = iSize/iBlockSize;
106 //sio << "Myriad: iSize=" << iSize << ", iBlockSize=" << iBlockSize
107 // << ", iBlocks=" << iBlocks << ", iStreams=" << iStreams << sio.nl;
108
109 int iHeaderSize = 14 + 8 + 4;
110 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
111
112 while( iHeaderSize > iHeaderBlocks*iBlockSize )
113 {
114 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
115 iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
116 }
117
118 //sio << "Myriad: iHeaderSize=" << iHeaderSize
119 // << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
120
121 Stream *pFakeHdr = new Stream;
122 pFakeHdr->iId = 0;
123 pFakeHdr->iSize = iHeaderSize;
124 for( int j = 0; j < iHeaderBlocks; j++ )
125 {
126 pFakeHdr->aBlocks.append( j );
127 }
128
129 bsBlockUsed.setSize( iBlocks, true );
130
131// bool bCanSkip = false; // Can skip around, post initial header stream i/o
132 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
133 pIn->setPos( sStore.tell() );
134 for( int j = 0; j < iStreams; j++ )
135 {
136 aStreams.append( new Stream() );
137 Stream &s = *aStreams[j];
138 pIn->read( &s.iId, 4 );
139 pIn->read( &s.iSize, 4 );
140 int iSBlocks = blkDiv(s.iSize, iBlockSize);
141 // sio << "Myriad: - Stream::iId=" << s.iId
142 // << ", Stream::iSize=" << s.iSize
143 // << ", Stream::aBlocks=" << iSBlocks
144 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
145 for( int k = 0; k < iSBlocks; k++ )
146 {
147 int iBId;
148 pIn->read( &iBId, 4 );
149 // sio << "Myriad: - iBId=" << iBId
150 // << ", iStartPos=" << iBId*iBlockSize
151 // << ", pIn->tell()=" << pIn->tell() << sio.nl;
152 s.aBlocks.append( iBId );
153 bsBlockUsed.setBit( iBId );
154 iUsed++;
155 if( (j == 0 && k == iHeaderBlocks-1) )
156 {
157 // sio << "Myriad: - End of prepartition, unlocking skipping."
158 // << sio.nl;
159// bCanSkip = true;
160 MyriadStream *pTmp = new MyriadStream( *this, aStreams[0] );
161 // sio << "Myriad - Position = " << pIn->tell() << sio.nl;
162 pTmp->setPos( pIn->tell() );
163 delete pIn;
164 delete pFakeHdr;
165 pIn = pTmp;
166 }
167 }
168 }
169 delete pIn;
170
171 //sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl;
172}
173
174void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
175{
176 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
177 {
178 delete *i;
179 }
180 aStreams.clear();
181 iUsed = 0;
182
183 int iHeaderSize = 14 + 8 + 4;
184 int iHeaderBlocks = 0; //blkDiv( iHeaderSize+4, iBlockSize );
185 char cBuf = 1;
186 int iBuf = 0;
187
188 Stream *pStr = new Stream;
189 pStr->iId = 0;
190
191 while( iHeaderSize > iHeaderBlocks*iBlockSize )
192 {
193 iHeaderBlocks = blkDiv( iHeaderSize+4, iBlockSize );
194 iHeaderSize = 14 + 8 + 4*iHeaderBlocks;
195 }
196
197 iPreAllocate += iHeaderBlocks;
198
199 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize="
200 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
201
202 bsBlockUsed.setSize( iPreAllocate, true );
203 iUsed++;
204
205 char *pBlock = new char[iBlockSize];
206 memset( pBlock, 0, iBlockSize );
207 for( int j = 0; j < iPreAllocate; j++ )
208 {
209 sStore.write( pBlock, iBlockSize );
210 }
211 delete[] (char *)pBlock;
212
213 sStore.setPos( 0 );
214
215 // Magic number
216 sStore.write( Myriad_MAGIC_CODE, 4 );
217
218 // Version (0)
219 sStore.write( &cBuf, 1 );
220
221 // Bits per int
222 cBuf = 32;
223 sStore.write( &cBuf, 1 );
224
225 // The size of each block
226 sStore.write( &iBlockSize, 4 );
227
228 iBuf = 1;
229 // The number of streams
230 sStore.write( &iBuf, 4 );
231
232 // Stream header
233 iBuf = 0;
234 sStore.write( &iBuf, 4 );
235 sStore.write( &iHeaderSize, 4 );
236 for( iBuf = 0; iBuf < iHeaderBlocks; iBuf++ )
237 {
238 sStore.write( &iBuf, 4 );
239 }
240
241 this->iBlockSize = iBlockSize;
242 this->iBlocks = iPreAllocate;
243
244 pStr->iSize = sStore.tell();
245// sio << "Myriad: Actual end of header stream = " << pStr->iSize << sio.nl;
246
247 pStr->iSize = iHeaderSize;
248 for( int j = 0; j < iHeaderBlocks; j++ )
249 {
250 pStr->aBlocks.append( j );
251 bsBlockUsed.setBit( j );
252 iUsed++;
253 }
254
255 aStreams.append( pStr );
256
257 //sio << bsBlockUsed.toString() << " - " << pStr->aBlocks << sio.nl;
258
259 bHeaderChanged = true;
260 //hStreams.insert( 0, BlockArray( 0 ) );
261}
262
263void Bu::Myriad::updateHeader()
264{
265 if( bHeaderChanged == false )
266 return;
267 if( !sStore.canWrite() )
268 return;
269
270 char cBuf;
271 int iBuf;
272
273 //for( StreamArray::iterator i = aStreams.begin(); i; i++ )
274 //{
275 // sio << "Myriad: Stream " << Fmt(4) << (*i)->iId << ": " << (*i)->aBlocks << sio.nl;
276 //}
277
278 // Compute the new size of the header.
279 int iHeaderSize = 14 + 8*aStreams.getSize();
280// sio << "Myriad: updateHeader: aStreams.getSize() = " << aStreams.getSize()
281// << sio.nl;
282 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
283 {
284 iHeaderSize += 4*(*i)->aBlocks.getSize();
285// sio << "Myriad: updateHeader: (*i)->aBlocks.getSize() = "
286// << (*i)->aBlocks.getSize() << sio.nl;
287 }
288 int iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
289 while( iNewBlocks > aStreams[0]->aBlocks.getSize() )
290 {
291 int iBlock = findEmptyBlock();
292// sio << "Myriad: updateHeader: Appending block " << iBlock
293// << " to header." << sio.nl;
294 aStreams[0]->aBlocks.append( iBlock );
295 bsBlockUsed.setBit( iBlock );
296 iUsed++;
297 iHeaderSize += 4;
298 iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
299 }
300 aStreams[0]->iSize = iHeaderSize;
301// sio << "Myriad: updateHeader: iHeaderSize=" << iHeaderSize
302// << ", iNewBlocks=" << iNewBlocks << ", curBlocks="
303// << aStreams[0]->aBlocks.getSize() << sio.nl;
304
305 MyriadStream sHdr( *this, aStreams[0] );
306 sHdr.write( Myriad_MAGIC_CODE, 4 );
307
308 // Version (1)
309 cBuf = 1;
310 sHdr.write( &cBuf, 1 );
311
312 // Bits per int
313 cBuf = 32;
314 sHdr.write( &cBuf, 1 );
315
316 // The size of each block
317 sHdr.write( &iBlockSize, 4 );
318
319 iBuf = aStreams.getSize();
320 // The number of streams
321 sHdr.write( &iBuf, 4 );
322
323 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
324 {
325 sHdr.write( &(*i)->iId, 4 );
326 sHdr.write( &(*i)->iSize, 4 );
327 int iUsedBlocks = blkDiv( (*i)->iSize, iBlockSize );
328// for( BlockArray::iterator j = (*i)->aBlocks.begin(); j; j++ )
329 for( int j = 0; j < iUsedBlocks; j++ )
330 {
331 sHdr.write( &(*i)->aBlocks[j], 4 );
332 }
333 }
334
335 bHeaderChanged = false;
336}
337
338int Bu::Myriad::createStream( int iPreAllocate )
339{
340 Stream *pStr = new Stream();
341 pStr->iId = aStreams.last()->iId+1;
342 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
343 // << iPreAllocate << sio.nl;
344 pStr->iSize = 0;
345 aStreams.append( pStr );
346
347 for( int j = 0; j < iPreAllocate; j++ )
348 {
349 int iFreeBlock = findEmptyBlock();
350// sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
351 pStr->aBlocks.append( iFreeBlock );
352 bsBlockUsed.setBit( iFreeBlock );
353 iUsed++;
354 }
355
356 bHeaderChanged = true;
357
358 return pStr->iId;
359}
360
361int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate )
362{
363 try
364 {
365 findStream( iId );
366 throw MyriadException( MyriadException::streamExists,
367 "There is already a stream with the given id.");
368 }
369 catch( MyriadException &e )
370 {
371 Stream *pStr = new Stream();
372 pStr->iId = iId;
373 //sio << "Myriad: New stream id=" << pStr->iId << ", iPreAllocate="
374 // << iPreAllocate << sio.nl;
375 pStr->iSize = 0;
376 if( aStreams.last()->iId < iId )
377 {
378 aStreams.append( pStr );
379 }
380 else
381 {
382 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
383 {
384 if( (*i)->iId > iId )
385 {
386 aStreams.insert( i, pStr );
387 break;
388 }
389 }
390 }
391
392 for( int j = 0; j < iPreAllocate; j++ )
393 {
394 int iFreeBlock = findEmptyBlock();
395 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
396 pStr->aBlocks.append( iFreeBlock );
397 bsBlockUsed.setBit( iFreeBlock );
398 iUsed++;
399 }
400
401 bHeaderChanged = true;
402
403 return pStr->iId;
404 }
405}
406
407int Bu::Myriad::findEmptyBlock()
408{
409 bHeaderChanged = true;
410
411 for( int j = 0; j < bsBlockUsed.getSize(); j++ )
412 {
413 if( bsBlockUsed.getBit( j ) == false )
414 return j;
415 }
416// sio << "Myriad: findEmptyBlock(): No empty blocks, adding new one." << sio.nl;
417
418 bsBlockUsed.setSize( bsBlockUsed.getSize()+1, false );
419 /*
420 sStore.setPos( iBlockSize*iBlocks );
421
422 char *pBlock = new char[iBlockSize];
423 memset( pBlock, 0, iBlockSize );
424 sStore.write( pBlock, iBlockSize );
425 delete[] pBlock;
426 */
427
428 sStore.setSize( (iBlocks+1)*iBlockSize );
429
430 return iBlocks++;
431}
432
433void Bu::Myriad::deleteStream( int iId )
434{
435 if( iId < 0 )
436 throw MyriadException( MyriadException::invalidStreamId,
437 "Invalid stream id.");
438 if( iId == 0 )
439 throw MyriadException( MyriadException::protectedStream,
440 "You cannot delete stream zero, it is protected.");
441 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
442 {
443 if( (*i)->iId == iId )
444 {
445 Stream *pStream = *i;
446 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
447 {
448 bsBlockUsed.setBit( *j, false );
449 iUsed--;
450 }
451 aStreams.erase( i );
452 bHeaderChanged = true;
453 delete pStream;
454 return;
455 }
456 }
457}
458
459Bu::Array<int> Bu::Myriad::getStreamIds()
460{
461 Bu::Array<int> aRet( aStreams.getSize() );
462 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
463 {
464 aRet.append( (*i)->iId );
465 }
466
467 return aRet;
468}
469
470int Bu::Myriad::getStreamSize( int iId )
471{
472 return findStream( iId )->iSize;
473}
474
475bool Bu::Myriad::hasStream( int iId )
476{
477 try
478 {
479 findStream( iId );
480 return true;
481 }catch(...)
482 {
483 return false;
484 }
485}
486
487Bu::MyriadStream Bu::Myriad::openStream( int iId )
488{
489 //sio << "Myriad: Request to open stream: " << iId << sio.nl;
490 return MyriadStream( *this, findStream( iId ) );
491}
492
493int Bu::Myriad::getNumStreams()
494{
495 return aStreams.getSize();
496}
497
498int Bu::Myriad::getBlockSize()
499{
500 return iBlockSize;
501}
502
503int Bu::Myriad::getNumBlocks()
504{
505 return iBlocks;
506}
507
508int Bu::Myriad::getNumUsedBlocks()
509{
510 return iUsed;
511}
512
513int Bu::Myriad::getTotalUsedBytes()
514{
515 int iTotalSize = 0;
516 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
517 {
518 iTotalSize += (*i)->iSize;
519 }
520 return iTotalSize;
521}
522
523int Bu::Myriad::getTotalUnusedBytes()
524{
525 int iTotalSize = (iBlocks-iUsed)*iBlockSize;
526 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
527 {
528 iTotalSize += iBlockSize - ((*i)->iSize%iBlockSize);
529 }
530 return iTotalSize;
531}
532
533int Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize )
534{
535 int iTotalSize = (iBlocks-iUsed)*iFakeBlockSize;
536 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
537 {
538 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize);
539 }
540 return iTotalSize;
541}
542
543Bu::Myriad::Stream *Bu::Myriad::findStream( int iId )
544{
545 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
546 {
547 if( (*i)->iId == iId )
548 return *i;
549 }
550
551 throw MyriadException( MyriadException::noSuchStream,
552 "The requested stream doesn't exist and cannot be opened." );
553
554 return NULL;
555}
556
557Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock )
558{
559// sio << "Myriad: Reading block " << iBlock << ", bytes "
560// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl;
561 Block *pBlock = new Block;
562 pBlock->pData = new char[iBlockSize];
563 sStore.setPos( iBlockSize * iBlock );
564 sStore.read( pBlock->pData, iBlockSize );
565 pBlock->bChanged = false;
566 pBlock->iBlockIndex = iBlock;
567
568 hActiveBlocks.insert( iBlock, pBlock );
569
570 return pBlock;
571}
572
573void Bu::Myriad::releaseBlock( Bu::Myriad::Block *pBlock )
574{
575 if( pBlock == NULL )
576 return;
577// sio << "Myriad: Releasing block " << pBlock->iBlockIndex << sio.nl;
578 syncBlock( pBlock );
579 hActiveBlocks.erase( pBlock->iBlockIndex );
580 delete[] pBlock->pData;
581 delete pBlock;
582}
583
584void Bu::Myriad::syncBlock( Block *pBlock )
585{
586 if( pBlock->bChanged )
587 {
588// sio << "Myriad: - Block changed, writing back to stream." << sio.nl;
589 sStore.setPos( iBlockSize * pBlock->iBlockIndex );
590 sStore.write( pBlock->pData, iBlockSize );
591 pBlock->bChanged = false;
592 }
593}
594
595int Bu::Myriad::streamAddBlock( Stream *pStream )
596{
597 int iBlock = findEmptyBlock();
598 pStream->aBlocks.append( iBlock );
599 bsBlockUsed.setBit( iBlock );
600 bHeaderChanged = true;
601 iUsed++;
602 return iBlock;
603}
604
605void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
606{
607 if( pStream->iSize == iSize )
608 {
609 return;
610 }
611 else if( pStream->iSize > iSize )
612 {
613 // Shrink
614 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
615 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize )
616 {
617 if( bsBlockUsed.getBit( pStream->aBlocks.last() ) )
618 iUsed--;
619 bsBlockUsed.setBit( pStream->aBlocks.last(), false );
620 pStream->aBlocks.eraseLast();
621 }
622 pStream->iSize = iSize;
623 bHeaderChanged = true;
624 }
625 else
626 {
627 // Grow
628 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
629 iNewSize < iSize; iNewSize += iBlockSize )
630 {
631 streamAddBlock( pStream );
632 }
633 pStream->iSize = iSize;
634 bHeaderChanged = true;
635 }
636}
637
638void Bu::Myriad::headerChanged()
639{
640 bHeaderChanged = true;
641}
642
643bool Bu::Myriad::isMyriad( Bu::Stream &sStore )
644{
645 sStore.setPos( 0 );
646
647 unsigned char buf[4];
648 if( sStore.read( buf, 4 ) < 4 )
649 throw MyriadException( MyriadException::emptyStream,
650 "Input stream appears to be empty.");
651 sStore.setPos( 0 );
652 if( memcmp( buf, Myriad_MAGIC_CODE, 4 ) )
653 {
654 return false;
655 }
656 return true;
657}
658
659const Bu::BitString &Bu::Myriad::getBlocksUsed() const
660{
661 return bsBlockUsed;
662}
663
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
new file mode 100644
index 0000000..3382ab5
--- /dev/null
+++ b/src/stable/myriad.h
@@ -0,0 +1,222 @@
1/*
2 * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_H
9#define BU_MYRIAD_H
10
11#include <stdint.h>
12#include "bu/bitstring.h"
13#include "bu/exceptionbase.h"
14#include "bu/array.h"
15#include "bu/hash.h"
16
17namespace Bu
18{
19 class Stream;
20 class MyriadStream;
21
22 subExceptionDeclBegin( MyriadException )
23 enum
24 {
25 emptyStream,
26 invalidFormat,
27 badVersion,
28 invalidWordSize,
29 noSuchStream,
30 streamExists,
31 invalidStreamId,
32 protectedStream
33 };
34 subExceptionDeclEnd();
35
36 /**
37 * Myriad block-allocated stream multiplexing system. This is a system for
38 * creating streams that contain other streams in a flexible and lightweight
39 * manner. Basically, you can create a file (or any other stream) that can
40 * store any number of flexible, growing streams. The streams within the
41 * Myriad stream are automatically numbered, not named. This works more
42 * or less like a filesystem, but without the extra layer for managing
43 * file and directory links. This would actually be very easy to add
44 * on top of Myriad, but is not required.
45 *
46 * Header format is as follows:
47 *
48 * MMMMvBssssSSSS*
49 * M = Magic number (0AD3FA84)
50 * v = version number
51 * B = Bits per int
52 * s = Blocksize in bytes
53 * S = Number of Streams
54 *
55 * The * represents the Stream headers, one per stream, as follows:
56 * IIIIssss$
57 * I = Id number of the stream
58 * s = size of stream in bytes
59 *
60 * The $ represents the Block headers, one per used block, as follows:
61 * IIII
62 * I = Index of the block
63 *
64 * The stream/block data is interleaved in the header, so all blocks stored
65 * with one stream are together. The block headers are in order, and the
66 * data in them is required to be "solid" you cannot fill partial blocks
67 * mid-way through a stream.
68 *
69 * The initial block starts with the nids header, and is both the zero block
70 * and the zero stream. For now, the minimum block size is the size needed
71 * to store the base header, the zero stream header, and the first two
72 * blocks of the zero stream, so 30 bytes. Since it's reccomended to use
73 * a size that will fit evenly into filesystem blocks, then a size of 32 is
74 * probably the smallest reccomended size because all powers of two equal
75 * to or greater than 32 are evenly divisible by 32.
76 *
77 * I have had a thought that if the block size were smaller than 42 bytes
78 * the header would consume the first N blocks where N * block size is
79 * enough space to house the initial header, the first stream header, and
80 * the first N block headers. This, of course, causes you to hit an
81 * infinite header if the block size is small enough.
82 */
83 class Myriad
84 {
85 friend class MyriadStream;
86 public:
87 /**
88 * Create a Myriad object that uses the given stream to store data.
89 * This stream must be random access. The block size and preallocate
90 * values passed in are values that will be used if the given stream
91 * is empty. In that case the stream will be "formatted" for myriad
92 * with the specified block size. If there is already a viable Myriad
93 * format present in the stream, then the blocksize and preallocate
94 * values will be ignored and the values from the stream will be used
95 * instead. If the stream doesn't appear to be Myriad formatted an
96 * exception will be thrown.
97 */
98 Myriad( Bu::Stream &sStore, int iBlockSize=512, int iPreallocate=8 );
99 virtual ~Myriad();
100
101 /**
102 * Destroy whatever data may be in the base stream and create a new
103 * Myriad system there with the given blocksize. Use this with care,
104 * it will destroy anything that was already in the stream, and
105 * generally, should not ever have to be used.
106 */
107 void initialize( int iBlockSize, int iPreAllocate=1 );
108
109 /**
110 * Create a new stream within the Myriad system. The ID of the new
111 * stream is returned.
112 */
113 int createStream( int iPreAllocate=1 );
114
115 /**
116 * Create a new stream within the Myriad system with a given id. The
117 * id that you provide will be the new id of the stream unless it's
118 * already used, in which case an error is thrown. This is primarilly
119 * useful when copying an old Myriad file into a new one.
120 */
121 int createStreamWithId( int iId, int iPreAllocate=1 );
122
123 /**
124 * Delete a stream that's already within the Myriad.
125 */
126 void deleteStream( int iId );
127
128 /**
129 * Return a new Stream object assosiated with the given stream ID.
130 */
131 MyriadStream openStream( int iId );
132
133 Bu::Array<int> getStreamIds();
134 int getStreamSize( int iId );
135 bool hasStream( int iId );
136
137 int getNumStreams();
138 int getBlockSize();
139 int getNumBlocks();
140 int getNumUsedBlocks();
141 int getTotalUsedBytes();
142 int getTotalUnusedBytes();
143 int getTotalUnusedBytes( int iFakeBlockSize );
144
145 /**
146 * Syncronize the header data, etc. with the storage stream. It's not
147 * a bad idea to call this periodically.
148 */
149 void sync();
150
151 /**
152 * Read the first few bytes from the given stream and return true/false
153 * depending on weather or not it's a Myriad stream. This will throw
154 * an exception if the stream is empty, or is not random access.
155 */
156 static bool isMyriad( Bu::Stream &sStore );
157
158 const Bu::BitString &getBlocksUsed() const;
159
160 private:
161 /**
162 * Initialize this object based on the data already in the assosiated
163 * stream. This will be called automatically for you if you forget,
164 * but if you want to pre-initialize for some reason, just call this
165 * once before you actually start doing anything with your Myriad.
166 */
167 void initialize();
168
169 enum
170 {
171 blockUnused = 0xFFFFFFFFUL
172 };
173
174 typedef Bu::Array<int> BlockArray;
175 class Stream
176 {
177 public:
178 int iId;
179 int iSize;
180 BlockArray aBlocks;
181 };
182 typedef Bu::Array<Stream *> StreamArray;
183
184 class Block
185 {
186 public:
187 char *pData;
188 bool bChanged;
189 int iBlockIndex;
190 };
191
192 void updateHeader();
193 int findEmptyBlock();
194
195 /**
196 *@todo Change this to use a binary search, it's nicer.
197 */
198 Stream *findStream( int iId );
199
200 Block *getBlock( int iBlock );
201 void releaseBlock( Block *pBlock );
202 void syncBlock( Block *pBlock );
203
204 int streamAddBlock( Stream *pStream );
205 void setStreamSize( Stream *pStream, long iSize );
206
207 void headerChanged();
208
209 private:
210 Bu::Stream &sStore;
211 int iBlockSize;
212 int iBlocks;
213 int iUsed;
214 Bu::BitString bsBlockUsed;
215 StreamArray aStreams;
216 typedef Bu::Hash<int, Block *> BlockHash;
217 BlockHash hActiveBlocks;
218 bool bHeaderChanged;
219 };
220};
221
222#endif
diff --git a/src/stable/myriadstream.cpp b/src/stable/myriadstream.cpp
new file mode 100644
index 0000000..e6e94bc
--- /dev/null
+++ b/src/stable/myriadstream.cpp
@@ -0,0 +1,309 @@
1/*
2 * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#include "bu/myriadstream.h"
9
10#include <string.h>
11
12// #define MYRIAD_STREAM_DEBUG 1
13
14#ifdef MYRIAD_STREAM_DEBUG
15#include "bu/sio.h"
16
17using Bu::sio;
18using Bu::Fmt;
19#endif
20
21Bu::MyriadStream::MyriadStream( Bu::Myriad &rMyriad,
22 Bu::Myriad::Stream *pStream ) :
23 rMyriad( rMyriad ),
24 pStream( pStream ),
25 pCurBlock( NULL ),
26 iPos( 0 )
27{
28#ifdef MYRIAD_STREAM_DEBUG
29 sio << "MyriadStream: " << __LINE__ << ": Created, iId=" << pStream->iId << ", iSize="
30 << pStream->iSize << sio.nl;
31#endif
32 //pCurBlock = rMyriad.newBlock();
33 //rMyriad.getBlock( uStream, pCurBlock );
34 //uSize = pCurBlock->uBytesUsed;
35}
36
37Bu::MyriadStream::~MyriadStream()
38{
39 if( pCurBlock )
40 rMyriad.releaseBlock( pCurBlock );
41 //rMyriad.updateStreamSize( uStream, uSize );
42 //rMyriad.deleteBlock( pCurBlock );
43}
44
45void Bu::MyriadStream::close()
46{
47}
48
49Bu::size Bu::MyriadStream::read( void *pBuf, Bu::size nBytes )
50{
51#ifdef MYRIAD_STREAM_DEBUG
52 sio << "MyriadStream: read: " << __LINE__ << ": Started, asked to read " << nBytes << "b."
53 << sio.nl;
54#endif
55 if( nBytes > (Bu::size)pStream->iSize-iPos )
56 nBytes = pStream->iSize-iPos;
57 if( nBytes <= 0 )
58 return 0;
59 int iLeft = nBytes;
60#ifdef MYRIAD_STREAM_DEBUG
61 sio << "MyriadStream: read: " << __LINE__ << ": Started, going to read " << nBytes << "b."
62 << sio.nl;
63#endif
64 if( pCurBlock == NULL )
65 {
66#ifdef MYRIAD_STREAM_DEBUG
67 sio << "MyriadStream: read: " << __LINE__ << ": No block loaded, loading initial block."
68 << sio.nl;
69#endif
70 pCurBlock = rMyriad.getBlock(
71 pStream->aBlocks[iPos/rMyriad.iBlockSize]
72 );
73 }
74 while( iLeft > 0 )
75 {
76 int iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
77 if( pCurBlock->iBlockIndex != iCurBlock )
78 {
79#ifdef MYRIAD_STREAM_DEBUG
80 sio << "MyriadStream: read: " << __LINE__ << ": Loading new block " << iCurBlock << "."
81 << sio.nl;
82#endif
83 rMyriad.releaseBlock( pCurBlock );
84 pCurBlock = rMyriad.getBlock( iCurBlock );
85 }
86
87 int iAmnt = Bu::min(
88 Bu::min(
89 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
90 iLeft
91 ),
92 pStream->iSize-iPos
93 );
94#ifdef MYRIAD_STREAM_DEBUG
95 sio << "MyriadStream: read: " << __LINE__ << ": Copying out bytes: "
96 << iPos << "(" << (iPos%rMyriad.iBlockSize) << ")+"
97 << iAmnt
98 << ", " << iLeft << "b left." << sio.nl;
99#endif
100 memcpy(
101 pBuf,
102 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
103 iAmnt
104 );
105 iPos += iAmnt;
106 pBuf = &((char *)pBuf)[iAmnt];
107 iLeft -= iAmnt;
108 }
109 return nBytes;
110}
111
112Bu::size Bu::MyriadStream::write( const void *pBuf, Bu::size nBytes )
113{
114 if( nBytes <= 0 )
115 return 0;
116
117#ifdef MYRIAD_STREAM_DEBUG
118 sio << "MyriadStream: write: " << __LINE__ << ": Started, asked to write " << nBytes << "b."
119 << sio.nl;
120#endif
121 if( nBytes <= 0 )
122 return 0;
123 int iLeft = nBytes;
124 /*
125 if( pCurBlock == NULL )
126 {
127#ifdef MYRIAD_STREAM_DEBUG
128 sio << "MyriadStream: write: No block loaded, loading initial block."
129 << sio.nl;
130#endif
131 pCurBlock = rMyriad.getBlock(
132 pStream->aBlocks[iPos/rMyriad.iBlockSize]
133 );
134 }*/
135
136 while( iLeft > 0 )
137 {
138 int iCurBlock;
139 if( iPos/rMyriad.iBlockSize < pStream->aBlocks.getSize() )
140 {
141 iCurBlock = pStream->aBlocks[iPos/rMyriad.iBlockSize];
142 }
143 else
144 {
145 iCurBlock = rMyriad.streamAddBlock( pStream );
146#ifdef MYRIAD_STREAM_DEBUG
147 sio << "MyriadStream: write: " << __LINE__ << ": New block allocated and appended: "
148 << iCurBlock << "." << sio.nl;
149
150#endif
151 }
152 if( !pCurBlock || pCurBlock->iBlockIndex != iCurBlock )
153 {
154#ifdef MYRIAD_STREAM_DEBUG
155 sio << "MyriadStream: write: " << __LINE__ << ": Loading new block " << iCurBlock << "."
156 << sio.nl;
157#endif
158 rMyriad.releaseBlock( pCurBlock );
159 pCurBlock = rMyriad.getBlock( iCurBlock );
160 }
161 pCurBlock->bChanged = true;
162
163 // There are two main writing modes when it comes down to it.
164 // Overwrite mode and append mode. Append is what pretty much always
165 // happens when creating a new stream.
166 if( iPos < pStream->iSize )
167 {
168 int iAmnt = Bu::min(
169 Bu::min(
170 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
171 iLeft
172 ),
173 pStream->iSize-iPos
174 );
175#ifdef MYRIAD_STREAM_DEBUG
176 sio << "MyriadStream: write (ovr): " << __LINE__ << ": Copying in bytes: "
177 << (iPos%rMyriad.iBlockSize) << "+"
178 << iAmnt
179 << ", " << iLeft << "b left." << sio.nl;
180#endif
181 memcpy(
182 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
183 pBuf,
184 iAmnt
185 );
186 iPos += iAmnt;
187 pBuf = &((char *)pBuf)[iAmnt];
188 iLeft -= iAmnt;
189 }
190 else
191 {
192 int iAmnt = Bu::min(
193 rMyriad.iBlockSize - iPos%rMyriad.iBlockSize,
194 iLeft
195 );
196#ifdef MYRIAD_STREAM_DEBUG
197 sio << "MyriadStream: write (app): " << __LINE__ << ": Copying in bytes: "
198 << (iPos%rMyriad.iBlockSize) << "+"
199 << iAmnt
200 << ", " << iLeft << "b left." << sio.nl;
201#endif
202 memcpy(
203 pCurBlock->pData+(iPos%rMyriad.iBlockSize),
204 pBuf,
205 iAmnt
206 );
207 iPos += iAmnt;
208 pStream->iSize += iAmnt;
209 rMyriad.headerChanged();
210 pBuf = &((char *)pBuf)[iAmnt];
211 iLeft -= iAmnt;
212 }
213 }
214
215 return nBytes;
216}
217
218Bu::size Bu::MyriadStream::tell()
219{
220 return iPos;
221}
222
223void Bu::MyriadStream::seek( Bu::size offset )
224{
225 iPos += offset;
226}
227
228void Bu::MyriadStream::setPos( Bu::size pos )
229{
230 iPos = pos;
231}
232
233void Bu::MyriadStream::setPosEnd( Bu::size pos )
234{
235 iPos = pStream->iSize-pos;
236}
237
238bool Bu::MyriadStream::isEos()
239{
240 return iPos >= pStream->iSize;
241}
242
243bool Bu::MyriadStream::isOpen()
244{
245 return true;
246}
247
248void Bu::MyriadStream::flush()
249{
250}
251
252bool Bu::MyriadStream::canRead()
253{
254 return true;
255}
256
257bool Bu::MyriadStream::canWrite()
258{
259 return true;
260}
261
262bool Bu::MyriadStream::isReadable()
263{
264 return true;
265}
266
267bool Bu::MyriadStream::isWritable()
268{
269 return true;
270}
271
272bool Bu::MyriadStream::isSeekable()
273{
274 return true;
275}
276
277bool Bu::MyriadStream::isBlocking()
278{
279 return true;
280}
281
282void Bu::MyriadStream::setBlocking( bool /*bBlocking*/ )
283{
284}
285
286void Bu::MyriadStream::setSize( Bu::size iSize )
287{
288 if( iSize < 0 )
289 iSize = 0;
290 rMyriad.setStreamSize( pStream, iSize );
291 if( iPos > iSize )
292 iPos = iSize;
293}
294
295Bu::size Bu::MyriadStream::getSize() const
296{
297 return pStream->iSize;
298}
299
300Bu::size Bu::MyriadStream::getBlockSize() const
301{
302 return rMyriad.getBlockSize();
303}
304
305Bu::String Bu::MyriadStream::getLocation() const
306{
307 return Bu::String("%1").arg( pStream->iId );
308}
309
diff --git a/src/stable/myriadstream.h b/src/stable/myriadstream.h
new file mode 100644
index 0000000..fdad669
--- /dev/null
+++ b/src/stable/myriadstream.h
@@ -0,0 +1,61 @@
1/*
2 * Copyright (C) 2007-2011 Xagasoft, All rights reserved.
3 *
4 * This file is part of the libbu++ library and is released under the
5 * terms of the license contained in the file LICENSE.
6 */
7
8#ifndef BU_MYRIAD_STREAM_H
9#define BU_MYRIAD_STREAM_H
10
11#include "bu/stream.h"
12#include "bu/myriad.h"
13
14namespace Bu
15{
16 class MyriadStream : public Bu::Stream
17 {
18 friend class Myriad;
19 private:
20 /**
21 * These can only be created by the Myriad class.
22 */
23 MyriadStream( Myriad &rMyriad, Myriad::Stream *pStream );
24
25 public:
26 virtual ~MyriadStream();
27
28 virtual void close();
29 virtual Bu::size read( void *pBuf, Bu::size nBytes );
30 virtual Bu::size write( const void *pBuf, Bu::size nBytes );
31 using Stream::write;
32 virtual Bu::size tell();
33 virtual void seek( Bu::size offset );
34 virtual void setPos( Bu::size pos );
35 virtual void setPosEnd( Bu::size pos );
36 virtual bool isEos();
37 virtual bool isOpen();
38 virtual void flush();
39 virtual bool canRead();
40 virtual bool canWrite();
41 virtual bool isReadable();
42 virtual bool isWritable();
43 virtual bool isSeekable();
44 virtual bool isBlocking();
45 virtual void setBlocking( bool bBlocking=true );
46 virtual void setSize( Bu::size iSize );
47
48 virtual size getSize() const;
49 virtual size getBlockSize() const;
50 virtual Bu::String getLocation() const;
51
52 private:
53 Myriad &rMyriad;
54 Myriad::Stream *pStream;
55 Myriad::Block *pCurBlock;
56 int iBlockSize;
57 int iPos;
58 };
59};
60
61#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-27 10:13:36 +0000