aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/stable/file.cpp14
-rw-r--r--src/stable/myriad.cpp109
-rw-r--r--src/stable/myriad.h13
-rw-r--r--src/tests/bigmyriad.cpp25
4 files changed, 108 insertions, 53 deletions
diff --git a/src/stable/file.cpp b/src/stable/file.cpp
index 33420f1..14e4f3c 100644
--- a/src/stable/file.cpp
+++ b/src/stable/file.cpp
@@ -91,7 +91,11 @@ Bu::size Bu::File::tell()
91 if( fd < 0 ) 91 if( fd < 0 )
92 throw FileException("File not open."); 92 throw FileException("File not open.");
93 93
94#ifdef USE_64BIT_IO
95 return lseek64( fd, 0, SEEK_CUR );
96#else
94 return lseek( fd, 0, SEEK_CUR ); 97 return lseek( fd, 0, SEEK_CUR );
98#endif
95} 99}
96 100
97void Bu::File::seek( Bu::size offset ) 101void Bu::File::seek( Bu::size offset )
@@ -99,7 +103,11 @@ void Bu::File::seek( Bu::size offset )
99 if( fd < 0 ) 103 if( fd < 0 )
100 throw FileException("File not open."); 104 throw FileException("File not open.");
101 105
106#ifdef USE_64BIT_IO
107 lseek64( fd, offset, SEEK_CUR );
108#else
102 lseek( fd, offset, SEEK_CUR ); 109 lseek( fd, offset, SEEK_CUR );
110#endif
103 bEos = false; 111 bEos = false;
104} 112}
105 113
@@ -108,7 +116,11 @@ void Bu::File::setPos( Bu::size pos )
108 if( fd < 0 ) 116 if( fd < 0 )
109 throw FileException("File not open."); 117 throw FileException("File not open.");
110 118
119#ifdef USE_64BIT_IO
120 lseek64( fd, pos, SEEK_SET );
121#else
111 lseek( fd, pos, SEEK_SET ); 122 lseek( fd, pos, SEEK_SET );
123#endif
112 bEos = false; 124 bEos = false;
113} 125}
114 126
@@ -117,7 +129,7 @@ void Bu::File::setPosEnd( Bu::size pos )
117 if( fd < 0 ) 129 if( fd < 0 )
118 throw FileException("File not open."); 130 throw FileException("File not open.");
119 131
120 lseek( fd, pos, SEEK_END ); 132 lseek64( fd, pos, SEEK_END );
121 bEos = false; 133 bEos = false;
122} 134}
123 135
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 7246df3..4f65583 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -84,8 +84,9 @@ void Bu::Myriad::sync()
84void Bu::Myriad::initialize() 84void Bu::Myriad::initialize()
85{ 85{
86 MutexLocker mLock( mHeader ); 86 MutexLocker mLock( mHeader );
87 lFreeBlocks.clear();
87 sStore.setPosEnd( 0 ); 88 sStore.setPosEnd( 0 );
88 int iSize = sStore.tell(); 89 Bu::size iSize = sStore.tell();
89 sStore.setPos( 0 ); 90 sStore.setPos( 0 );
90 91
91 unsigned char buf[4]; 92 unsigned char buf[4];
@@ -131,8 +132,11 @@ void Bu::Myriad::initialize()
131 { 132 {
132 pFakeHdr->aBlocks.append( j ); 133 pFakeHdr->aBlocks.append( j );
133 } 134 }
134 135
135 bsBlockUsed.setSize( iBlocks, true ); 136// sio << "Blocks: " << iBlocks << " (size = " << iSize << "/" << iBlockSize
137// << ")" << sio.nl;
138 Bu::BitString bsBlockUsed( iBlocks, false );
139 bsBlockUsed.clear();
136 140
137// bool bCanSkip = false; // Can skip around, post initial header stream i/o 141// bool bCanSkip = false; // Can skip around, post initial header stream i/o
138 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr ); 142 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
@@ -174,12 +178,21 @@ void Bu::Myriad::initialize()
174 } 178 }
175 delete pIn; 179 delete pIn;
176 180
177 //sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; 181 for( int j = 0; j < iBlocks; j++ )
182 {
183 if( bsBlockUsed.getBit( j ) == false )
184 {
185// sio << "Preinitialized block " << j << " is free." << sio.nl;
186 lFreeBlocks.append( j );
187 }
188 }
189// sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl;
178} 190}
179 191
180void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) 192void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
181{ 193{
182 MutexLocker mLock( mHeader ); 194 MutexLocker mLock( mHeader );
195 lFreeBlocks.clear();
183 196
184 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 197 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
185 { 198 {
@@ -207,7 +220,7 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
207 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" 220 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize="
208 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; 221 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
209 222
210 bsBlockUsed.setSize( iPreAllocate, true ); 223// bsBlockUsed.setSize( iPreAllocate, true );
211 iUsed++; 224 iUsed++;
212 225
213 char *pBlock = new char[iBlockSize]; 226 char *pBlock = new char[iBlockSize];
@@ -255,10 +268,16 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
255 pStr->iSize = iHeaderSize; 268 pStr->iSize = iHeaderSize;
256 for( int j = 0; j < iHeaderBlocks; j++ ) 269 for( int j = 0; j < iHeaderBlocks; j++ )
257 { 270 {
271// sio << "Started block " << j << " is header." << sio.nl;
258 pStr->aBlocks.append( j ); 272 pStr->aBlocks.append( j );
259 bsBlockUsed.setBit( j ); 273// bsBlockUsed.setBit( j );
260 iUsed++; 274 iUsed++;
261 } 275 }
276 for( int j = iHeaderBlocks; j < this->iBlocks; j++ )
277 {
278// sio << "Started block " << j << " is free." << sio.nl;
279 lFreeBlocks.append( j );
280 }
262 281
263 aStreams.append( pStr ); 282 aStreams.append( pStr );
264 283
@@ -302,7 +321,7 @@ void Bu::Myriad::updateHeader()
302// sio << "Myriad: updateHeader: Appending block " << iBlock 321// sio << "Myriad: updateHeader: Appending block " << iBlock
303// << " to header." << sio.nl; 322// << " to header." << sio.nl;
304 aStreams[0]->aBlocks.append( iBlock ); 323 aStreams[0]->aBlocks.append( iBlock );
305 bsBlockUsed.setBit( iBlock ); 324// bsBlockUsed.setBit( iBlock );
306 iUsed++; 325 iUsed++;
307 iHeaderSize += 4; 326 iHeaderSize += 4;
308 iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); 327 iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
@@ -361,7 +380,7 @@ int Bu::Myriad::createStream( int iPreAllocate )
361 int iFreeBlock = findEmptyBlock(); 380 int iFreeBlock = findEmptyBlock();
362// sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 381// sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
363 pStr->aBlocks.append( iFreeBlock ); 382 pStr->aBlocks.append( iFreeBlock );
364 bsBlockUsed.setBit( iFreeBlock ); 383// bsBlockUsed.setBit( iFreeBlock );
365 iUsed++; 384 iUsed++;
366 } 385 }
367 386
@@ -408,7 +427,7 @@ int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate )
408 int iFreeBlock = findEmptyBlock(); 427 int iFreeBlock = findEmptyBlock();
409 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 428 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
410 pStr->aBlocks.append( iFreeBlock ); 429 pStr->aBlocks.append( iFreeBlock );
411 bsBlockUsed.setBit( iFreeBlock ); 430// bsBlockUsed.setBit( iFreeBlock );
412 iUsed++; 431 iUsed++;
413 } 432 }
414 433
@@ -422,26 +441,15 @@ int Bu::Myriad::findEmptyBlock()
422{ 441{
423 bHeaderChanged = true; 442 bHeaderChanged = true;
424 443
425 for( int j = 0; j < bsBlockUsed.getSize(); j++ ) 444 if( lFreeBlocks.isEmpty() )
426 { 445 {
427 if( bsBlockUsed.getBit( j ) == false ) 446 sStore.setSize( (iBlocks+1)*(Bu::size)iBlockSize );
428 return j; 447 return iBlocks++;
448 }
449 else
450 {
451 return lFreeBlocks.dequeue();
429 } 452 }
430// sio << "Myriad: findEmptyBlock(): No empty blocks, adding new one." << sio.nl;
431
432 bsBlockUsed.setSize( bsBlockUsed.getSize()+1, false );
433 /*
434 sStore.setPos( iBlockSize*iBlocks );
435
436 char *pBlock = new char[iBlockSize];
437 memset( pBlock, 0, iBlockSize );
438 sStore.write( pBlock, iBlockSize );
439 delete[] pBlock;
440 */
441
442 sStore.setSize( (iBlocks+1)*iBlockSize );
443
444 return iBlocks++;
445} 453}
446 454
447void Bu::Myriad::deleteStream( int iId ) 455void Bu::Myriad::deleteStream( int iId )
@@ -461,7 +469,8 @@ void Bu::Myriad::deleteStream( int iId )
461 Stream *pStream = *i; 469 Stream *pStream = *i;
462 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ ) 470 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
463 { 471 {
464 bsBlockUsed.setBit( *j, false ); 472 lFreeBlocks.append( *j );
473// bsBlockUsed.setBit( *j, false );
465 iUsed--; 474 iUsed--;
466 } 475 }
467 aStreams.erase( i ); 476 aStreams.erase( i );
@@ -536,11 +545,11 @@ int Bu::Myriad::getNumUsedBlocks()
536 return iUsed; 545 return iUsed;
537} 546}
538 547
539int Bu::Myriad::getTotalUsedBytes() 548Bu::size Bu::Myriad::getTotalUsedBytes()
540{ 549{
541 MutexLocker mLock( mHeader ); 550 MutexLocker mLock( mHeader );
542 551
543 int iTotalSize = 0; 552 Bu::size iTotalSize = 0;
544 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 553 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
545 { 554 {
546 iTotalSize += (*i)->iSize; 555 iTotalSize += (*i)->iSize;
@@ -548,23 +557,23 @@ int Bu::Myriad::getTotalUsedBytes()
548 return iTotalSize; 557 return iTotalSize;
549} 558}
550 559
551int Bu::Myriad::getTotalUnusedBytes() 560Bu::size Bu::Myriad::getTotalUnusedBytes()
552{ 561{
553 MutexLocker mLock( mHeader ); 562 MutexLocker mLock( mHeader );
554 563
555 int iTotalSize = (iBlocks-iUsed)*iBlockSize; 564 Bu::size iTotalSize = (iBlocks-iUsed)*iBlockSize;
556 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 565 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
557 { 566 {
558 iTotalSize += iBlockSize - ((*i)->iSize%iBlockSize); 567 iTotalSize += iBlockSize - ((Bu::size)(*i)->iSize%iBlockSize);
559 } 568 }
560 return iTotalSize; 569 return iTotalSize;
561} 570}
562 571
563int Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize ) 572Bu::size Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize )
564{ 573{
565 MutexLocker mLock( mHeader ); 574 MutexLocker mLock( mHeader );
566 575
567 int iTotalSize = (iBlocks-iUsed)*iFakeBlockSize; 576 Bu::size iTotalSize = (iBlocks-iUsed)*iFakeBlockSize;
568 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 577 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
569 { 578 {
570 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize); 579 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize);
@@ -592,7 +601,7 @@ Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock )
592// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl; 601// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl;
593 Block *pBlock = new Block; 602 Block *pBlock = new Block;
594 pBlock->pData = new char[iBlockSize]; 603 pBlock->pData = new char[iBlockSize];
595 sStore.setPos( iBlockSize * iBlock ); 604 sStore.setPos( iBlockSize * (Bu::size)iBlock );
596 sStore.read( pBlock->pData, iBlockSize ); 605 sStore.read( pBlock->pData, iBlockSize );
597 pBlock->bChanged = false; 606 pBlock->bChanged = false;
598 pBlock->iBlockIndex = iBlock; 607 pBlock->iBlockIndex = iBlock;
@@ -623,7 +632,7 @@ void Bu::Myriad::syncBlock( Block *pBlock )
623 if( pBlock->bChanged ) 632 if( pBlock->bChanged )
624 { 633 {
625// sio << "Myriad: - Block changed, writing back to stream." << sio.nl; 634// sio << "Myriad: - Block changed, writing back to stream." << sio.nl;
626 sStore.setPos( iBlockSize * pBlock->iBlockIndex ); 635 sStore.setPos( iBlockSize * (Bu::size)pBlock->iBlockIndex );
627 sStore.write( pBlock->pData, iBlockSize ); 636 sStore.write( pBlock->pData, iBlockSize );
628 pBlock->bChanged = false; 637 pBlock->bChanged = false;
629 } 638 }
@@ -635,8 +644,8 @@ int Bu::Myriad::streamAddBlock( Stream *pStream )
635 644
636 int iBlock = findEmptyBlock(); 645 int iBlock = findEmptyBlock();
637 pStream->aBlocks.append( iBlock ); 646 pStream->aBlocks.append( iBlock );
638 bsBlockUsed.setBit( iBlock ); 647// bsBlockUsed.setBit( iBlock );
639 bHeaderChanged = true; 648// bHeaderChanged = true;
640 iUsed++; 649 iUsed++;
641 return iBlock; 650 return iBlock;
642} 651}
@@ -655,11 +664,12 @@ void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
655 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; 664 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
656 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize ) 665 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize )
657 { 666 {
658 if( bsBlockUsed.getBit( pStream->aBlocks.last() ) ) 667// if( bsBlockUsed.getBit( pStream->aBlocks.last() ) )
659 iUsed--; 668 iUsed--;
660 else 669// else
661 sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl; 670// sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl;
662 bsBlockUsed.setBit( pStream->aBlocks.last(), false ); 671 lFreeBlocks.enqueue( pStream->aBlocks.last() );
672// bsBlockUsed.setBit( pStream->aBlocks.last(), false );
663 pStream->aBlocks.eraseLast(); 673 pStream->aBlocks.eraseLast();
664 } 674 }
665 pStream->iSize = iSize; 675 pStream->iSize = iSize;
@@ -674,8 +684,8 @@ void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
674 //streamAddBlock( pStream ); 684 //streamAddBlock( pStream );
675 int iBlock = findEmptyBlock(); 685 int iBlock = findEmptyBlock();
676 pStream->aBlocks.append( iBlock ); 686 pStream->aBlocks.append( iBlock );
677 bsBlockUsed.setBit( iBlock ); 687// bsBlockUsed.setBit( iBlock );
678 bHeaderChanged = true; 688// bHeaderChanged = true;
679 iUsed++; 689 iUsed++;
680 } 690 }
681 pStream->iSize = iSize; 691 pStream->iSize = iSize;
@@ -712,8 +722,13 @@ bool Bu::Myriad::isMyriad( Bu::Stream &sStore, uint8_t &uTmp )
712 return true; 722 return true;
713} 723}
714 724
715const Bu::BitString &Bu::Myriad::getBlocksUsed() const 725const Bu::BitString Bu::Myriad::getBlocksUsed() const
716{ 726{
717 return bsBlockUsed; 727 Bu::BitString bs( iBlocks, false );
728 for( int j = 0; j < iBlocks; j++ )
729 bs.setBit( j );
730 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
731 bs.setBit( *i, false );
732 return bs;
718} 733}
719 734
diff --git a/src/stable/myriad.h b/src/stable/myriad.h
index 80b5c2b..6b1db41 100644
--- a/src/stable/myriad.h
+++ b/src/stable/myriad.h
@@ -14,6 +14,7 @@
14#include "bu/array.h" 14#include "bu/array.h"
15#include "bu/hash.h" 15#include "bu/hash.h"
16#include "bu/mutex.h" 16#include "bu/mutex.h"
17#include "bu/extratypes.h"
17 18
18namespace Bu 19namespace Bu
19{ 20{
@@ -139,9 +140,9 @@ namespace Bu
139 int getBlockSize(); 140 int getBlockSize();
140 int getNumBlocks(); 141 int getNumBlocks();
141 int getNumUsedBlocks(); 142 int getNumUsedBlocks();
142 int getTotalUsedBytes(); 143 Bu::size getTotalUsedBytes();
143 int getTotalUnusedBytes(); 144 Bu::size getTotalUnusedBytes();
144 int getTotalUnusedBytes( int iFakeBlockSize ); 145 Bu::size getTotalUnusedBytes( int iFakeBlockSize );
145 146
146 /** 147 /**
147 * Syncronize the header data, etc. with the storage stream. It's not 148 * Syncronize the header data, etc. with the storage stream. It's not
@@ -163,7 +164,7 @@ namespace Bu
163 */ 164 */
164 static bool isMyriad( Bu::Stream &sStore ); 165 static bool isMyriad( Bu::Stream &sStore );
165 166
166 const Bu::BitString &getBlocksUsed() const; 167 const Bu::BitString getBlocksUsed() const;
167 168
168 private: 169 private:
169 /** 170 /**
@@ -219,7 +220,9 @@ namespace Bu
219 int iBlockSize; 220 int iBlockSize;
220 int iBlocks; 221 int iBlocks;
221 int iUsed; 222 int iUsed;
222 Bu::BitString bsBlockUsed; 223 typedef Bu::List<int> IndexList;
224 IndexList lFreeBlocks;
225// Bu::BitString bsBlockUsed;
223 StreamArray aStreams; 226 StreamArray aStreams;
224 typedef Bu::Hash<int, Block *> BlockHash; 227 typedef Bu::Hash<int, Block *> BlockHash;
225 BlockHash hActiveBlocks; 228 BlockHash hActiveBlocks;
diff --git a/src/tests/bigmyriad.cpp b/src/tests/bigmyriad.cpp
new file mode 100644
index 0000000..6661bf1
--- /dev/null
+++ b/src/tests/bigmyriad.cpp
@@ -0,0 +1,25 @@
1#include <bu/myriad.h>
2#include <bu/myriadstream.h>
3#include <bu/file.h>
4
5int main()
6{
7 Bu::File f("big.myr", Bu::File::Read|Bu::File::Write|Bu::File::Create );
8 Bu::Myriad m( f, 2048 );
9
10 char *buf = new char[1024*1024*10];
11 memset( buf, 0, 1024*1024*10 );
12
13 for( int j = 0; j < 250; j++ )
14 {
15 m.openStream( m.createStream() ).write( buf, 1024*1024*10 );
16// m.sync();
17 printf("\r%03d%%", (j+1)*100/250 );
18 fflush( stdout );
19 }
20
21 printf("\n\n");
22
23 return 0;
24}
25