aboutsummaryrefslogtreecommitdiff
path: root/src/stable/myriad.cpp
diff options
context:
space:
mode:
authorMike Buland <eichlan@xagasoft.com>2012-06-15 13:27:59 +0000
committerMike Buland <eichlan@xagasoft.com>2012-06-15 13:27:59 +0000
commit7ea5c06059ee6668d6e6d04c3b3dcb8557061696 (patch)
tree9ae2f93d09346c0272240015443e1a2bc660dd68 /src/stable/myriad.cpp
parent026cd6f71a15edd1a242c59d5cb9f8271a108506 (diff)
downloadlibbu++-7ea5c06059ee6668d6e6d04c3b3dcb8557061696.tar.gz
libbu++-7ea5c06059ee6668d6e6d04c3b3dcb8557061696.tar.bz2
libbu++-7ea5c06059ee6668d6e6d04c3b3dcb8557061696.tar.xz
libbu++-7ea5c06059ee6668d6e6d04c3b3dcb8557061696.zip
Bu::Myriad now only uses BitString during initialization, and I'm going to
replace that with just an array, no problem. It's many, many, many times faster while streams are growing, and it should be constant time, not linear like it was before. It also handles myriad files in excess of 2gb correctly now, at least, it seems to just fine :)
Diffstat (limited to 'src/stable/myriad.cpp')
-rw-r--r--src/stable/myriad.cpp109
1 files changed, 62 insertions, 47 deletions
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 7246df3..4f65583 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -84,8 +84,9 @@ void Bu::Myriad::sync()
84void Bu::Myriad::initialize() 84void Bu::Myriad::initialize()
85{ 85{
86 MutexLocker mLock( mHeader ); 86 MutexLocker mLock( mHeader );
87 lFreeBlocks.clear();
87 sStore.setPosEnd( 0 ); 88 sStore.setPosEnd( 0 );
88 int iSize = sStore.tell(); 89 Bu::size iSize = sStore.tell();
89 sStore.setPos( 0 ); 90 sStore.setPos( 0 );
90 91
91 unsigned char buf[4]; 92 unsigned char buf[4];
@@ -131,8 +132,11 @@ void Bu::Myriad::initialize()
131 { 132 {
132 pFakeHdr->aBlocks.append( j ); 133 pFakeHdr->aBlocks.append( j );
133 } 134 }
134 135
135 bsBlockUsed.setSize( iBlocks, true ); 136// sio << "Blocks: " << iBlocks << " (size = " << iSize << "/" << iBlockSize
137// << ")" << sio.nl;
138 Bu::BitString bsBlockUsed( iBlocks, false );
139 bsBlockUsed.clear();
136 140
137// bool bCanSkip = false; // Can skip around, post initial header stream i/o 141// bool bCanSkip = false; // Can skip around, post initial header stream i/o
138 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr ); 142 MyriadStream *pIn = new MyriadStream( *this, pFakeHdr );
@@ -174,12 +178,21 @@ void Bu::Myriad::initialize()
174 } 178 }
175 delete pIn; 179 delete pIn;
176 180
177 //sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl; 181 for( int j = 0; j < iBlocks; j++ )
182 {
183 if( bsBlockUsed.getBit( j ) == false )
184 {
185// sio << "Preinitialized block " << j << " is free." << sio.nl;
186 lFreeBlocks.append( j );
187 }
188 }
189// sio << "Myriad: Blocks used: " << bsBlockUsed.toString() << sio.nl;
178} 190}
179 191
180void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate ) 192void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
181{ 193{
182 MutexLocker mLock( mHeader ); 194 MutexLocker mLock( mHeader );
195 lFreeBlocks.clear();
183 196
184 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 197 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
185 { 198 {
@@ -207,7 +220,7 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
207 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize=" 220 //sio << "Myriad: iHeaderSize=" << iHeaderSize << ", iBlockSize="
208 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl; 221 // << iBlockSize << ", iHeaderBlocks=" << iHeaderBlocks << sio.nl;
209 222
210 bsBlockUsed.setSize( iPreAllocate, true ); 223// bsBlockUsed.setSize( iPreAllocate, true );
211 iUsed++; 224 iUsed++;
212 225
213 char *pBlock = new char[iBlockSize]; 226 char *pBlock = new char[iBlockSize];
@@ -255,10 +268,16 @@ void Bu::Myriad::initialize( int iBlockSize, int iPreAllocate )
255 pStr->iSize = iHeaderSize; 268 pStr->iSize = iHeaderSize;
256 for( int j = 0; j < iHeaderBlocks; j++ ) 269 for( int j = 0; j < iHeaderBlocks; j++ )
257 { 270 {
271// sio << "Started block " << j << " is header." << sio.nl;
258 pStr->aBlocks.append( j ); 272 pStr->aBlocks.append( j );
259 bsBlockUsed.setBit( j ); 273// bsBlockUsed.setBit( j );
260 iUsed++; 274 iUsed++;
261 } 275 }
276 for( int j = iHeaderBlocks; j < this->iBlocks; j++ )
277 {
278// sio << "Started block " << j << " is free." << sio.nl;
279 lFreeBlocks.append( j );
280 }
262 281
263 aStreams.append( pStr ); 282 aStreams.append( pStr );
264 283
@@ -302,7 +321,7 @@ void Bu::Myriad::updateHeader()
302// sio << "Myriad: updateHeader: Appending block " << iBlock 321// sio << "Myriad: updateHeader: Appending block " << iBlock
303// << " to header." << sio.nl; 322// << " to header." << sio.nl;
304 aStreams[0]->aBlocks.append( iBlock ); 323 aStreams[0]->aBlocks.append( iBlock );
305 bsBlockUsed.setBit( iBlock ); 324// bsBlockUsed.setBit( iBlock );
306 iUsed++; 325 iUsed++;
307 iHeaderSize += 4; 326 iHeaderSize += 4;
308 iNewBlocks = blkDiv( iHeaderSize, iBlockSize ); 327 iNewBlocks = blkDiv( iHeaderSize, iBlockSize );
@@ -361,7 +380,7 @@ int Bu::Myriad::createStream( int iPreAllocate )
361 int iFreeBlock = findEmptyBlock(); 380 int iFreeBlock = findEmptyBlock();
362// sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 381// sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
363 pStr->aBlocks.append( iFreeBlock ); 382 pStr->aBlocks.append( iFreeBlock );
364 bsBlockUsed.setBit( iFreeBlock ); 383// bsBlockUsed.setBit( iFreeBlock );
365 iUsed++; 384 iUsed++;
366 } 385 }
367 386
@@ -408,7 +427,7 @@ int Bu::Myriad::createStreamWithId( int iId, int iPreAllocate )
408 int iFreeBlock = findEmptyBlock(); 427 int iFreeBlock = findEmptyBlock();
409 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl; 428 // sio << "Myriad: Adding block " << iFreeBlock << sio.nl;
410 pStr->aBlocks.append( iFreeBlock ); 429 pStr->aBlocks.append( iFreeBlock );
411 bsBlockUsed.setBit( iFreeBlock ); 430// bsBlockUsed.setBit( iFreeBlock );
412 iUsed++; 431 iUsed++;
413 } 432 }
414 433
@@ -422,26 +441,15 @@ int Bu::Myriad::findEmptyBlock()
422{ 441{
423 bHeaderChanged = true; 442 bHeaderChanged = true;
424 443
425 for( int j = 0; j < bsBlockUsed.getSize(); j++ ) 444 if( lFreeBlocks.isEmpty() )
426 { 445 {
427 if( bsBlockUsed.getBit( j ) == false ) 446 sStore.setSize( (iBlocks+1)*(Bu::size)iBlockSize );
428 return j; 447 return iBlocks++;
448 }
449 else
450 {
451 return lFreeBlocks.dequeue();
429 } 452 }
430// sio << "Myriad: findEmptyBlock(): No empty blocks, adding new one." << sio.nl;
431
432 bsBlockUsed.setSize( bsBlockUsed.getSize()+1, false );
433 /*
434 sStore.setPos( iBlockSize*iBlocks );
435
436 char *pBlock = new char[iBlockSize];
437 memset( pBlock, 0, iBlockSize );
438 sStore.write( pBlock, iBlockSize );
439 delete[] pBlock;
440 */
441
442 sStore.setSize( (iBlocks+1)*iBlockSize );
443
444 return iBlocks++;
445} 453}
446 454
447void Bu::Myriad::deleteStream( int iId ) 455void Bu::Myriad::deleteStream( int iId )
@@ -461,7 +469,8 @@ void Bu::Myriad::deleteStream( int iId )
461 Stream *pStream = *i; 469 Stream *pStream = *i;
462 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ ) 470 for( BlockArray::iterator j = pStream->aBlocks.begin(); j; j++ )
463 { 471 {
464 bsBlockUsed.setBit( *j, false ); 472 lFreeBlocks.append( *j );
473// bsBlockUsed.setBit( *j, false );
465 iUsed--; 474 iUsed--;
466 } 475 }
467 aStreams.erase( i ); 476 aStreams.erase( i );
@@ -536,11 +545,11 @@ int Bu::Myriad::getNumUsedBlocks()
536 return iUsed; 545 return iUsed;
537} 546}
538 547
539int Bu::Myriad::getTotalUsedBytes() 548Bu::size Bu::Myriad::getTotalUsedBytes()
540{ 549{
541 MutexLocker mLock( mHeader ); 550 MutexLocker mLock( mHeader );
542 551
543 int iTotalSize = 0; 552 Bu::size iTotalSize = 0;
544 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 553 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
545 { 554 {
546 iTotalSize += (*i)->iSize; 555 iTotalSize += (*i)->iSize;
@@ -548,23 +557,23 @@ int Bu::Myriad::getTotalUsedBytes()
548 return iTotalSize; 557 return iTotalSize;
549} 558}
550 559
551int Bu::Myriad::getTotalUnusedBytes() 560Bu::size Bu::Myriad::getTotalUnusedBytes()
552{ 561{
553 MutexLocker mLock( mHeader ); 562 MutexLocker mLock( mHeader );
554 563
555 int iTotalSize = (iBlocks-iUsed)*iBlockSize; 564 Bu::size iTotalSize = (iBlocks-iUsed)*iBlockSize;
556 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 565 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
557 { 566 {
558 iTotalSize += iBlockSize - ((*i)->iSize%iBlockSize); 567 iTotalSize += iBlockSize - ((Bu::size)(*i)->iSize%iBlockSize);
559 } 568 }
560 return iTotalSize; 569 return iTotalSize;
561} 570}
562 571
563int Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize ) 572Bu::size Bu::Myriad::getTotalUnusedBytes( int iFakeBlockSize )
564{ 573{
565 MutexLocker mLock( mHeader ); 574 MutexLocker mLock( mHeader );
566 575
567 int iTotalSize = (iBlocks-iUsed)*iFakeBlockSize; 576 Bu::size iTotalSize = (iBlocks-iUsed)*iFakeBlockSize;
568 for( StreamArray::iterator i = aStreams.begin(); i; i++ ) 577 for( StreamArray::iterator i = aStreams.begin(); i; i++ )
569 { 578 {
570 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize); 579 iTotalSize += iFakeBlockSize - ((*i)->iSize%iFakeBlockSize);
@@ -592,7 +601,7 @@ Bu::Myriad::Block *Bu::Myriad::getBlock( int iBlock )
592// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl; 601// << iBlockSize*iBlock << "-" << iBlockSize*(iBlock+1) << sio.nl;
593 Block *pBlock = new Block; 602 Block *pBlock = new Block;
594 pBlock->pData = new char[iBlockSize]; 603 pBlock->pData = new char[iBlockSize];
595 sStore.setPos( iBlockSize * iBlock ); 604 sStore.setPos( iBlockSize * (Bu::size)iBlock );
596 sStore.read( pBlock->pData, iBlockSize ); 605 sStore.read( pBlock->pData, iBlockSize );
597 pBlock->bChanged = false; 606 pBlock->bChanged = false;
598 pBlock->iBlockIndex = iBlock; 607 pBlock->iBlockIndex = iBlock;
@@ -623,7 +632,7 @@ void Bu::Myriad::syncBlock( Block *pBlock )
623 if( pBlock->bChanged ) 632 if( pBlock->bChanged )
624 { 633 {
625// sio << "Myriad: - Block changed, writing back to stream." << sio.nl; 634// sio << "Myriad: - Block changed, writing back to stream." << sio.nl;
626 sStore.setPos( iBlockSize * pBlock->iBlockIndex ); 635 sStore.setPos( iBlockSize * (Bu::size)pBlock->iBlockIndex );
627 sStore.write( pBlock->pData, iBlockSize ); 636 sStore.write( pBlock->pData, iBlockSize );
628 pBlock->bChanged = false; 637 pBlock->bChanged = false;
629 } 638 }
@@ -635,8 +644,8 @@ int Bu::Myriad::streamAddBlock( Stream *pStream )
635 644
636 int iBlock = findEmptyBlock(); 645 int iBlock = findEmptyBlock();
637 pStream->aBlocks.append( iBlock ); 646 pStream->aBlocks.append( iBlock );
638 bsBlockUsed.setBit( iBlock ); 647// bsBlockUsed.setBit( iBlock );
639 bHeaderChanged = true; 648// bHeaderChanged = true;
640 iUsed++; 649 iUsed++;
641 return iBlock; 650 return iBlock;
642} 651}
@@ -655,11 +664,12 @@ void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
655 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize; 664 for( int iNewSize = pStream->aBlocks.getSize()*iBlockSize;
656 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize ) 665 iNewSize-iBlockSize > iSize; iNewSize -= iBlockSize )
657 { 666 {
658 if( bsBlockUsed.getBit( pStream->aBlocks.last() ) ) 667// if( bsBlockUsed.getBit( pStream->aBlocks.last() ) )
659 iUsed--; 668 iUsed--;
660 else 669// else
661 sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl; 670// sio << "Unused block used in stream? " << pStream->aBlocks.last() << sio.nl;
662 bsBlockUsed.setBit( pStream->aBlocks.last(), false ); 671 lFreeBlocks.enqueue( pStream->aBlocks.last() );
672// bsBlockUsed.setBit( pStream->aBlocks.last(), false );
663 pStream->aBlocks.eraseLast(); 673 pStream->aBlocks.eraseLast();
664 } 674 }
665 pStream->iSize = iSize; 675 pStream->iSize = iSize;
@@ -674,8 +684,8 @@ void Bu::Myriad::setStreamSize( Stream *pStream, long iSize )
674 //streamAddBlock( pStream ); 684 //streamAddBlock( pStream );
675 int iBlock = findEmptyBlock(); 685 int iBlock = findEmptyBlock();
676 pStream->aBlocks.append( iBlock ); 686 pStream->aBlocks.append( iBlock );
677 bsBlockUsed.setBit( iBlock ); 687// bsBlockUsed.setBit( iBlock );
678 bHeaderChanged = true; 688// bHeaderChanged = true;
679 iUsed++; 689 iUsed++;
680 } 690 }
681 pStream->iSize = iSize; 691 pStream->iSize = iSize;
@@ -712,8 +722,13 @@ bool Bu::Myriad::isMyriad( Bu::Stream &sStore, uint8_t &uTmp )
712 return true; 722 return true;
713} 723}
714 724
715const Bu::BitString &Bu::Myriad::getBlocksUsed() const 725const Bu::BitString Bu::Myriad::getBlocksUsed() const
716{ 726{
717 return bsBlockUsed; 727 Bu::BitString bs( iBlocks, false );
728 for( int j = 0; j < iBlocks; j++ )
729 bs.setBit( j );
730 for( IndexList::const_iterator i = lFreeBlocks.begin(); i; i++ )
731 bs.setBit( *i, false );
732 return bs;
718} 733}
719 734