aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorMike Buland <mike@xagasoft.com>2024-10-01 11:21:13 -0700
committerMike Buland <mike@xagasoft.com>2024-10-01 11:21:13 -0700
commit00651aeaf50f8481a2c894f9462cd3b8eb6971d6 (patch)
tree72630f5c1918e8188b92e6f93f0122dbea8022d1 /src
parent373b8f42902686ea1181b1edd8c2c06555cad8fd (diff)
downloadlibbu++-00651aeaf50f8481a2c894f9462cd3b8eb6971d6.tar.gz
libbu++-00651aeaf50f8481a2c894f9462cd3b8eb6971d6.tar.bz2
libbu++-00651aeaf50f8481a2c894f9462cd3b8eb6971d6.tar.xz
libbu++-00651aeaf50f8481a2c894f9462cd3b8eb6971d6.zip
More myriad fixes, it passes all existing unit tests.
Diffstat (limited to '')
-rw-r--r--src/stable/myriad.cpp26
-rw-r--r--src/tests/myriad.cpp23
-rw-r--r--src/unit/myriad.unit385
3 files changed, 421 insertions, 13 deletions
diff --git a/src/stable/myriad.cpp b/src/stable/myriad.cpp
index 85daa0d..f9fcd3e 100644
--- a/src/stable/myriad.cpp
+++ b/src/stable/myriad.cpp
@@ -12,13 +12,13 @@
12#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \ 12#define MyriadRead( target, size ) if( rBacking.read( target, size ) < size ) \
13{ \ 13{ \
14 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ 14 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
15 "Insufficent data reading myriad data from backing stream."); \ 15 "Insufficient data reading myriad data from backing stream."); \
16} (void)0 16} (void)0
17 17
18#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \ 18#define ReqRead( stream, target, size ) if( stream.read( target, size ) < size ) \
19{ \ 19{ \
20 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \ 20 throw Bu::MyriadException( Bu::MyriadException::invalidFormat, \
21 "Insufficent data reading from myriad stream."); \ 21 "Insufficient data reading from myriad stream."); \
22} (void)0 22} (void)0
23 23
24namespace Bu 24namespace Bu
@@ -112,7 +112,7 @@ Bu::MyriadStream Bu::Myriad::open( Bu::Myriad::StreamId iStream,
112 } 112 }
113 { 113 {
114 Bu::MutexLocker l2( mBacking ); 114 Bu::MutexLocker l2( mBacking );
115 if( (eMode&Write) && rBacking.isWritable() ) 115 if( (eMode&Write) && !rBacking.isWritable() )
116 { 116 {
117 throw Bu::MyriadException( MyriadException::badMode, 117 throw Bu::MyriadException( MyriadException::badMode,
118 "Backing stream does not support writing."); 118 "Backing stream does not support writing.");
@@ -398,6 +398,8 @@ bool Bu::Myriad::loadMyriad()
398 398
399 lFreeBlocks = hUnusedBlocks.getKeys(); 399 lFreeBlocks = hUnusedBlocks.getKeys();
400 //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() ); 400 //Bu::println("Free blocks: %1").arg( lFreeBlocks.getSize() );
401
402 bIsNewStream = false;
401 403
402 return true; 404 return true;
403} 405}
@@ -434,14 +436,11 @@ void Bu::Myriad::createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks )
434 //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize ); 436 //Bu::println(" Adjustment: %1 bytes / %2 cur blocks, %3 computed blocks (%4 target bytes).").arg( iHeaderStreamBytes+(iHeaderStreamBlocks*4) ).arg( iHeaderStreamBlocks ).arg( blkDiv((iHeaderStreamBytes+(iHeaderStreamBlocks*4)), iBlockSize) ).arg( iHeaderStreamBlocks*iBlockSize );
435 } 437 }
436 438
437 if( iPreallocateBlocks > iHeaderStreamBlocks ) 439 if( iPreallocateBlocks < iHeaderStreamBlocks )
438 { 440 {
439 rBacking.setSize( iBlockSize*iPreallocateBlocks ); 441 iPreallocateBlocks = iHeaderStreamBlocks;
440 }
441 else
442 {
443 rBacking.setSize( iBlockSize*iHeaderStreamBlocks );
444 } 442 }
443 rBacking.setSize( iBlockSize*iPreallocateBlocks );
445 444
446 // 445 //
447 // Write Myriad header 446 // Write Myriad header
@@ -540,14 +539,15 @@ void Bu::Myriad::writeHeader()
540 uint32_t uStreamSize = pStream->getSize(); 539 uint32_t uStreamSize = pStream->getSize();
541 mbHeader.write( &uStreamId, 4 ); 540 mbHeader.write( &uStreamId, 4 );
542 mbHeader.write( &uStreamSize, 4 ); 541 mbHeader.write( &uStreamSize, 4 );
542 int32_t iBlocks = Bu::blkDiv( uStreamSize, (uint32_t)iBlockSize );
543 Bu::Array<int32_t> aBlocks = pStream->getBlockList(); 543 Bu::Array<int32_t> aBlocks = pStream->getBlockList();
544 544
545 //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) ); 545 //Bu::println(" Stream %1 is %2 bytes %3 blocks (%4 blocks computed)").arg( *i ).arg( uStreamSize ).arg( aBlocks.getSize() ).arg( Bu::blkDiv( (int)uStreamSize, (int)iBlockSize ) );
546 546
547 for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ ) 547// for( Bu::Array<int32_t>::iterator i = aBlocks.begin(); i; i++ )
548 for( int j = 0; j < iBlocks; j++ )
548 { 549 {
549 int32_t iIdx = *i; 550 mbHeader.write( &aBlocks[j], 4 );
550 mbHeader.write( &iIdx, 4 );
551 } 551 }
552 } 552 }
553 553
@@ -768,7 +768,7 @@ int32_t Bu::Myriad::Stream::read( int32_t iStart, void *pTarget,
768 768
769 if( iStart+iSize >= this->iSize ) 769 if( iStart+iSize >= this->iSize )
770 { 770 {
771 int32_t iDiff = this->iSize-(iStart+iSize); 771 int32_t iDiff = (iStart+iSize)-this->iSize;
772 iSize -= iDiff; 772 iSize -= iDiff;
773 iStart += iDiff; 773 iStart += iDiff;
774 } 774 }
diff --git a/src/tests/myriad.cpp b/src/tests/myriad.cpp
new file mode 100644
index 0000000..ee4eac4
--- /dev/null
+++ b/src/tests/myriad.cpp
@@ -0,0 +1,23 @@
1#include <bu/myriad.h>
2#include <bu/myriadstream.h>
3#include <bu/string.h>
4#include <bu/sio.h>
5#include <bu/file.h>
6
7int main( int , char *[] )
8{
9 Bu::File fMyriad("test.myr", Bu::File::WriteNew|Bu::File::Read );
10 Bu::Myriad m( fMyriad, 32 );
11
12 Bu::MyriadStream ms = m.create( Bu::Myriad::ReadWrite );
13 ms.setSize( 150 );
14 ms.setPos( 145 );
15 char stuff[10];
16 int32_t iRead = ms.read( stuff, 10 );
17
18 Bu::println("Tried to read 10, expect 5, got %1").arg( iRead );
19
20
21 return 0;
22}
23
diff --git a/src/unit/myriad.unit b/src/unit/myriad.unit
new file mode 100644
index 0000000..f7bea97
--- /dev/null
+++ b/src/unit/myriad.unit
@@ -0,0 +1,385 @@
1// vim: syntax=cpp
2/*
3 * Copyright (C) 2007-2023 Xagasoft, All rights reserved.
4 *
5 * This file is part of the libbu++ library and is released under the
6 * terms of the license contained in the file LICENSE.
7 */
8
9#include "bu/string.h"
10#include "bu/file.h"
11#include "bu/myriad.h"
12#include "bu/myriadstream.h"
13#include "bu/array.h"
14
15#include "bu/sio.h"
16#include "bu/archive.h"
17#include "bu/md5.h"
18#include "bu/unitsuite.h"
19
20#include <stdlib.h>
21
22using namespace Bu;
23
24class VerifyObject
25{
26friend Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo );
27friend Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo );
28public:
29 VerifyObject( int iUnits ) :
30 iUnits( iUnits ),
31 iBytesWritten( 0 )
32 {
33 }
34
35 virtual ~VerifyObject()
36 {
37 }
38
39 int getBytesWritten()
40 {
41 return iBytesWritten;
42 }
43
44private:
45 int iUnits;
46 mutable int iBytesWritten;
47};
48
49Bu::ArchiveBase &operator<<( Bu::ArchiveBase &ar, const VerifyObject &vo )
50{
51 Md5 sum;
52 ar << vo.iUnits;
53 vo.iBytesWritten = sizeof(int);
54 sum.addData( &vo.iUnits, sizeof(int) );
55 for( int j = 0; j < vo.iUnits; j++ )
56 {
57 int iRand = random()%128;
58// ar << iRand;
59 Bu::String sDat( iRand );
60 for( int j = 0; j < iRand; j++ )
61 sDat[j] = (char)((uint8_t)(random()%256));
62 ar << sDat;
63 sum.addData( &iRand, sizeof(int) );
64 sum.addData( sDat.getStr(), iRand );
65 vo.iBytesWritten += sizeof(long) + iRand;
66 }
67 Bu::String sRes = sum.getResult();
68 ar << sRes;
69 vo.iBytesWritten += sizeof(long) + sRes.getSize();
70 return ar;
71}
72
73Bu::ArchiveBase &operator>>( Bu::ArchiveBase &ar, VerifyObject &vo )
74{
75 Md5 sum;
76 ar >> vo.iUnits;
77 sum.addData( &vo.iUnits, sizeof(int) );
78 for( int j = 0; j < vo.iUnits; j++ )
79 {
80 int iRand;
81// ar >> iRand;
82 Bu::String sStr;
83 ar >> sStr;
84 iRand = sStr.getSize();
85 sum.addData( &iRand, sizeof(int) );
86 sum.addData( sStr.getStr(), iRand );
87 }
88 Bu::String sSum;
89 ar >> sSum;
90 unitTest( sSum == sum.getResult() );
91 int iTooMuch;
92 try
93 {
94 ar >> iTooMuch;
95 unitFailed("should have thrown an exception.");
96 }
97 catch( Bu::ExceptionBase &e )
98 {
99 }
100 return ar;
101}
102
103suite Myriad
104{
105 test setSize
106 {
107 String sFileName("myriad-XXXXXXX");
108
109 File fMyriad = tempFile( sFileName );
110 Myriad m( fMyriad, 32 );
111
112 MyriadStream ms = m.create( Myriad::ReadWrite );
113 ms.setSize( 150 );
114 ms.setPos( 145 );
115 char stuff[10];
116 unitTest( ms.read( stuff, 10 ) == 5 );
117
118 ms.setSize( 12 );
119 unitTest( ms.read( stuff, 10 ) == 0 );
120 unitTest( ms.write( "hello", 5 ) == 5 );
121 unitTest( ms.tell() == 17 );
122
123 ms.setSize( 500 );
124 unitTest( ms.tell() == 17 );
125 }
126
127 void addBlock( Stream &s, bool bAppend=true )
128 {
129 if( bAppend )
130 s.setPosEnd( 0 );
131 int iSize = (random()%1016)+8;
132 s.write( &iSize, 4 );
133 char *buf = new char[iSize-8];
134 for( int j = 0; j < iSize-8; j++ )
135 {
136 buf[j] = (j+iSize)%256;
137 }
138 if( random()%2 == 0 )
139 {
140 s.write( buf, iSize-8 );
141 }
142 else
143 {
144 for( int j = 0; j < iSize-8; )
145 {
146 int iAmnt = (random()%8)+1;
147 if( iAmnt+j > iSize-8 )
148 iAmnt = iSize-8-j;
149 iAmnt = s.write( buf+j, iAmnt );
150 j += iAmnt;
151 }
152 }
153 delete[] buf;
154 iSize = ~iSize;
155 s.write( &iSize, 4 );
156 }
157
158 void verifyBlock( Stream &s )
159 {
160 int iSize, iInv;
161 if( s.read( &iSize, 4 ) == 0 )
162 return;
163 if( iSize < 8 || iSize > 1024 )
164 throw ExceptionBase("Read bad data, %d", iSize );
165 char *buf = new char[iSize-8];
166 if( s.read( buf, iSize-8 ) < (Bu::size)iSize-8 )
167 {
168 delete[] buf;
169 throw ExceptionBase("Block failed verify (insuffient block data).");
170 }
171 for( int j = 0; j < iSize-8; j++ )
172 {
173 if( buf[j] != (char)((j+iSize)%256) )
174 {
175 char b = buf[j];
176 delete[] buf;
177 throw ExceptionBase("Block failed computed data verify "
178 "(%02X==%02X).", b, (char)((j+iSize)%256) );
179 }
180 }
181 delete[] buf;
182 if( s.read( &iInv, 4 ) < 4 )
183 throw ExceptionBase("Block failed verify (insufficient data).");
184 if( iInv != ~iSize )
185 throw ExceptionBase("Block failed inversion verify.");
186 }
187
188 void verifyStream( Stream &s )
189 {
190 s.setPos( 0 );
191 while( !s.isEos() )
192 verifyBlock( s );
193 }
194
195 test stressGrow
196 {
197 String sFileName("myriad-XXXXXXX");
198
199 File fMyriad = tempFile( sFileName );
200 Myriad m( fMyriad, 64 );
201
202 Array<int> aStreams;
203 for( int j = 0; j < 5; j++ )
204 {
205 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
206 }
207
208 srandom( 512 );
209
210 for( int j = 0; j < 2500; j++ )
211 {
212 switch( random()%5 )
213 {
214 case 0:
215 aStreams.append( m.create( Bu::Myriad::Read ).getId() );
216 break;
217
218 case 1:
219 if( aStreams.getSize() > 0 )
220 {
221 int iStream = random()%aStreams.getSize();
222 {
223 MyriadStream ms = m.open( aStreams[iStream], Myriad::Read );
224 verifyStream( ms );
225 }
226 m.erase( aStreams[iStream] );
227 Array<int>::iterator i = aStreams.begin();
228 for( int k = 0; k < iStream; k++ )
229 i++;
230 aStreams.erase( i );
231 }
232 break;
233
234 default:
235 if( aStreams.getSize() == 0 )
236 {
237 aStreams.append(
238 m.create( Bu::Myriad::Read ).getId()
239 );
240 }
241 {
242 int iStream = random()%aStreams.getSize();
243 MyriadStream ms = m.open( aStreams[iStream], Myriad::ReadWrite );
244 addBlock( ms );
245 verifyStream( ms );
246 }
247 break;
248 }
249 }
250
251 for( Array<int>::iterator i = aStreams.begin(); i; i++ )
252 {
253 MyriadStream ms = m.open( *i, Myriad::Read );
254 verifyStream( ms );
255 }
256 }
257
258 test stressTruncate
259 {
260 String sFileName("myriad-XXXXXXX");
261
262 File fMyriad = tempFile( sFileName );
263 Myriad m( fMyriad, 128 );
264
265 Array<int> aStream;
266
267 for( int j = 0; j < 5; j++ )
268 {
269 aStream.append( m.create( Bu::Myriad::Read ).getId() );
270 }
271
272 srandom( 1024 );
273
274 char b;
275 for( int iter = 0; iter < 2500; iter++ )
276 {
277 for( Array<int>::iterator i = aStream.begin(); i; i++ )
278 {
279 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
280 addBlock( ms, false );
281 ms.setSize( ms.tell() );
282 unitTest( ms.read( &b, 1 ) == 0 );
283 ms.setPos( 0 );
284 verifyBlock( ms );
285 unitTest( ms.read( &b, 1 ) == 0 );
286 }
287 }
288 }
289
290 test stressTruncate2
291 {
292 String sFileName("myriad-XXXXXXX");
293
294 Array<int> aStream;
295
296 setStepCount( 5*2500 + 5 );
297
298 {
299 File fMyriad = tempFile( sFileName );
300 Myriad m( fMyriad, 128 );
301
302 for( int j = 0; j < 5; j++ )
303 {
304 aStream.append( m.create( Bu::Myriad::Read ).getId() );
305 incProgress();
306 }
307 }
308
309 srandom( 1024 );
310
311 char b;
312 for( int iter = 0; iter < 2500; iter++ )
313 {
314 File fMyriad( sFileName, File::ReadWrite );
315 Myriad m( fMyriad );
316 for( Array<int>::iterator i = aStream.begin(); i; i++ )
317 {
318 MyriadStream ms = m.open( *i, Myriad::ReadWrite );
319 addBlock( ms, false );
320 ms.setSize( ms.tell() );
321 unitTest( ms.read( &b, 1 ) == 0 );
322 ms.setPos( 0 );
323 verifyBlock( ms );
324 unitTest( ms.read( &b, 1 ) == 0 );
325 incProgress();
326 }
327 }
328 }
329
330 test stressArchive
331 {
332 String sFileName("myriad-XXXXXX");
333 Array<int> aStream;
334
335 srandom( 2096 );
336
337 setStepCount( 15*250 + 15 );
338
339 {
340 File fMyriad = tempFile( sFileName );
341 Myriad m( fMyriad, 1024 );
342
343 for( int j = 0; j < 15; j++ )
344 {
345 MyriadStream ms = m.create( Myriad::Write );
346 int iStream = ms.getId();
347 aStream.append( iStream );
348 VerifyObject vo( random()%1024 );
349 {
350 Archive ar( ms, Archive::save );
351 ar << vo;
352 unitTest( ms.tell() == vo.getBytesWritten() );
353 ms.setSize( ms.tell() );
354 }
355 unitTest( m.getSize( iStream ) == vo.getBytesWritten() );
356 incProgress();
357 }
358 }
359
360 for( int iter = 0; iter < 250; iter++ )
361 {
362 File fMyriad( sFileName, File::ReadWrite );
363 Myriad m( fMyriad );
364 for( Array<int>::iterator i = aStream.begin(); i; i++ )
365 {
366 VerifyObject vo( random()%1024 );
367 {
368 MyriadStream ms = m.open( *i, Myriad::Read );
369 Archive ar( ms, Archive::load );
370 ar >> vo;
371 }
372 {
373 MyriadStream ms = m.open( *i, Myriad::WriteNew );
374 Archive ar( ms, Archive::save );
375 ar << vo;
376 unitTest( ms.tell() == vo.getBytesWritten() );
377 ms.setSize( ms.tell() );
378 }
379 unitTest( m.getSize( *i ) == vo.getBytesWritten() );
380 incProgress();
381 }
382 }
383 }
384}
385