aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMike Buland <eichlan@xagasoft.com>2009-01-27 15:25:46 +0000
committerMike Buland <eichlan@xagasoft.com>2009-01-27 15:25:46 +0000
commit9098237f5bb16b204a5ea999b702e5eb170f68ac (patch)
treeddf0c3013f0877d1f406401c6b4509d11bfb23e3
parent8bc5ac336d5d684341a05e97d1cb1b18ecba0331 (diff)
downloadlibbu++-9098237f5bb16b204a5ea999b702e5eb170f68ac.tar.gz
libbu++-9098237f5bb16b204a5ea999b702e5eb170f68ac.tar.bz2
libbu++-9098237f5bb16b204a5ea999b702e5eb170f68ac.tar.xz
libbu++-9098237f5bb16b204a5ea999b702e5eb170f68ac.zip
Corrected some larger read/write issues in corner cases that I hit suprisingly
often within nids. There's still a problem somewhere, but I'll find it. Also, even after having the file class canRead and canWrite functions work properly, and using them before trying to write to a nids to update info, we never ever write anything, so something is still wrong there. For now, all utilities that open a nids stream read-only will crash when it closes. Pretty minor really.
-rw-r--r--src/cachestorenids.h10
-rw-r--r--src/file.cpp8
-rw-r--r--src/nids.cpp40
-rw-r--r--src/nids.h7
-rw-r--r--src/nidsstream.cpp65
-rw-r--r--src/tests/nidstool.cpp124
-rw-r--r--src/tests/rh.cpp52
7 files changed, 286 insertions, 20 deletions
diff --git a/src/cachestorenids.h b/src/cachestorenids.h
index 62d1555..bd0fcc7 100644
--- a/src/cachestorenids.h
+++ b/src/cachestorenids.h
@@ -7,6 +7,10 @@
7#include "bu/nidsstream.h" 7#include "bu/nidsstream.h"
8#include "bu/cachestore.h" 8#include "bu/cachestore.h"
9 9
10#include "bu/file.h"
11
12static int iCnt = 0;
13
10namespace Bu 14namespace Bu
11{ 15{
12 template<class obtype, class keytype> 16 template<class obtype, class keytype>
@@ -44,6 +48,12 @@ namespace Bu
44 NidsStream ns = nStore.openStream( 0 ); 48 NidsStream ns = nStore.openStream( 0 );
45 Bu::Archive ar( ns, Bu::Archive::save ); 49 Bu::Archive ar( ns, Bu::Archive::save );
46 ar << hId; 50 ar << hId;
51
52 Bu::FString sName;
53 sName.format("hash-%d.%02d", time( NULL ), iCnt++ );
54 Bu::File sTmp( sName, Bu::File::Write|Bu::File::Create );
55 Bu::Archive artmp( sTmp, Bu::Archive::save );
56 artmp << hId;
47 } 57 }
48 58
49 virtual obtype *load( const keytype &key ) 59 virtual obtype *load( const keytype &key )
diff --git a/src/file.cpp b/src/file.cpp
index 7c18a06..20ff5c9 100644
--- a/src/file.cpp
+++ b/src/file.cpp
@@ -110,12 +110,20 @@ bool Bu::File::isEOS()
110 110
111bool Bu::File::canRead() 111bool Bu::File::canRead()
112{ 112{
113#ifdef WIN32
113 return true; 114 return true;
115#else
116 return (fcntl( fd, F_GETFL, 0 )&O_RDONLY) == O_RDONLY;
117#endif
114} 118}
115 119
116bool Bu::File::canWrite() 120bool Bu::File::canWrite()
117{ 121{
122#ifdef WIN32
118 return true; 123 return true;
124#else
125 return (fcntl( fd, F_GETFL, 0 )&O_WRONLY) == O_WRONLY;
126#endif
119} 127}
120 128
121bool Bu::File::isReadable() 129bool Bu::File::isReadable()
diff --git a/src/nids.cpp b/src/nids.cpp
index 06895ac..00d5f2b 100644
--- a/src/nids.cpp
+++ b/src/nids.cpp
@@ -21,18 +21,19 @@ Bu::Nids::Nids( Bu::Stream &sStore ) :
21 sStore( sStore ), 21 sStore( sStore ),
22 iBlockSize( 0 ), 22 iBlockSize( 0 ),
23 iBlocks( 0 ), 23 iBlocks( 0 ),
24 iBlockStart( -1 ) 24 iBlockStart( -1 ),
25 iUsed( 0 )
25{ 26{
26} 27}
27 28
28Bu::Nids::~Nids() 29Bu::Nids::~Nids()
29{ 30{
31 updateHeader();
30} 32}
31 33
32void Bu::Nids::initialize() 34void Bu::Nids::initialize()
33{ 35{
34 unsigned char buf[4]; 36 unsigned char buf[4];
35 int iUsed;
36 if( sStore.read( buf, 4 ) < 4 ) 37 if( sStore.read( buf, 4 ) < 4 )
37 throw NidsException("Input stream appears to be empty."); 38 throw NidsException("Input stream appears to be empty.");
38 if( memcmp( buf, NIDS_MAGIC_CODE, 4 ) ) 39 if( memcmp( buf, NIDS_MAGIC_CODE, 4 ) )
@@ -111,6 +112,15 @@ void Bu::Nids::initialize( int iBlockSize, int iPreAllocate )
111 delete[] (char *)block; 112 delete[] (char *)block;
112} 113}
113 114
115void Bu::Nids::updateHeader()
116{
117// if( !sStore.canWrite() )
118// return;
119 sStore.setPos( 10 ); // Skip the magic number, version, bpi, block size
120 sStore.write( &iBlocks, 4 );
121 sStore.write( &iUsed, 4 );
122}
123
114void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock, 124void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
115 uint32_t uPrevBlock, bool bNew ) 125 uint32_t uPrevBlock, bool bNew )
116{ 126{
@@ -129,6 +139,7 @@ void Bu::Nids::initBlock( uint32_t uPos, uint32_t uFirstBlock,
129 sStore.write( buf, iSize ); 139 sStore.write( buf, iSize );
130 delete[] buf; 140 delete[] buf;
131 } 141 }
142 iUsed++;
132} 143}
133 144
134uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock, 145uint32_t Bu::Nids::createBlock( uint32_t uFirstBlock, uint32_t uPrevBlock,
@@ -171,6 +182,27 @@ int Bu::Nids::getBlockSize()
171{ 182{
172 return iBlockSize; 183 return iBlockSize;
173} 184}
185
186int Bu::Nids::getNumBlocks()
187{
188 return iBlocks;
189}
190
191int Bu::Nids::getNumUsedBlocks()
192{
193 return iUsed;
194}
195
196int Bu::Nids::getBlockStart()
197{
198 return iBlockStart;
199}
200
201int Bu::Nids::getBlockOverhead()
202{
203 return sizeof(Block);
204}
205
174/* 206/*
175void Bu::Nids::extendStream( int iID, int iBlockCount ) 207void Bu::Nids::extendStream( int iID, int iBlockCount )
176{ 208{
@@ -190,6 +222,8 @@ void Bu::Nids::setBlock( uint32_t uIndex, Bu::Nids::Block *pBlock )
190 222
191void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize ) 223void Bu::Nids::updateStreamSize( uint32_t uIndex, uint32_t uSize )
192{ 224{
225// if( !sStore.canWrite() )
226// return;
193 sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 ); 227 sStore.setPos( iBlockStart + (iBlockSize*uIndex)+4*3 );
194 sStore.write( &uSize, 4 ); 228 sStore.write( &uSize, 4 );
195} 229}
@@ -206,6 +240,8 @@ uint32_t Bu::Nids::getNextBlock( uint32_t uIndex,
206 sStore.setPos( iBlockStart + (iBlockSize*uIndex)+1*4 ); 240 sStore.setPos( iBlockStart + (iBlockSize*uIndex)+1*4 );
207 sStore.write( &uNew, 4 ); 241 sStore.write( &uNew, 4 );
208 getBlock( uNew, pBlock ); 242 getBlock( uNew, pBlock );
243 printf("Allocated new block (%u) for stream %u.\n",
244 uNew, pBlock->uFirstBlock );
209 } 245 }
210 else 246 else
211 { 247 {
diff --git a/src/nids.h b/src/nids.h
index d297fa1..63d7061 100644
--- a/src/nids.h
+++ b/src/nids.h
@@ -65,6 +65,10 @@ namespace Bu
65 NidsStream openStream( int iID ); 65 NidsStream openStream( int iID );
66 66
67 int getBlockSize(); 67 int getBlockSize();
68 int getNumBlocks();
69 int getNumUsedBlocks();
70 int getBlockStart();
71 int getBlockOverhead();
68 72
69 private: 73 private:
70 typedef struct Block 74 typedef struct Block
@@ -92,6 +96,8 @@ namespace Bu
92 uint32_t getNextBlock( uint32_t uIndex, struct Nids::Block *pBlock, 96 uint32_t getNextBlock( uint32_t uIndex, struct Nids::Block *pBlock,
93 bool bCreate=true); 97 bool bCreate=true);
94 98
99 void updateHeader();
100
95 // Block allocation routines 101 // Block allocation routines
96 Block *newBlock(); 102 Block *newBlock();
97 void deleteBlock( Block *pBlock ); 103 void deleteBlock( Block *pBlock );
@@ -101,6 +107,7 @@ namespace Bu
101 int iBlockSize; 107 int iBlockSize;
102 int iBlocks; 108 int iBlocks;
103 int iBlockStart; 109 int iBlockStart;
110 int iUsed;
104 Bu::BitString bsBlockUsed; 111 Bu::BitString bsBlockUsed;
105 }; 112 };
106}; 113};
diff --git a/src/nidsstream.cpp b/src/nidsstream.cpp
index 4f692c6..fff73f6 100644
--- a/src/nidsstream.cpp
+++ b/src/nidsstream.cpp
@@ -45,15 +45,27 @@ void Bu::NidsStream::close()
45 45
46size_t Bu::NidsStream::read( void *pBuf, size_t nBytes ) 46size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
47{ 47{
48 if( uPos%uBlockSize+nBytes < uBlockSize ) 48 if( nBytes == 0 )
49 return 0;
50 if( nBytes + uPos > uSize )
51 nBytes = uSize - uPos;
52 if( (uPos%uBlockSize)+nBytes < uBlockSize )
49 { 53 {
50 size_t iRead = nBytes; 54 size_t iRead = nBytes;
51 if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) ) 55 if( iRead > pCurBlock->uBytesUsed-(uPos%uBlockSize) )
52 iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); 56 iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
53 memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead ); 57 memcpy( pBuf, pCurBlock->pData+(uPos%uBlockSize), iRead );
58 //printf("buffill: %ub, %u-%u/%u -> %d-%d/%d (a:%u)",
59 // iRead, uPos, uPos+iRead-1, uSize, 0, iRead-1, nBytes, uCurBlock );
54 uPos += iRead; 60 uPos += iRead;
55 //printf("a: block %u = %ub (%ub total)\n", 61 //printf(" -- %u\n", uPos%uBlockSize );
56 // uCurBlock, pCurBlock->uBytesUsed, uSize ); 62 //printf("ra: block %u = %ub:%u (%ub total)\n",
63 // uCurBlock, uPos, nBytes, uSize );
64
65 // This can't happen, if we're right on a boundery, it goes to the
66 // other case
67 //if( uPos%uBlockSize == 0 )
68 // uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false );
57 return iRead; 69 return iRead;
58 } 70 }
59 else 71 else
@@ -68,15 +80,18 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
68 iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize); 80 iRead = pCurBlock->uBytesUsed-(uPos%uBlockSize);
69 memcpy( ((char *)pBuf)+nTotal, 81 memcpy( ((char *)pBuf)+nTotal,
70 pCurBlock->pData+(uPos%uBlockSize), iRead ); 82 pCurBlock->pData+(uPos%uBlockSize), iRead );
83 //printf("buffill: %ub, %u-%u/%u -> %d-%d/%d (b:%u)\n",
84 // iRead, uPos, uPos+iRead-1, uSize,
85 // nTotal, nTotal+nBytes-1, nBytes, uCurBlock );
71 uPos += iRead; 86 uPos += iRead;
72 nBytes -= iRead; 87 nBytes -= iRead;
73 nTotal += iRead; 88 nTotal += iRead;
74 //printf("r: block %u = %ub/%ub (%ub total)\n", 89 //printf("rb: block %u = %ub:%u (%ub total)\n",
75 // uCurBlock, iRead, pCurBlock->uBytesUsed, uSize ); 90 // uCurBlock, uPos, iRead, uSize );
76 if( nBytes == 0 || uPos == uSize ) 91 if( uPos%uBlockSize == 0 )
77 return nTotal;
78 if( nTotal%uBlockSize == 0 )
79 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false ); 92 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock, false );
93 if( nBytes == 0 || uPos >= uSize )
94 return nTotal;
80 } 95 }
81 } 96 }
82 return 0; 97 return 0;
@@ -84,15 +99,26 @@ size_t Bu::NidsStream::read( void *pBuf, size_t nBytes )
84 99
85size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes ) 100size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes )
86{ 101{
87 if( uPos%uBlockSize+nBytes < uBlockSize ) 102 if( nBytes == 0 )
103 return 0;
104/* if( pCurBlock->uBytesUsed >= uBlockSize )
88 { 105 {
106 // We're at the end of our current block, allocate another before we do
107 // anything.
108 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock );
109 } */
110 if( (uPos%uBlockSize)+nBytes < uBlockSize )
111 {
112 //printf("wa: %u:%u:%u:%u -> ", uPos, uPos%uBlockSize, uSize, pCurBlock->uBytesUsed );
89 memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes ); 113 memcpy( pCurBlock->pData+(uPos%uBlockSize), pBuf, nBytes );
90 pCurBlock->uBytesUsed += nBytes; 114 if( (uPos%uBlockSize)+nBytes > pCurBlock->uBytesUsed )
115 pCurBlock->uBytesUsed = (uPos%uBlockSize)+nBytes;
91 rNids.setBlock( uCurBlock, pCurBlock ); 116 rNids.setBlock( uCurBlock, pCurBlock );
92 uPos += nBytes; 117 uPos += nBytes;
93 uSize += nBytes; 118 if( uPos > uSize )
94 //printf("a: block %u = %ub (%ub total)\n", 119 uSize = uPos;
95 // uCurBlock, pCurBlock->uBytesUsed, uSize ); 120 //printf("block %u = %ub (%ub total) %d:%u\n",
121 // uCurBlock, pCurBlock->uBytesUsed, uSize, nBytes, uPos );
96 return nBytes; 122 return nBytes;
97 } 123 }
98 else 124 else
@@ -101,22 +127,25 @@ size_t Bu::NidsStream::write( const void *pBuf, size_t nBytes )
101 for(;;) 127 for(;;)
102 { 128 {
103 uint32_t uNow = uBlockSize-(uPos%uBlockSize); 129 uint32_t uNow = uBlockSize-(uPos%uBlockSize);
130 //printf("uNow: %u (%u-(%u%%%u)) %d req\n", uNow, uBlockSize, uPos, uBlockSize, nBytes );
104 if( nBytes < uNow ) 131 if( nBytes < uNow )
105 uNow = nBytes; 132 uNow = nBytes;
106 memcpy( pCurBlock->pData+(uPos%uBlockSize), 133 memcpy( pCurBlock->pData+(uPos%uBlockSize),
107 &((char *)pBuf)[nTotal], uNow ); 134 &((char *)pBuf)[nTotal], uNow );
108 pCurBlock->uBytesUsed += uNow; 135 if( (uPos%uBlockSize)+uNow > pCurBlock->uBytesUsed )
136 pCurBlock->uBytesUsed = (uPos%uBlockSize)+uNow;
109 rNids.setBlock( uCurBlock, pCurBlock ); 137 rNids.setBlock( uCurBlock, pCurBlock );
110 uSize += uNow;
111 uPos += uNow; 138 uPos += uNow;
139 if( uPos > uSize )
140 uSize = uPos;
112 nTotal += uNow; 141 nTotal += uNow;
113 nBytes -= uNow; 142 nBytes -= uNow;
114 //printf("b: block %u = %ub (%ub total)\n", 143 //printf("wb: block %u = %ub (%ub total)\n",
115 // uCurBlock, pCurBlock->uBytesUsed, uSize ); 144 // uCurBlock, pCurBlock->uBytesUsed, uSize );
116 if( nBytes == 0 )
117 return nTotal;
118 if( pCurBlock->uBytesUsed == uBlockSize ) 145 if( pCurBlock->uBytesUsed == uBlockSize )
119 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock ); 146 uCurBlock = rNids.getNextBlock( uCurBlock, pCurBlock );
147 if( nBytes == 0 )
148 return nTotal;
120 } 149 }
121 } 150 }
122} 151}
diff --git a/src/tests/nidstool.cpp b/src/tests/nidstool.cpp
new file mode 100644
index 0000000..546534e
--- /dev/null
+++ b/src/tests/nidstool.cpp
@@ -0,0 +1,124 @@
1#include "bu/file.h"
2#include "bu/nids.h"
3#include "bu/nidsstream.h"
4#include "bu/paramproc.h"
5
6#include <stdlib.h>
7
8class Param : public Bu::ParamProc
9{
10public:
11 Param( int argc, char *argv[] )
12 {
13 addHelpBanner("nidstool - Do stuff with nids files.\n\n");
14 addParam("info", 'i', mkproc(Param::procInfo),
15 "Print some info about the file.");
16 addParam("dump", 'd', mkproc(Param::procDump),
17 "Dump a stream to a file");
18 addParam("help", 'h', mkproc(Bu::ParamProc::help), "This help.");
19 process( argc, argv );
20 }
21
22 virtual ~Param()
23 {
24 }
25
26 int procInfo( int argc, char *argv[] )
27 {
28 if( argc < 1 )
29 {
30 printf("You must provide a file name.\n");
31 exit( 1 );
32 }
33
34 Bu::File fIn( argv[0], Bu::File::Read );
35 Bu::Nids n( fIn );
36 n.initialize();
37
38 printf("Block size: %db\n", n.getBlockSize() );
39 printf("Block count: %d\n", n.getNumBlocks() );
40 printf("Blocks used: %d (%d%%)\n", n.getNumUsedBlocks(),
41 n.getNumUsedBlocks()*100/n.getNumBlocks() );
42 printf("Block start: %db\n", n.getBlockStart() );
43 printf("Block overhead: %db\n", n.getBlockOverhead() );
44 printf("Block storage: %db (%d%%)\n",
45 n.getBlockSize()-n.getBlockOverhead(),
46 (n.getBlockSize()-n.getBlockOverhead())*100/n.getBlockSize() );
47
48 if( argc >= 2 )
49 {
50 typedef struct Block
51 {
52 uint32_t uFirstBlock;
53 uint32_t uNextBlock;
54 uint32_t uPrevBlock;
55 uint32_t uBytesUsed;
56 uint32_t uReserved;
57 } Block;
58
59 uint32_t uStream = strtoul( argv[1], NULL, 0 );
60 uint32_t uBlock = uStream;
61
62 Block b;
63
64 for(;;)
65 {
66 fIn.setPos( n.getBlockStart()+n.getBlockSize()*uBlock );
67 fIn.read( &b, sizeof(Block) );
68 printf("Stream %u: block %u, next %u, prev %u, %ub used.\n",
69 uStream, uBlock, b.uNextBlock, b.uPrevBlock, b.uBytesUsed
70 );
71 if( b.uNextBlock == 0xFFFFFFFFUL )
72 break;
73 uBlock = b.uNextBlock;
74 }
75 printf("Stream End.\n");
76
77 return 2;
78 }
79
80 return 1;
81 }
82
83 int procDump( int argc, char *argv[] )
84 {
85 if( argc < 3 )
86 {
87 printf("You must provide a nids file, a stream id, and an output "
88 "file.\n");
89 exit( 1 );
90 }
91
92 Bu::File fIn( argv[0], Bu::File::Read );
93 Bu::Nids n( fIn );
94 n.initialize();
95
96 int iStream = strtol( argv[1], NULL, 0 );
97 Bu::NidsStream sIn = n.openStream( iStream );
98
99 Bu::File fOut( argv[2], Bu::File::Write|Bu::File::Create );
100 int iTotal = 0;
101 char buf[100];
102 for(;;)
103 {
104 int iRead = sIn.read( buf, 100 );
105 iTotal += fOut.write( buf, iRead );
106 if( iRead < 100 )
107 break;
108 }
109
110 printf("Wrote %db from stream %d in %s to %s.\n",
111 iTotal, iStream, argv[0], argv[2] );
112 return 3;
113 }
114
115};
116
117
118int main( int argc, char *argv[] )
119{
120 Param p( argc, argv );
121
122 return 0;
123}
124
diff --git a/src/tests/rh.cpp b/src/tests/rh.cpp
new file mode 100644
index 0000000..70abcb7
--- /dev/null
+++ b/src/tests/rh.cpp
@@ -0,0 +1,52 @@
1#include "bu/file.h"
2#include "bu/hash.h"
3#include "bu/archive.h"
4#include "bu/fstring.h"
5#include "bu/nids.h"
6#include "bu/nidsstream.h"
7
8int main( int argc, char *argv[] )
9{
10 if( argv[1][0] == 'r' )
11 {
12 typedef Bu::Hash<Bu::FString, int> Hsh;
13
14 Bu::File fIn( argv[2], Bu::File::Read );
15 Bu::Archive ar( fIn, Bu::Archive::load );
16
17 Hsh h;
18 ar >> h;
19
20 printf("Read %d.\n", h.getSize() );
21 for( Hsh::iterator i = h.begin(); i != h.end(); i++ )
22 {
23 printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() );
24 }
25
26 printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() );
27 }
28 else if( argv[1][0] == 'n' )
29 {
30 typedef Bu::Hash<Bu::FString, int> Hsh;
31
32 Bu::File fIn( argv[2], Bu::File::Read );
33 Bu::Nids n( fIn );
34 n.initialize();
35 Bu::NidsStream sIn = n.openStream( 0 );
36 Bu::Archive ar( sIn, Bu::Archive::load );
37
38 Hsh h;
39 ar >> h;
40
41 printf("Read %d.\n", h.getSize() );
42 for( Hsh::iterator i = h.begin(); i != h.end(); i++ )
43 {
44 printf(" - \"%s\" = %d\n", i.getKey().getStr(), i.getValue() );
45 }
46
47 printf("%d vs. %d\n", h.getSize(), h.getKeys().getSize() );
48 }
49
50 return 0;
51}
52