1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
#ifndef BU_MYRIAD_H
#define BU_MYRIAD_H
#include "bu/stream.h"
#include "bu/exceptionbase.h"
#include "bu/mutex.h"
#include "bu/array.h"
#include "bu/hash.h"
#include "bu/bitstring.h"
namespace Bu
{
class MyriadStream;
subExceptionDeclBegin( MyriadException )
enum
{
emptyStream,
invalidFormat,
badVersion,
invalidWordSize,
noSuchStream,
streamExists,
invalidStreamId,
protectedStream,
invalidParameter,
invalidBackingStream,
badMode,
streamOpen,
};
subExceptionDeclEnd();
/**
* Myriad Stream Multiplexer. This is a system that allows you to store
* many streams within a single backing stream. This is great for databases,
* caching, etc. It's fairly lightweight, and allows all streams to grow
* dynamically using a block-allocation scheme. This is used extensively
* by the caching system and MyriadFs as well as other systems within
* libbu++.
*/
class Myriad
{
public:
typedef int32_t StreamId;
typedef Bu::Array<StreamId> StreamIdArray;
typedef Bu::List<StreamId> StreamIdList;
enum Mode : int32_t {
None = 0x00,
// Flags
Read = 0x01, ///< Open file for reading
Write = 0x02, ///< Open file for writing
Create = 0x04, ///< Create file if it doesn't exist
Truncate = 0x08, ///< Truncate file if it does exist
Append = 0x10, ///< Start writing at end of file
//NonBlock = 0x20, ///< Open file in non-blocking mode
Exclusive = 0x40, ///< Create file, if it exists then fail
// Helpful mixes
ReadWrite = 0x03, ///< Open for reading and writing
WriteNew = 0x0E ///< Create a file (or truncate) for writing.
/// Same as Write|Create|Truncate
};
public:
/**
* Open existing Myriad container, or initialize a new one if the
* backing stream is empty. If other data is already in the provided
* backing stream an error is thrown.
*
* Myriad format V0
* 0 - 3: Myriad_MAGIC_CODE (0ad3fa84)
* 4 - 4: Version Id (1)
* 5 - 5: Bits per integer (32)
* 6 - 9: Block size in bytes.
* 10 - 13: Number of streams.
* 14 - ...: Stream Data
*
* Stream Data:
* 0 - 3: Stream Id
* 4 - 7: Size of stream in bytes
* 8 - ...: List of blocks in stream (4 bytes per block
*/
Myriad( Bu::Stream &rBacking, int32_t iBlockSize=-1,
int32_t iPreallocateBlocks=-1 );
virtual ~Myriad();
/**
* Creates a new stream open in the specified eMode and, optionally,
* preallocates the specificed amount of space. The stream is zero
* bytes even if space is preallocated. The open stream is returned,
* ready for use. Use this if you don't care what the id is of the
* newly created stream.
*/
MyriadStream create( Mode eMode, int32_t iPreallocateBytes=-1 );
/**
* Open an existing stream or create a new stream with the specified
* id (iStream) with the specified eMode. This respects the normal file
* modes, see Bu::Myriad::Mode for details.
*/
MyriadStream open( StreamId iStream, Mode eMode );
/**
* Allocate a new stream but do not open it, just ensure it exists and
* return the id of the newly allocated stream.
*/
StreamId allocate();
/**
* Erase the stream specified by iStream. This only can work when the
* stream is not open at the moment.
*/
void erase( StreamId iStream );
void setSize( StreamId iStream, int32_t iNewSize );
int32_t getSize( StreamId iStream ) const;
bool exists( StreamId iStream ) const;
Bu::String getLocation() const;
int32_t getBlockSize() const;
int32_t getTotalBlocks() const;
int32_t getUsedBlocks() const;
int32_t getFreeBlocks() const;
int32_t getTotalStreams() const;
int32_t getTotalUsedBytes() const;
int32_t getTotalUnusedBytes( int32_t iAssumeBlockSize=-1 ) const;
Bu::BitString buildBlockUseMap() const;
StreamIdArray buildBlockMap() const;
/**
* Lists all stream ids that you are allowed to open. Technically there
* is always a zero stream, but it is used by Myriad for stream/block
* accounting. It works like a normal stream but you should not open
* it.
*/
StreamIdList getStreamList() const;
/**
* Flush all caches to the backing stream, write all structural and
* header changes.
*/
void sync();
private:
bool loadMyriad();
void createMyriad( int32_t iBlockSize, int32_t iPreallocateBlocks );
void writeHeader();
int32_t __calcHeaderSize();
int32_t allocateBlock();
int32_t __allocateBlock();
void releaseBlock( int32_t iBlockId, bool bBlank=true );
void __releaseBlock( int32_t iBlockId, bool bBlank=true );
void blankBlock( int32_t iBlockId );
void openStream( StreamId id );
void closeStream( StreamId id );
/**
* Block restricted read, it will not read past the end of the block
* that iStart places it in.
*/
int32_t blockRead( int32_t iBlock, int32_t iStart,
void *pTarget, int32_t iSize );
/**
* Block restricted write, it will not write past the end of the block
* that iStart places it in. If this returns a non-zero number it's an
* indication that you need to allocate a new block.
*/
int32_t blockWrite( int32_t iBlock, int32_t iStart,
const void *pTarget, int32_t iSize );
public:
/**
* Bridge/communication/tracking class for individual Myriad streams.
* Not for general use, this is used by Myriad and MyriadStream to
* control access.
*/
class Stream
{
friend Bu::Myriad;
private:
Stream( Myriad &rParent, StreamId iStream, int32_t iSize );
virtual ~Stream();
public:
int32_t getSize() const;
int32_t getBlockSize() const;
StreamId getStreamId() const;
int32_t getOpenCount() const;
void setSize( int32_t iNewSize );
int32_t read( int32_t iStart, void *pTarget, int32_t iSize );
int32_t write( int32_t iStart, const void *pTarget, int32_t iSize );
Bu::String getLocation() const;
Bu::Array<int32_t> getBlockList() const;
/**
* Doesn't actually open, just increments the open counter.
* If the open counter is non-zero then at least one stream has
* a lock on this stream.
*/
void open();
/**
* Doesn't actually close, just decrements the open counter.
*@returns true if there are still handles open, false if no
* streams have a lock.
*/
bool close();
private:
mutable Bu::Mutex mAccess;
Myriad &rParent;
StreamId iStream;
int32_t iSize;
Bu::Array<int32_t> aBlocks;
int32_t iOpenCount;
};
private:
typedef Bu::Hash<StreamId, Stream *> StreamHash;
typedef Bu::List<int32_t> IndexList;
mutable Bu::Mutex mAccess;
mutable Bu::Mutex mBacking;
Bu::Stream &rBacking;
int32_t iBlockSize;
int32_t iBlockCount;
bool bIsNewStream;
bool bStructureChanged;
mutable Bu::Mutex mhStream;
StreamHash hStream;
IndexList lFreeBlocks;
StreamId iLastUsedIndex;
};
constexpr Myriad::Mode operator&( Myriad::Mode a, Myriad::Mode b )
{
return static_cast<Myriad::Mode>(
static_cast<std::underlying_type<Myriad::Mode>::type>(a) &
static_cast<std::underlying_type<Myriad::Mode>::type>(b)
);
}
constexpr Myriad::Mode operator|( Myriad::Mode a, Myriad::Mode b )
{
return static_cast<Myriad::Mode>(
static_cast<std::underlying_type<Myriad::Mode>::type>(a) |
static_cast<std::underlying_type<Myriad::Mode>::type>(b)
);
}
};
#endif
|
