This commit has a minor tweak to the variant class to make it easier to use,

and introduces the parser and lexer classes.  I also made a test for parser and
put it in the tools directory.  That is silly, it shouldn't be.  However, it's
necesarry right now, because I don't want to do a full build to compile all
the parser tests.

However, this commit doesn't actually build yet.  It will soon, I just wanted 
to get it all committed.

1 files changed, 44 insertions, 0 deletions

diff --git a/src/lexer.h b/src/lexer.h
new file mode 100644
index 0000000..37d268f
--- /dev/null
+++ b/src/lexer.h
@@ -0,0 +1,44 @@
+#ifndef BU_LEXER_H
+#define BU_LEXER_H
+
+#include "bu/variant.h"
+
+namespace Bu
+{
+        class Stream;
+
+        /**
+         * The base class for creating a lexical analyzer.  This is designed to work
+         * in tandem with the Bu::Parser class, which uses this to tokenize textual
+         * input.  It can be used by just about anything that cares about tokens
+         * more than raw input, though.
+         */
+        class Lexer
+        {
+        public:
+                Lexer();
+                virtual ~Lexer();
+
+                class Token
+                {
+                public:
+                        Token();
+                        Token( int iToken );
+
+                        template<class t>
+                        Token( int iToken, const t &v ) :
+                                iToken( iToken ),
+                                vExtra( v )
+                        {
+                        }
+                        int iToken;
+                        Bu::Variant vExtra;
+                };
+
+                virtual Token *nextToken()=0;
+
+                virtual Bu::FString tokenToString( const Token &t );
+        };
+};
+
+#endif