Fixed an issue in the reader, it was tokenizing.

1 files changed, 40 insertions, 19 deletions

diff --git a/src/formatter.cpp b/src/formatter.cpp
index ce92caa..7eaa1e2 100644
--- a/src/formatter.cpp
+++ b/src/formatter.cpp
@@ -120,30 +120,45 @@ void Bu::Formatter::read( void *sStr, int iLen )
 Bu::FString Bu::Formatter::readToken()
 {
         Bu::FString sRet;
-        for(;;)
+        if( fLast.bTokenize )
         {
-                char buf;
-                int iRead = rStream.read( &buf, 1 );
-                if( iRead == 0 )
-                        return sRet;
-                if( buf == ' ' || buf == '\t' || buf == '\n' || buf == '\r' )
-                        continue;
-                else
+                for(;;)
                 {
-                        sRet += buf;
-                        break;
+                        char buf;
+                        int iRead = rStream.read( &buf, 1 );
+                        if( iRead == 0 )
+                                return sRet;
+                        if( buf == ' ' || buf == '\t' || buf == '\n' || buf == '\r' )
+                                continue;
+                        else
+                        {
+                                sRet += buf;
+                                break;
+                        }
+                }
+                for(;;)
+                {
+                        char buf;
+                        int iRead = rStream.read( &buf, 1 );
+                        if( iRead == 0 )
+                                return sRet;
+                        if( buf == ' ' || buf == '\t' || buf == '\n' || buf == '\r' )
+                                return sRet;
+                        else
+                                sRet += buf;
                 }
         }
-        for(;;)
+        else
         {
-                char buf;
-                int iRead = rStream.read( &buf, 1 );
-                if( iRead == 0 )
-                        return sRet;
-                if( buf == ' ' || buf == '\t' || buf == '\n' || buf == '\r' )
-                        return sRet;
-                else
-                        sRet += buf;
+                for(;;)
+                {
+                        char buf;
+                        int iRead = rStream.read( &buf, 1 );
+                        if( iRead == 0 )
+                                return sRet;
+                        else
+                                sRet += buf;
+                }
         }
 }
 
@@ -228,6 +243,12 @@ Bu::Formatter::Fmt &Bu::Formatter::Fmt::caps( bool bCaps )
         return *this;
 }
 
+Bu::Formatter::Fmt &Bu::Formatter::Fmt::tokenize( bool bTokenize )
+{
+        this->bTokenize = bTokenize;
+        return *this;
+}
+
 Bu::Formatter &Bu::operator<<( Bu::Formatter &f, const Bu::Formatter::Fmt &fmt )
 {
         f.setTempFormat( fmt );