I made some awesome progress on the UtfString system, it stores in native utf16

encoding to make things easier (little endian in our case).  It can currently
read utf8 and utf16be, but not BOM.  It will give you full unicode code points
instead of the raw utf16 values, which is pretty slick.

4 files changed, 164 insertions, 5 deletions

diff --git a/src/config.h b/src/config.h
index 3046b59..ce954de 100644
--- a/src/config.h
+++ b/src/config.h
@@ -17,4 +17,7 @@
 
 #include "bu/extratypes.h"
 
+// Later if we need autoconfig stuff, here's where it'll go.
+// #include "bu/autoconfig.h"
+
 #endif
diff --git a/src/tests/utf.cpp b/src/tests/utf.cpp
index 59d49c6..9e075e2 100644
--- a/src/tests/utf.cpp
+++ b/src/tests/utf.cpp
@@ -16,7 +16,8 @@ int main( int argc, char *argv[] )
                         int iAmnt = fIn.read( buf, 4096 );
                         sUtf8.append( buf, iAmnt );
                 }
-                Bu::UtfString::debugUtf8( sUtf8 );
+                Bu::UtfString us( sUtf8, Bu::UtfString::Utf16 );
+                us.debug();
         }
 }
 
diff --git a/src/utfstring.cpp b/src/utfstring.cpp
index 0e2060b..bb0a011 100644
--- a/src/utfstring.cpp
+++ b/src/utfstring.cpp
@@ -9,17 +9,156 @@
 
 #include "bu/string.h"
 
+#include <endian.h>
+
 Bu::UtfString::UtfString()
 {
 }
 
+Bu::UtfString::UtfString( const Bu::String &sInput, Encoding eEnc )
+{
+        set( sInput, eEnc );
+}
+
 Bu::UtfString::~UtfString()
 {
 }
 
+void Bu::UtfString::set( const Bu::String &sInput, Encoding eEnc )
+{
+        switch( eEnc )
+        {
+                case Utf8:
+                        setUtf8( sInput );
+                        break;
+
+                case Utf16:
+                case Utf16be:
+                        setUtf16( sInput );
+                        break;
+
+                case Utf16le:
+                        throw Bu::ExceptionBase("Utf16le not supported yet.");
+                        break;
+
+                case Utf32:
+                        throw Bu::ExceptionBase("Utf32 not supported yet.");
+                        break;
+
+                case Ucs16:
+                        throw Bu::ExceptionBase("Ucs16 not supported yet.");
+                        break;
+
+                case GuessEncoding:
+                        throw Bu::ExceptionBase("Guessing mode not supported yet.");
+                        break;
+        }
+}
+
+void Bu::UtfString::append( UtfChar ch )
+{
+        if( ch >= 0x10000 )
+        {
+                ch -= 0x10000;
+                append16( ((ch>>10)&0x3FF)| 0xD800u );
+                append16( (ch&0x3FF)| 0xDC00u );
+        }
+        else
+        {
+                append16( (uint16_t)(ch) );
+        }
+}
+
+void Bu::UtfString::setUtf8( const Bu::String &sInput )
+{
+        static uint8_t lmask[8] = {
+                0x00,
+                0x01,
+                0x03,
+                0x07,
+                0x0f,
+                0x1f,
+                0x3f,
+                0x7f
+        };
+        for( Bu::String::const_iterator i = sInput.begin(); i; i++ )
+        {
+                if( ((int)(uint8_t)*i)&0x80 )
+                {
+                        int iBytes = 1;
+                        for(; (((uint8_t)(*i))<<iBytes)&0x80; iBytes++ ) { }
+                        Bu::UtfChar uPt = ((*i) & lmask[7-iBytes])<<(6*(iBytes-1));
+                        for( iBytes--; iBytes >= 1; iBytes-- )
+                        {
+                                i++;
+                                uPt |= ((*i)&lmask[6])<<(6*(iBytes-1));
+                        }
+                        append( uPt );
+                }
+                else
+                {
+                        append( (Bu::UtfChar)(*i) );
+                }
+        }
+}
+
+void Bu::UtfString::setUtf16( const Bu::String &sInput )
+{
+        uint16_t hi, lo;
+        for( Bu::String::const_iterator i = sInput.begin(); i; i++ )
+        {
+                hi = (((uint8_t)*i)<<8) | ((uint8_t)*(++i));
+                append16( hi );
+                if( (hi&0xD800u) == 0xD800u )
+                {
+                        lo = (((uint8_t)*(++i))<<8) | ((uint8_t)*(++i));
+                        append16( lo );
+                }
+        }
+}
+
 #include "bu/sio.h"
 using Bu::sio;
 
+Bu::UtfChar Bu::UtfString::get( int iIndex )
+{
+        Bu::UtfChar i = aData[iIndex];
+        switch( i&0xFC00 )
+        {
+                case 0xD800:
+                        sio << "(hi) ";
+                        return (((i&0x3FF)<<10) | ((aData[iIndex+1]&0x3FF)))+0x10000;
+
+                case 0xDC00:
+                        sio << "(lo) ";
+                        return 0;
+
+                default:
+                        sio << "(--) ";
+                        return i&0xFC00;
+        }
+}
+
+void Bu::UtfString::debug()
+{
+        sio << "Raw Utf16: ";
+        for( int i = 0; i < aData.getSize(); i++ )
+        {
+                if( i > 0 )
+                        sio << ", ";
+                sio << "0x" << Fmt::hex() << aData[i];
+        }
+        sio << sio.nl;
+        sio << "Code Points: ";
+        for( int i = 0; i < aData.getSize(); i++ )
+        {
+                if( i > 0 )
+                        sio << ", ";
+                sio << "0x" << Fmt::hex() << get( i );
+        }
+        sio << sio.nl;
+}
+/*
 void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 )
 {
         static uint8_t lmask[8] = {
@@ -43,7 +182,7 @@ void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 )
                         int iBytes = 1;
                         for(; (((uint8_t)(*i))<<iBytes)&0x80; iBytes++ ) { }
 //                      sio << "iBytes = " << iBytes << sio.nl;
-                        point uPt = ((*i) & lmask[7-iBytes])<<(6*(iBytes-1));
+                        Bu::UtfChar uPt = ((*i) & lmask[7-iBytes])<<(6*(iBytes-1));
 //                      sio << "mask: " << Bu::Fmt().radix(2).width(8).fill('0')
 //                              << (int)lmask[7-iBytes] << sio.nl;
                         for( iBytes--; iBytes >= 1; iBytes-- )
@@ -68,4 +207,4 @@ void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 )
         }
         sio << sio.nl;
 }
-
+*/
diff --git a/src/utfstring.h b/src/utfstring.h
index 6f85e93..79ef62e 100644
--- a/src/utfstring.h
+++ b/src/utfstring.h
@@ -9,9 +9,12 @@
 #define BU_UTF_STRING_H
 
 #include <stdint.h>
+#include "bu/array.h"
 
 namespace Bu
 {
+        class String;
+
         /**
          * UtfChar isn't actually a character, unicode specifies "code points" not
          * characters.  The main reason for this is that not all code points define
@@ -40,10 +43,23 @@ namespace Bu
                 UtfString( const Bu::String &sInput, Encoding eEnc=Utf8 );
                 virtual ~UtfString();
 
-                static void debugUtf8( const Bu::String &sUtf8 );
+                void append( UtfChar ch );
+
+                void set( const Bu::String &sInput, Encoding eEnc=Utf8 );
+                void setUtf8( const Bu::String &sInput );
+                void setUtf16( const Bu::String &sInput );
+//              void setUtf16be( const Bu::String &sInput );
+//              void setUtf16le( const Bu::String &sInput );
+
+                void debug();
+
+                UtfChar get( int iIndex );
+
+        private:
+                void append16( uint16_t i ) { aData.append( i ); }
 
         private:
-                uint16_t *pData;
+                Bu::Array<uint16_t> aData;
                 int iRawLen;
                 int iCharLen;
         };