Pretty sure all utf encoders and decoders are complete and tested.

3 files changed, 174 insertions, 13 deletions

diff --git a/src/tests/utf.cpp b/src/tests/utf.cpp
index 9e075e2..01bac7e 100644
--- a/src/tests/utf.cpp
+++ b/src/tests/utf.cpp
@@ -4,6 +4,46 @@
 
 int main( int argc, char *argv[] )
 {
+        Bu::File fIn("utf8.in", Bu::File::Read );
+        Bu::String sUtf8;
+        char buf[4096];
+        while( !fIn.isEos() )
+        {
+                int iAmnt = fIn.read( buf, 4096 );
+                sUtf8.append( buf, iAmnt );
+        }
+        Bu::UtfString us( sUtf8, Bu::UtfString::Utf8 );
+        us.debug();
+        {
+                Bu::File fOut("utf8.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf8 );
+        }
+        {
+                Bu::File fOut("utf16.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf16 );
+        }
+        {
+                Bu::File fOut("utf16le.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf16le );
+        }
+        {
+                Bu::File fOut("utf16be.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf16be );
+        }
+        {
+                Bu::File fOut("utf32.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf32 );
+        }
+        {
+                Bu::File fOut("utf32le.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf32le );
+        }
+        {
+                Bu::File fOut("utf32be.out", Bu::File::WriteNew );
+                us.write( fOut, Bu::UtfString::Utf32be );
+        }
+
+        /*
         argc--, argv++;
 
         for( char **sFile = argv; *sFile; sFile++ )
@@ -17,7 +57,9 @@ int main( int argc, char *argv[] )
                         sUtf8.append( buf, iAmnt );
                 }
                 Bu::UtfString us( sUtf8, Bu::UtfString::Utf16 );
+
                 us.debug();
         }
+        */
 }
 
diff --git a/src/utfstring.cpp b/src/utfstring.cpp
index c9da52f..3f57618 100644
--- a/src/utfstring.cpp
+++ b/src/utfstring.cpp
@@ -259,8 +259,8 @@ void Bu::UtfString::write( Bu::Stream &sOut, Encoding eEnc )
                         break;
 
                 case Utf16:
-                        writeUtf16( sOut );
-                        break;
+//                      writeUtf16( sOut );
+//                      break;
 
                 case Utf16be:
                         writeUtf16be( sOut );
@@ -271,8 +271,8 @@ void Bu::UtfString::write( Bu::Stream &sOut, Encoding eEnc )
                         break;
 
                 case Utf32:
-                        writeUtf32( sOut );
-                        break;
+//                      writeUtf32( sOut );
+//                      break;
 
                 case Utf32be:
                         writeUtf32be( sOut );
@@ -300,30 +300,151 @@ void Bu::UtfString::write( Bu::Stream &sOut, Encoding eEnc )
 
 void Bu::UtfString::writeUtf8( Bu::Stream &sOut )
 {
+        int iPos = 0;
+        while( iPos < aData.getSize() )
+        {
+                uint8_t uByte;
+                Bu::UtfChar chr = nextChar( iPos );
+                if( chr >= 0x010000 )
+                {
+                        // Four bytes
+                        // 111 111111 111111 111111
+                        uByte = (chr>>18)|0xF0;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr>>12)&0x3F|0x80;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr>>6)&0x3F|0x80;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr&0x3F)|0x80;
+                        sOut.write( &uByte, 1 );
+                }
+                else if( chr >= 0x800 )
+                {
+                        // Three bytes
+                        // 1111 111111 111111
+                        uByte = (chr>>12)|0xE0;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr>>6)&0x3F|0x80;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr&0x3F)|0x80;
+                        sOut.write( &uByte, 1 );
+                }
+                else if( chr >= 0x80 )
+                {
+                        // Two bytes
+                        // 11111 111111
+                        uByte = (chr>>6)|0xC0;
+                        sOut.write( &uByte, 1 );
+                        uByte = (chr&0x3F)|0x80;
+                        sOut.write( &uByte, 1 );
+                }
+                else
+                {
+                        // One byte
+                        uByte = chr;
+                        sOut.write( &uByte, 1 );
+                }
+        }
 }
-
+/*
 void Bu::UtfString::writeUtf16( Bu::Stream &sOut )
 {
 }
-
+*/
 void Bu::UtfString::writeUtf16be( Bu::Stream &sOut )
 {
+#if BYTE_ORDER == BIG_ENDIAN
+        uint16_t iTmp = 0xFEFF; // Byte Order Marker
+        sOut.write( &iTmp, 2 );
+        for( Array<uint16_t>::iterator i = aData.begin(); i; i++ )
+        {
+                iTmp = *i;
+                sOut.write( &iTmp, 2 );
+        }
+#else
+        uint16_t iTmp = 0xFEFF; // Byte Order Marker
+        iTmp = (iTmp>>8) | (iTmp<<8);
+        sOut.write( &iTmp, 2 );
+        for( Array<uint16_t>::iterator i = aData.begin(); i; i++ )
+        {
+                iTmp = *i;
+                iTmp = (iTmp>>8) | (iTmp<<8);
+                sOut.write( &iTmp, 2 );
+        }
+#endif
 }
 
 void Bu::UtfString::writeUtf16le( Bu::Stream &sOut )
 {
-}
-
-void Bu::UtfString::writeUtf32( Bu::Stream &sOut )
-{
+#if BYTE_ORDER == LITTLE_ENDIAN
+        uint16_t iTmp = 0xFEFF; // Byte Order Marker
+        sOut.write( &iTmp, 2 );
+        for( Array<uint16_t>::iterator i = aData.begin(); i; i++ )
+        {
+                iTmp = *i;
+                sOut.write( &iTmp, 2 );
+        }
+#else
+        uint16_t iTmp = 0xFEFF; // Byte Order Marker
+        iTmp = (iTmp>>8) | (iTmp<<8);
+        sOut.write( &iTmp, 2 );
+        for( Array<uint16_t>::iterator i = aData.begin(); i; i++ )
+        {
+                iTmp = *i;
+                iTmp = (iTmp>>8) | (iTmp<<8);
+                sOut.write( &iTmp, 2 );
+        }
+#endif
 }
 
 void Bu::UtfString::writeUtf32be( Bu::Stream &sOut )
 {
+#if BYTE_ORDER == BIG_ENDIAN
+        uint32_t iTmp = 0xFEFF; // Byte Order Marker
+        sOut.write( &iTmp, 4 );
+        int i = 0;
+        while( i < aData.getSize() )
+        {
+                iTmp = nextChar( i );
+                sOut.write( &iTmp, 4 );
+        }
+#else
+        uint32_t iTmp = 0xFEFF; // Byte Order Marker
+        iTmp = (iTmp>>24)|(iTmp<<24)|((iTmp&0xff0000)>>8)|((iTmp&0xff00)<<8);
+        sOut.write( &iTmp, 4 );
+        int i = 0;
+        while( i < aData.getSize() )
+        {
+                iTmp = nextChar( i );
+                iTmp = (iTmp>>24)|(iTmp<<24)|((iTmp&0xff0000)>>8)|((iTmp&0xff00)<<8);
+                sOut.write( &iTmp, 4 );
+        }
+#endif
 }
 
 void Bu::UtfString::writeUtf32le( Bu::Stream &sOut )
 {
+#if BYTE_ORDER == LITTLE_ENDIAN
+        uint32_t iTmp = 0xFEFF; // Byte Order Marker
+        sOut.write( &iTmp, 4 );
+        int i = 0;
+        while( i < aData.getSize() )
+        {
+                iTmp = nextChar( i );
+                sOut.write( &iTmp, 4 );
+        }
+#else
+        uint32_t iTmp = 0xFEFF; // Byte Order Marker
+        iTmp = (iTmp>>24)|(iTmp<<24)|((iTmp&0xff0000)>>8)|((iTmp&0xff00)<<8);
+        sOut.write( &iTmp, 4 );
+        int i = 0;
+        while( i < aData.getSize() )
+        {
+                iTmp = nextChar( i );
+                iTmp = (iTmp>>24)|(iTmp<<24)|((iTmp&0xff0000)>>8)|((iTmp&0xff00)<<8);
+                sOut.write( &iTmp, 4 );
+        }
+#endif
 }
 
 Bu::UtfChar Bu::UtfString::get( int iIndex )
@@ -362,7 +483,7 @@ void Bu::UtfString::debug()
         {
                 if( i > 0 )
                         sio << ", ";
-                sio << "0x" << Fmt::hex() << get( i );
+                sio << "0x" << Fmt::hex() << nextChar( i );
         }
         sio << sio.nl;
 }
diff --git a/src/utfstring.h b/src/utfstring.h
index 8448ea4..be3e6ad 100644
--- a/src/utfstring.h
+++ b/src/utfstring.h
@@ -86,10 +86,8 @@ namespace Bu
 
                 void write( Bu::Stream &sOut, Encoding eEnc=Utf8 );
                 void writeUtf8( Bu::Stream &sOut );
-                void writeUtf16( Bu::Stream &sOut );
                 void writeUtf16be( Bu::Stream &sOut );
                 void writeUtf16le( Bu::Stream &sOut );
-                void writeUtf32( Bu::Stream &sOut );
                 void writeUtf32be( Bu::Stream &sOut );
                 void writeUtf32le( Bu::Stream &sOut );