From abbf45c1da7f3e3a542e6c6339a1bab31283f22e Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 4 Apr 2011 07:22:10 +0000 Subject: I made some awesome progress on the UtfString system, it stores in native utf16 encoding to make things easier (little endian in our case). It can currently read utf8 and utf16be, but not BOM. It will give you full unicode code points instead of the raw utf16 values, which is pretty slick. --- autoconfig.cpp | 30 ++++++++++++ default.bld | 37 ++++++++++++-- src/config.h | 3 ++ src/tests/utf.cpp | 3 +- src/utfstring.cpp | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++- src/utfstring.h | 20 +++++++- 6 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 autoconfig.cpp diff --git a/autoconfig.cpp b/autoconfig.cpp new file mode 100644 index 0000000..aa8b6a4 --- /dev/null +++ b/autoconfig.cpp @@ -0,0 +1,30 @@ +#include +#include + +void detectEndianness() +{ + uint16_t x=0x0100; + fprintf( stderr, + "#define LITTLE_ENDIAN 0\n" + "#define BIG_ENDIAN 1\n" + "#define ENDIANNESS %d\n\n", + ((uint8_t *)&x)[0] + ); + printf("Archetecture is: %s Endian\n", ((uint8_t *)&x)[0]?"Big":"Little" ); +} + +int main() +{ + fprintf( stderr, + "#ifndef BU_AUTO_CONFIG_H\n" + "#define BU_AUTO_CONFIG_H\n\n" + ); + + // huh, turns out #include covers this... +// detectEndianness(); + + fprintf( stderr, "#endif\n"); + + return 0; +} + diff --git a/default.bld b/default.bld index aa7f4cb..1aca56a 100644 --- a/default.bld +++ b/default.bld @@ -15,7 +15,8 @@ CXXFLAGS += "-ggdb -W -Wall -I."; action "default" { - build: [targets("header-links"), "libbu++.a", targets("tools")]; + build: [targets("header-links"), "libbu++.a", + targets("tools")]; } action "pkg" @@ -25,14 +26,44 @@ action "pkg" action "all" { - build: [targets("header-links"), "libbu++.a", targets("tools"), - targets("tests")]; + build: [targets("header-links"), "libbu++.a", + targets("tools"), targets("tests")]; } action "unit" { build: targets("unit tests"); } +/* +target "src/autoconfig.h" +{ + input "autoconfig"; + display "autoconfig"; + profile "build" + { + execute("./autoconfig 2>src/autoconfig.h"); + } +} + +target "bu/autoconfig.h" +{ + tag "header-links"; + display "symlink"; + input "src/autoconfig.h"; + profile "build" + { + execute("echo ${INPUT}"); + execute("echo ${OUTPUT}"); + execute("mkdir -p $(dirname ${OUTPUT}); ln -s ../${INPUT} ${OUTPUT}"); + } +} + +target "autoconfig" +{ + rule "exe"; + input "autoconfig.cpp"; +} +*/ target files("src/*.h").replace("src/", "bu/") { diff --git a/src/config.h b/src/config.h index 3046b59..ce954de 100644 --- a/src/config.h +++ b/src/config.h @@ -17,4 +17,7 @@ #include "bu/extratypes.h" +// Later if we need autoconfig stuff, here's where it'll go. +// #include "bu/autoconfig.h" + #endif diff --git a/src/tests/utf.cpp b/src/tests/utf.cpp index 59d49c6..9e075e2 100644 --- a/src/tests/utf.cpp +++ b/src/tests/utf.cpp @@ -16,7 +16,8 @@ int main( int argc, char *argv[] ) int iAmnt = fIn.read( buf, 4096 ); sUtf8.append( buf, iAmnt ); } - Bu::UtfString::debugUtf8( sUtf8 ); + Bu::UtfString us( sUtf8, Bu::UtfString::Utf16 ); + us.debug(); } } diff --git a/src/utfstring.cpp b/src/utfstring.cpp index 0e2060b..bb0a011 100644 --- a/src/utfstring.cpp +++ b/src/utfstring.cpp @@ -9,17 +9,156 @@ #include "bu/string.h" +#include + Bu::UtfString::UtfString() { } +Bu::UtfString::UtfString( const Bu::String &sInput, Encoding eEnc ) +{ + set( sInput, eEnc ); +} + Bu::UtfString::~UtfString() { } +void Bu::UtfString::set( const Bu::String &sInput, Encoding eEnc ) +{ + switch( eEnc ) + { + case Utf8: + setUtf8( sInput ); + break; + + case Utf16: + case Utf16be: + setUtf16( sInput ); + break; + + case Utf16le: + throw Bu::ExceptionBase("Utf16le not supported yet."); + break; + + case Utf32: + throw Bu::ExceptionBase("Utf32 not supported yet."); + break; + + case Ucs16: + throw Bu::ExceptionBase("Ucs16 not supported yet."); + break; + + case GuessEncoding: + throw Bu::ExceptionBase("Guessing mode not supported yet."); + break; + } +} + +void Bu::UtfString::append( UtfChar ch ) +{ + if( ch >= 0x10000 ) + { + ch -= 0x10000; + append16( ((ch>>10)&0x3FF)| 0xD800u ); + append16( (ch&0x3FF)| 0xDC00u ); + } + else + { + append16( (uint16_t)(ch) ); + } +} + +void Bu::UtfString::setUtf8( const Bu::String &sInput ) +{ + static uint8_t lmask[8] = { + 0x00, + 0x01, + 0x03, + 0x07, + 0x0f, + 0x1f, + 0x3f, + 0x7f + }; + for( Bu::String::const_iterator i = sInput.begin(); i; i++ ) + { + if( ((int)(uint8_t)*i)&0x80 ) + { + int iBytes = 1; + for(; (((uint8_t)(*i))<= 1; iBytes-- ) + { + i++; + uPt |= ((*i)&lmask[6])<<(6*(iBytes-1)); + } + append( uPt ); + } + else + { + append( (Bu::UtfChar)(*i) ); + } + } +} + +void Bu::UtfString::setUtf16( const Bu::String &sInput ) +{ + uint16_t hi, lo; + for( Bu::String::const_iterator i = sInput.begin(); i; i++ ) + { + hi = (((uint8_t)*i)<<8) | ((uint8_t)*(++i)); + append16( hi ); + if( (hi&0xD800u) == 0xD800u ) + { + lo = (((uint8_t)*(++i))<<8) | ((uint8_t)*(++i)); + append16( lo ); + } + } +} + #include "bu/sio.h" using Bu::sio; +Bu::UtfChar Bu::UtfString::get( int iIndex ) +{ + Bu::UtfChar i = aData[iIndex]; + switch( i&0xFC00 ) + { + case 0xD800: + sio << "(hi) "; + return (((i&0x3FF)<<10) | ((aData[iIndex+1]&0x3FF)))+0x10000; + + case 0xDC00: + sio << "(lo) "; + return 0; + + default: + sio << "(--) "; + return i&0xFC00; + } +} + +void Bu::UtfString::debug() +{ + sio << "Raw Utf16: "; + for( int i = 0; i < aData.getSize(); i++ ) + { + if( i > 0 ) + sio << ", "; + sio << "0x" << Fmt::hex() << aData[i]; + } + sio << sio.nl; + sio << "Code Points: "; + for( int i = 0; i < aData.getSize(); i++ ) + { + if( i > 0 ) + sio << ", "; + sio << "0x" << Fmt::hex() << get( i ); + } + sio << sio.nl; +} +/* void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 ) { static uint8_t lmask[8] = { @@ -43,7 +182,7 @@ void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 ) int iBytes = 1; for(; (((uint8_t)(*i))<= 1; iBytes-- ) @@ -68,4 +207,4 @@ void Bu::UtfString::debugUtf8( const Bu::String &sUtf8 ) } sio << sio.nl; } - +*/ diff --git a/src/utfstring.h b/src/utfstring.h index 6f85e93..79ef62e 100644 --- a/src/utfstring.h +++ b/src/utfstring.h @@ -9,9 +9,12 @@ #define BU_UTF_STRING_H #include +#include "bu/array.h" namespace Bu { + class String; + /** * UtfChar isn't actually a character, unicode specifies "code points" not * characters. The main reason for this is that not all code points define @@ -40,10 +43,23 @@ namespace Bu UtfString( const Bu::String &sInput, Encoding eEnc=Utf8 ); virtual ~UtfString(); - static void debugUtf8( const Bu::String &sUtf8 ); + void append( UtfChar ch ); + + void set( const Bu::String &sInput, Encoding eEnc=Utf8 ); + void setUtf8( const Bu::String &sInput ); + void setUtf16( const Bu::String &sInput ); +// void setUtf16be( const Bu::String &sInput ); +// void setUtf16le( const Bu::String &sInput ); + + void debug(); + + UtfChar get( int iIndex ); + + private: + void append16( uint16_t i ) { aData.append( i ); } private: - uint16_t *pData; + Bu::Array aData; int iRawLen; int iCharLen; }; -- cgit v1.2.3