From da1e0ef0772b078bd295301bd675afdee00d40e9 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Sun, 23 Oct 2011 07:43:50 +0000 Subject: Switched ito* to synchro*, except the server, I'm thinking of takeing the core in a different direction anyway. Added the Deflate class, it uses zlib, and can do raw (headerless) deflate streams, zlib format, or gzip format. It's easy to use and quite versitile. --- src/deflate.h | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/deflate.h (limited to 'src/deflate.h') diff --git a/src/deflate.h b/src/deflate.h new file mode 100644 index 0000000..cab9b51 --- /dev/null +++ b/src/deflate.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2007-2011 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#ifndef BU_DEFLATE_H +#define BU_DEFLATE_H + +#include +#include + +#include "bu/filter.h" + +namespace Bu +{ + /** + * + *@ingroup Streams + */ + class Deflate : public Bu::Filter + { + public: + enum Format + { + Raw = 0x01, + Zlib = 0x02, + Gzip = 0x03, + AutoDetect = 0x04, + + AutoRaw = 0x04|0x01, + AutoZlib = 0x04|0x02, + AutoGzip = 0x04|0x03 + }; + + Deflate( Bu::Stream &rNext, int nCompression=9, Format eFmt=AutoRaw ); + virtual ~Deflate(); + + virtual void start(); + virtual Bu::size stop(); + virtual Bu::size read( void *pBuf, Bu::size nBytes ); + virtual Bu::size write( const void *pBuf, Bu::size nBytes ); + + virtual bool isOpen(); + virtual bool isEos(); + + Bu::size getCompressedSize(); + + private: + void zError( int code ); + z_stream zState; + bool bReading; + int nCompression; + char *pBuf; + uint32_t nBufSize; + Bu::size sTotalOut; + Format eFmt; + bool bEos; + }; +} + +#endif -- cgit v1.2.3 From e2fbc414b932ae9fd305c8e9fc315a306a876a09 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 24 Oct 2011 16:19:09 +0000 Subject: Lzma filter added. Now we're really getting somewhere. Libbu++ now supports all major, common compression algorithms. --- default.bld | 5 ++ src/deflate.h | 2 +- src/lzma.cpp | 248 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lzma.h | 56 ++++++++++++ src/tests/lzma.cpp | 53 ++++++++++++ 5 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 src/lzma.cpp create mode 100644 src/lzma.h create mode 100644 src/tests/lzma.cpp (limited to 'src/deflate.h') diff --git a/default.bld b/default.bld index 1b6eb4d..18a2d72 100644 --- a/default.bld +++ b/default.bld @@ -172,6 +172,11 @@ target ["tests/deflate"] LDFLAGS += "-lz"; } +target ["tests/lzma"] +{ + LDFLAGS += "-llzma"; +} + target ["tests/itoserver", "tests/socketblock", "tests/itoheap", "tests/itoqueue1", "tests/itoqueue2", "tests/conduit"] { diff --git a/src/deflate.h b/src/deflate.h index cab9b51..8ce283b 100644 --- a/src/deflate.h +++ b/src/deflate.h @@ -34,7 +34,7 @@ namespace Bu AutoGzip = 0x04|0x03 }; - Deflate( Bu::Stream &rNext, int nCompression=9, Format eFmt=AutoRaw ); + Deflate( Bu::Stream &rNext, int nCompression=-1, Format eFmt=AutoRaw ); virtual ~Deflate(); virtual void start(); diff --git a/src/lzma.cpp b/src/lzma.cpp new file mode 100644 index 0000000..6ed0806 --- /dev/null +++ b/src/lzma.cpp @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2007-2011 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/lzma.h" +#include "bu/trace.h" + +#include + +#define pState ((lzma_stream *)prState) + +using namespace Bu; + +Bu::Lzma::Lzma( Bu::Stream &rNext, int nCompression, Format eFmt ) : + Bu::Filter( rNext ), + prState( NULL ), + nCompression( nCompression ), + sTotalOut( 0 ), + eFmt( eFmt ), + bEos( false ) +{ + TRACE( nCompression ); + start(); +} + +Bu::Lzma::~Lzma() +{ + TRACE(); + stop(); +} + +void Bu::Lzma::start() +{ + TRACE(); + nBufSize = 64*1024; + pBuf = new char[nBufSize]; +} + +Bu::size Bu::Lzma::stop() +{ + TRACE(); + if( pState ) + { + if( bReading ) + { + lzma_end( pState ); + delete[] pBuf; + pBuf = NULL; + delete pState; + prState = NULL; + return 0; + } + else + { + for(;;) + { + pState->next_in = NULL; + pState->avail_in = 0; + pState->avail_out = nBufSize; + pState->next_out = (uint8_t *)pBuf; + int res = lzma_code( pState, LZMA_FINISH ); + if( pState->avail_out < nBufSize ) + { + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); + } + if( res == LZMA_STREAM_END ) + break; + } + lzma_end( pState ); + delete[] pBuf; + pBuf = NULL; + delete pState; + prState = NULL; + return sTotalOut; + } + } + return 0; +} + +void Bu::Lzma::lzmaError( int code ) +{ + TRACE( code ); + switch( code ) + { + case LZMA_OK: + case LZMA_STREAM_END: + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + break; + + case LZMA_MEM_ERROR: + throw ExceptionBase("Lzma: Memory allocation error."); + + case LZMA_MEMLIMIT_ERROR: + throw ExceptionBase("Lzma: Memory usage limit was reached."); + + case LZMA_FORMAT_ERROR: + throw ExceptionBase("Lzma: File format not recognized."); + + case LZMA_OPTIONS_ERROR: + throw ExceptionBase("Lzma: Invalid or unsupported options."); + + case LZMA_DATA_ERROR: + throw ExceptionBase("Lzma: Data is corrupt."); + + case LZMA_BUF_ERROR: + throw ExceptionBase("Lzma: No progress is possible."); + + case LZMA_PROG_ERROR: + throw ExceptionBase("Lzma: Programming error."); + + default: + throw ExceptionBase("Lzma: Unknown error encountered." ); + } +} + +Bu::size Bu::Lzma::read( void *pData, Bu::size nBytes ) +{ + TRACE( pData, nBytes ); + if( !pState ) + { + prState = new ::lzma_stream; + lzma_stream zEmpty = LZMA_STREAM_INIT; + Bu::memcpy( prState, &zEmpty, sizeof(lzma_stream) ); + + bReading = true; + lzmaError( lzma_auto_decoder( pState, UINT64_MAX, 0 ) ); + pState->next_in = (uint8_t *)pBuf; + pState->avail_in = 0; + } + if( bReading == false ) + throw ExceptionBase("This lzma filter is in writing mode, you can't read."); + + int nRead = 0; + int nReadTotal = pState->total_out; + pState->next_out = (uint8_t *)pData; + pState->avail_out = nBytes; + for(;;) + { + int ret = lzma_code( pState, LZMA_RUN ); + printf("inflate returned %d; avail in=%d, out=%d\n", ret, + pState->avail_in, pState->avail_out ); + + nReadTotal += nRead-pState->avail_out; + + if( ret == LZMA_STREAM_END ) + { + bEos = true; + if( pState->avail_in > 0 ) + { + if( rNext.isSeekable() ) + { + rNext.seek( -pState->avail_in ); + } + } + return nBytes-pState->avail_out; + } +// if( ret != LZMA_BUF_ERROR ) + lzmaError( ret ); + + if( pState->avail_out ) + { + if( pState->avail_in == 0 ) + { + nRead = rNext.read( pBuf, nBufSize ); + if( nRead == 0 && rNext.isEos() ) + { + throw Bu::ExceptionBase("Premature end of underlying " + "stream found reading deflate stream."); + } + pState->next_in = (uint8_t *)pBuf; + pState->avail_in = nRead; + } + } + else + { + return nBytes-pState->avail_out; + } + } + return 0; +} + +Bu::size Bu::Lzma::write( const void *pData, Bu::size nBytes ) +{ + TRACE( pData, nBytes ); + if( !pState ) + { + prState = new ::lzma_stream; + lzma_stream zEmpty = LZMA_STREAM_INIT; + Bu::memcpy( prState, &zEmpty, sizeof(lzma_stream) ); + + bReading = false; + if( eFmt == Xz ) + lzmaError( + lzma_easy_encoder( pState, nCompression, LZMA_CHECK_CRC64 ) + ); + else if( eFmt == LzmaAlone ) + { + lzma_options_lzma opt; + lzma_lzma_preset( &opt, nCompression ); + lzmaError( lzma_alone_encoder( pState, &opt ) ); + } + else + throw Bu::ExceptionBase("Invalid format for lzma."); + } + if( bReading == true ) + throw ExceptionBase("This lzma filter is in reading mode, you can't write."); + + pState->next_in = (uint8_t *)pData; + pState->avail_in = nBytes; + for(;;) + { + pState->avail_out = nBufSize; + pState->next_out = (uint8_t *)pBuf; + + lzmaError( lzma_code( pState, LZMA_RUN ) ); + + if( pState->avail_out < nBufSize ) + { + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); + } + if( pState->avail_in == 0 ) + break; + } + + return nBytes; +} + +bool Bu::Lzma::isOpen() +{ + TRACE(); + return (pState != NULL); +} + +bool Bu::Lzma::isEos() +{ + TRACE(); + return bEos; +} + +Bu::size Bu::Lzma::getCompressedSize() +{ + return sTotalOut; +} + diff --git a/src/lzma.h b/src/lzma.h new file mode 100644 index 0000000..21da6e8 --- /dev/null +++ b/src/lzma.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2007-2011 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#ifndef BU_LZMA_H +#define BU_LZMA_H + +#include + +#include "bu/filter.h" + +namespace Bu +{ + /** + * + *@ingroup Streams + */ + class Lzma : public Bu::Filter + { + public: + enum Format + { + Xz = 0x01, + LzmaAlone = 0x02, + }; + + Lzma( Bu::Stream &rNext, int nCompression=6, Format eFmt=Xz ); + virtual ~Lzma(); + + virtual void start(); + virtual Bu::size stop(); + virtual Bu::size read( void *pBuf, Bu::size nBytes ); + virtual Bu::size write( const void *pBuf, Bu::size nBytes ); + + virtual bool isOpen(); + virtual bool isEos(); + + Bu::size getCompressedSize(); + + private: + void lzmaError( int code ); + void *prState; + bool bReading; + int nCompression; + char *pBuf; + uint32_t nBufSize; + Bu::size sTotalOut; + Format eFmt; + bool bEos; + }; +} + +#endif diff --git a/src/tests/lzma.cpp b/src/tests/lzma.cpp new file mode 100644 index 0000000..752357a --- /dev/null +++ b/src/tests/lzma.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2007-2011 Xagasoft, All rights reserved. + * + * This file is part of the libbu++ library and is released under the + * terms of the license contained in the file LICENSE. + */ + +#include "bu/lzma.h" +#include "bu/file.h" + +int main( int argc, char *argv[] ) +{ + if( argc < 3 ) + { + printf("usage: %s \n", argv[0] ); + return -1; + } + + char buf[1024]; + size_t nRead; + + /* + Bu::File fin( argv[1], Bu::File::Read ); + fin.seek( 4 ); + Bu::Deflate def( fin ); + + Bu::File f( argv[2], Bu::File::WriteNew ); + + for(;;) + { + nRead = def.read( buf, 1024 ); + if( nRead > 0 ) + f.write( buf, nRead ); + if( def.isEos() ) + break; + } + */ + + Bu::File fin( argv[1], Bu::File::Read ); + + Bu::File f( argv[2], Bu::File::WriteNew ); + Bu::Lzma def( f, 9 ); + + for(;;) + { + nRead = fin.read( buf, 1024 ); + if( nRead > 0 ) + def.write( buf, nRead ); + if( fin.isEos() ) + break; + } +} + -- cgit v1.2.3 From 9cb2695ad318dcda83a353b03c21b4fd71d0f9d6 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 24 Oct 2011 16:30:39 +0000 Subject: Made the encoder state opaque to the caller in Deflate and BZip2 to match Lzma. That means that when you use Bu::Deflate, Bu::Bzip2, or Bu::Lzma you don't get any of the respective libraries' header files. --- src/bzip2.cpp | 95 ++++++++++++++++++++++++-------------------- src/bzip2.h | 3 +- src/deflate.cpp | 120 ++++++++++++++++++++++++++++++-------------------------- src/deflate.h | 3 +- 4 files changed, 120 insertions(+), 101 deletions(-) (limited to 'src/deflate.h') diff --git a/src/bzip2.cpp b/src/bzip2.cpp index 0ff5444..ca007b0 100644 --- a/src/bzip2.cpp +++ b/src/bzip2.cpp @@ -8,10 +8,15 @@ #include "bu/bzip2.h" #include "bu/trace.h" +#include + +#define pState ((bz_stream *)prState) + using namespace Bu; Bu::BZip2::BZip2( Bu::Stream &rNext, int nCompression ) : Bu::Filter( rNext ), + prState( NULL ), nCompression( nCompression ), sTotalOut( 0 ) { @@ -28,10 +33,12 @@ Bu::BZip2::~BZip2() void Bu::BZip2::start() { TRACE(); - bzState.state = NULL; - bzState.bzalloc = NULL; - bzState.bzfree = NULL; - bzState.opaque = NULL; + + prState = new bz_stream; + pState->state = NULL; + pState->bzalloc = NULL; + pState->bzfree = NULL; + pState->opaque = NULL; nBufSize = 64*1024; pBuf = new char[nBufSize]; @@ -40,13 +47,15 @@ void Bu::BZip2::start() Bu::size Bu::BZip2::stop() { TRACE(); - if( bzState.state ) + if( pState->state ) { if( bReading ) { - BZ2_bzDecompressEnd( &bzState ); + BZ2_bzDecompressEnd( pState ); delete[] pBuf; pBuf = NULL; + delete pState; + prState = NULL; return 0; } else @@ -54,21 +63,23 @@ Bu::size Bu::BZip2::stop() // Bu::size sTotal = 0; for(;;) { - bzState.next_in = NULL; - bzState.avail_in = 0; - bzState.avail_out = nBufSize; - bzState.next_out = pBuf; - int res = BZ2_bzCompress( &bzState, BZ_FINISH ); - if( bzState.avail_out < nBufSize ) + pState->next_in = NULL; + pState->avail_in = 0; + pState->avail_out = nBufSize; + pState->next_out = pBuf; + int res = BZ2_bzCompress( pState, BZ_FINISH ); + if( pState->avail_out < nBufSize ) { - sTotalOut += rNext.write( pBuf, nBufSize-bzState.avail_out ); + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); } if( res == BZ_STREAM_END ) break; } - BZ2_bzCompressEnd( &bzState ); + BZ2_bzCompressEnd( pState ); delete[] pBuf; pBuf = NULL; + delete pState; + prState = NULL; return sTotalOut; } } @@ -122,42 +133,42 @@ void Bu::BZip2::bzError( int code ) Bu::size Bu::BZip2::read( void *pData, Bu::size nBytes ) { TRACE( pData, nBytes ); - if( !bzState.state ) + if( !pState->state ) { bReading = true; - BZ2_bzDecompressInit( &bzState, 0, 0 ); - bzState.next_in = pBuf; - bzState.avail_in = 0; + BZ2_bzDecompressInit( pState, 0, 0 ); + pState->next_in = pBuf; + pState->avail_in = 0; } if( bReading == false ) throw ExceptionBase("This bzip2 filter is in writing mode, you can't read."); int nRead = 0; - int nReadTotal = bzState.total_out_lo32; - bzState.next_out = (char *)pData; - bzState.avail_out = nBytes; + int nReadTotal = pState->total_out_lo32; + pState->next_out = (char *)pData; + pState->avail_out = nBytes; for(;;) { - int ret = BZ2_bzDecompress( &bzState ); + int ret = BZ2_bzDecompress( pState ); - nReadTotal += nRead-bzState.avail_out; + nReadTotal += nRead-pState->avail_out; if( ret == BZ_STREAM_END ) { - if( bzState.avail_in > 0 ) + if( pState->avail_in > 0 ) { if( rNext.isSeekable() ) { - rNext.seek( -bzState.avail_in ); + rNext.seek( -pState->avail_in ); } } - return nBytes-bzState.avail_out; + return nBytes-pState->avail_out; } bzError( ret ); - if( bzState.avail_out ) + if( pState->avail_out ) { - if( bzState.avail_in == 0 ) + if( pState->avail_in == 0 ) { nRead = rNext.read( pBuf, nBufSize ); if( nRead == 0 && rNext.isEos() ) @@ -165,13 +176,13 @@ Bu::size Bu::BZip2::read( void *pData, Bu::size nBytes ) throw Bu::ExceptionBase("Premature end of underlying " "stream found reading bzip2 stream."); } - bzState.next_in = pBuf; - bzState.avail_in = nRead; + pState->next_in = pBuf; + pState->avail_in = nRead; } } else { - return nBytes-bzState.avail_out; + return nBytes-pState->avail_out; } } return 0; @@ -180,29 +191,29 @@ Bu::size Bu::BZip2::read( void *pData, Bu::size nBytes ) Bu::size Bu::BZip2::write( const void *pData, Bu::size nBytes ) { TRACE( pData, nBytes ); - if( !bzState.state ) + if( !pState->state ) { bReading = false; - BZ2_bzCompressInit( &bzState, nCompression, 0, 30 ); + BZ2_bzCompressInit( pState, nCompression, 0, 30 ); } if( bReading == true ) throw ExceptionBase("This bzip2 filter is in reading mode, you can't write."); // Bu::size sTotalOut = 0; - bzState.next_in = (char *)pData; - bzState.avail_in = nBytes; + pState->next_in = (char *)pData; + pState->avail_in = nBytes; for(;;) { - bzState.avail_out = nBufSize; - bzState.next_out = pBuf; + pState->avail_out = nBufSize; + pState->next_out = pBuf; - bzError( BZ2_bzCompress( &bzState, BZ_RUN ) ); + bzError( BZ2_bzCompress( pState, BZ_RUN ) ); - if( bzState.avail_out < nBufSize ) + if( pState->avail_out < nBufSize ) { - sTotalOut += rNext.write( pBuf, nBufSize-bzState.avail_out ); + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); } - if( bzState.avail_in == 0 ) + if( pState->avail_in == 0 ) break; } @@ -212,7 +223,7 @@ Bu::size Bu::BZip2::write( const void *pData, Bu::size nBytes ) bool Bu::BZip2::isOpen() { TRACE(); - return (bzState.state != NULL); + return (pState->state != NULL); } Bu::size Bu::BZip2::getCompressedSize() diff --git a/src/bzip2.h b/src/bzip2.h index 6da3dff..0b5140d 100644 --- a/src/bzip2.h +++ b/src/bzip2.h @@ -9,7 +9,6 @@ #define BU_BZIP2_H #include -#include #include "bu/filter.h" @@ -36,7 +35,7 @@ namespace Bu private: void bzError( int code ); - bz_stream bzState; + void *prState; bool bReading; int nCompression; char *pBuf; diff --git a/src/deflate.cpp b/src/deflate.cpp index aec2a18..10a9c5f 100644 --- a/src/deflate.cpp +++ b/src/deflate.cpp @@ -8,10 +8,15 @@ #include "bu/deflate.h" #include "bu/trace.h" +#include + +#define pState ((z_stream *)prState) + using namespace Bu; Bu::Deflate::Deflate( Bu::Stream &rNext, int nCompression, Format eFmt ) : Bu::Filter( rNext ), + prState( NULL ), nCompression( nCompression ), sTotalOut( 0 ), eFmt( eFmt ), @@ -30,10 +35,11 @@ Bu::Deflate::~Deflate() void Bu::Deflate::start() { TRACE(); - zState.zalloc = NULL; - zState.zfree = NULL; - zState.opaque = NULL; - zState.state = NULL; + prState = new z_stream; + pState->zalloc = NULL; + pState->zfree = NULL; + pState->opaque = NULL; + pState->state = NULL; nBufSize = 64*1024; pBuf = new char[nBufSize]; @@ -42,34 +48,38 @@ void Bu::Deflate::start() Bu::size Bu::Deflate::stop() { TRACE(); - if( zState.state ) + if( pState && pState->state ) { if( bReading ) { - inflateEnd( &zState ); + inflateEnd( pState ); delete[] pBuf; pBuf = NULL; + delete pState; + prState = NULL; return 0; } else { for(;;) { - zState.next_in = NULL; - zState.avail_in = 0; - zState.avail_out = nBufSize; - zState.next_out = (Bytef *)pBuf; - int res = deflate( &zState, Z_FINISH ); - if( zState.avail_out < nBufSize ) + pState->next_in = NULL; + pState->avail_in = 0; + pState->avail_out = nBufSize; + pState->next_out = (Bytef *)pBuf; + int res = deflate( pState, Z_FINISH ); + if( pState->avail_out < nBufSize ) { - sTotalOut += rNext.write( pBuf, nBufSize-zState.avail_out ); + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); } if( res == Z_STREAM_END ) break; } - deflateEnd( &zState ); + deflateEnd( pState ); delete[] pBuf; pBuf = NULL; + delete pState; + prState = NULL; return sTotalOut; } } @@ -87,25 +97,25 @@ void Bu::Deflate::zError( int code ) return; case Z_ERRNO: - throw ExceptionBase("Deflate: Errno - %s", zState.msg ); + throw ExceptionBase("Deflate: Errno - %s", pState->msg ); case Z_STREAM_ERROR: - throw ExceptionBase("Deflate: Stream Error - %s", zState.msg ); + throw ExceptionBase("Deflate: Stream Error - %s", pState->msg ); case Z_DATA_ERROR: - throw ExceptionBase("Deflate: Data Error - %s", zState.msg ); + throw ExceptionBase("Deflate: Data Error - %s", pState->msg ); case Z_MEM_ERROR: - throw ExceptionBase("Deflate: Mem Error - %s", zState.msg ); + throw ExceptionBase("Deflate: Mem Error - %s", pState->msg ); case Z_BUF_ERROR: - throw ExceptionBase("Deflate: Buf Error - %s", zState.msg ); + throw ExceptionBase("Deflate: Buf Error - %s", pState->msg ); case Z_VERSION_ERROR: - throw ExceptionBase("Deflate: Version Error - %s", zState.msg ); + throw ExceptionBase("Deflate: Version Error - %s", pState->msg ); default: - throw ExceptionBase("Deflate: Unknown error encountered - %s.", zState.msg ); + throw ExceptionBase("Deflate: Unknown error encountered - %s.", pState->msg ); } } @@ -113,55 +123,55 @@ void Bu::Deflate::zError( int code ) Bu::size Bu::Deflate::read( void *pData, Bu::size nBytes ) { TRACE( pData, nBytes ); - if( !zState.state ) + if( !pState->state ) { bReading = true; if( eFmt&AutoDetect ) - inflateInit2( &zState, 32+15 ); // Auto-detect, large window + inflateInit2( pState, 32+15 ); // Auto-detect, large window else if( eFmt == Raw ) - inflateInit2( &zState, -15 ); // Raw + inflateInit2( pState, -15 ); // Raw else if( eFmt == Zlib ) - inflateInit2( &zState, 15 ); // Zlib + inflateInit2( pState, 15 ); // Zlib else if( eFmt == Gzip ) - inflateInit2( &zState, 16+15 ); // GZip + inflateInit2( pState, 16+15 ); // GZip else throw Bu::ExceptionBase("Format mode for deflate read."); - zState.next_in = (Bytef *)pBuf; - zState.avail_in = 0; + pState->next_in = (Bytef *)pBuf; + pState->avail_in = 0; } if( bReading == false ) throw ExceptionBase("This deflate filter is in writing mode, you can't read."); int nRead = 0; - int nReadTotal = zState.total_out; - zState.next_out = (Bytef *)pData; - zState.avail_out = nBytes; + int nReadTotal = pState->total_out; + pState->next_out = (Bytef *)pData; + pState->avail_out = nBytes; for(;;) { - int ret = inflate( &zState, Z_NO_FLUSH ); + int ret = inflate( pState, Z_NO_FLUSH ); printf("inflate returned %d; avail in=%d, out=%d\n", ret, - zState.avail_in, zState.avail_out ); + pState->avail_in, pState->avail_out ); - nReadTotal += nRead-zState.avail_out; + nReadTotal += nRead-pState->avail_out; if( ret == Z_STREAM_END ) { bEos = true; - if( zState.avail_in > 0 ) + if( pState->avail_in > 0 ) { if( rNext.isSeekable() ) { - rNext.seek( -zState.avail_in ); + rNext.seek( -pState->avail_in ); } } - return nBytes-zState.avail_out; + return nBytes-pState->avail_out; } if( ret != Z_BUF_ERROR ) zError( ret ); - if( zState.avail_out ) + if( pState->avail_out ) { - if( zState.avail_in == 0 ) + if( pState->avail_in == 0 ) { nRead = rNext.read( pBuf, nBufSize ); if( nRead == 0 && rNext.isEos() ) @@ -169,13 +179,13 @@ Bu::size Bu::Deflate::read( void *pData, Bu::size nBytes ) throw Bu::ExceptionBase("Premature end of underlying " "stream found reading deflate stream."); } - zState.next_in = (Bytef *)pBuf; - zState.avail_in = nRead; + pState->next_in = (Bytef *)pBuf; + pState->avail_in = nRead; } } else { - return nBytes-zState.avail_out; + return nBytes-pState->avail_out; } } return 0; @@ -184,18 +194,18 @@ Bu::size Bu::Deflate::read( void *pData, Bu::size nBytes ) Bu::size Bu::Deflate::write( const void *pData, Bu::size nBytes ) { TRACE( pData, nBytes ); - if( !zState.state ) + if( !pState->state ) { bReading = false; int iFmt = eFmt&Gzip; if( iFmt == Raw ) - deflateInit2( &zState, nCompression, Z_DEFLATED, -15, 9, + deflateInit2( pState, nCompression, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY ); else if( iFmt == Zlib ) - deflateInit2( &zState, nCompression, Z_DEFLATED, 15, 9, + deflateInit2( pState, nCompression, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY ); else if( iFmt == Gzip ) - deflateInit2( &zState, nCompression, Z_DEFLATED, 16+15, 9, + deflateInit2( pState, nCompression, Z_DEFLATED, 16+15, 9, Z_DEFAULT_STRATEGY ); else throw Bu::ExceptionBase("Invalid format for deflate."); @@ -203,20 +213,20 @@ Bu::size Bu::Deflate::write( const void *pData, Bu::size nBytes ) if( bReading == true ) throw ExceptionBase("This deflate filter is in reading mode, you can't write."); - zState.next_in = (Bytef *)pData; - zState.avail_in = nBytes; + pState->next_in = (Bytef *)pData; + pState->avail_in = nBytes; for(;;) { - zState.avail_out = nBufSize; - zState.next_out = (Bytef *)pBuf; + pState->avail_out = nBufSize; + pState->next_out = (Bytef *)pBuf; - zError( deflate( &zState, Z_NO_FLUSH ) ); + zError( deflate( pState, Z_NO_FLUSH ) ); - if( zState.avail_out < nBufSize ) + if( pState->avail_out < nBufSize ) { - sTotalOut += rNext.write( pBuf, nBufSize-zState.avail_out ); + sTotalOut += rNext.write( pBuf, nBufSize-pState->avail_out ); } - if( zState.avail_in == 0 ) + if( pState->avail_in == 0 ) break; } @@ -226,7 +236,7 @@ Bu::size Bu::Deflate::write( const void *pData, Bu::size nBytes ) bool Bu::Deflate::isOpen() { TRACE(); - return (zState.state != NULL); + return (pState != NULL && pState->state != NULL); } bool Bu::Deflate::isEos() diff --git a/src/deflate.h b/src/deflate.h index 8ce283b..20d609a 100644 --- a/src/deflate.h +++ b/src/deflate.h @@ -9,7 +9,6 @@ #define BU_DEFLATE_H #include -#include #include "bu/filter.h" @@ -49,7 +48,7 @@ namespace Bu private: void zError( int code ); - z_stream zState; + void *prState; bool bReading; int nCompression; char *pBuf; -- cgit v1.2.3 From 2f163b90c745491a995dfdbe78464c64a536ec54 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Mon, 24 Oct 2011 16:42:22 +0000 Subject: Added some more docs. --- src/bzip2.h | 2 ++ src/deflate.h | 4 ++++ src/doxy/groups.dox | 5 +++++ src/lzma.h | 3 +++ 4 files changed, 14 insertions(+) (limited to 'src/deflate.h') diff --git a/src/bzip2.h b/src/bzip2.h index 0b5140d..9a8d172 100644 --- a/src/bzip2.h +++ b/src/bzip2.h @@ -15,8 +15,10 @@ namespace Bu { /** + * Provides BZip2 type compression and decompression. * *@ingroup Streams + *@ingroup Compression */ class BZip2 : public Bu::Filter { diff --git a/src/deflate.h b/src/deflate.h index 20d609a..34e8657 100644 --- a/src/deflate.h +++ b/src/deflate.h @@ -15,8 +15,12 @@ namespace Bu { /** + * Provides Deflate (LZ77) support via zlib. This provides zlib, raw, and + * gzip stream types. By default it will autodetect the input type and + * encode into a raw deflate stream. * *@ingroup Streams + *@ingroup Compression */ class Deflate : public Bu::Filter { diff --git a/src/doxy/groups.dox b/src/doxy/groups.dox index 479186a..6b7cd57 100644 --- a/src/doxy/groups.dox +++ b/src/doxy/groups.dox @@ -29,3 +29,8 @@ *@defgroup Streams Streams * Streams are for data. */ + +/** + *@defgroup Compression Compression + * Filters that give access to common compression libraries. + */ diff --git a/src/lzma.h b/src/lzma.h index 21da6e8..090da8d 100644 --- a/src/lzma.h +++ b/src/lzma.h @@ -15,8 +15,11 @@ namespace Bu { /** + * Provides XZ compression and decompression, both LZMA1 (LzmaAlone) as + * well as the newer LZMA2 (xz) format. This uses .xz by default. * *@ingroup Streams + *@ingroup Compression */ class Lzma : public Bu::Filter { -- cgit v1.2.3 From 052da60c2c5c4ce80ec0986ea07482348e7aa30a Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Tue, 25 Oct 2011 16:04:43 +0000 Subject: Base64 does line wrapping correctly on write, and also doesn't try to flush the write buffer when reading is done. It's...strange, but yeah, it was doing that. Deflate also defaults to zlib compression now, which means you can compress & decompress without using any extra params. Turns out zlib auto-detect won't decompress raw streams, so this is the safest overall option, and the easiest to work with. zlib headers are small, and includes a crc at the end so you can be sure your data is accurate, raw does not. --- default.bld | 4 ++-- src/base64.cpp | 45 +++++++++++++++++++++++++++++++++++++-------- src/base64.h | 4 +++- src/deflate.cpp | 3 --- src/deflate.h | 2 +- 5 files changed, 43 insertions(+), 15 deletions(-) (limited to 'src/deflate.h') diff --git a/default.bld b/default.bld index 18a2d72..130c1a2 100644 --- a/default.bld +++ b/default.bld @@ -162,12 +162,12 @@ target files("src/tests/*.cpp").replace("src/","").replace(".cpp","") // Some tests need extra libs and whatnot, that goes here. -target ["tests/bzip2", "tests/streamstack", "tests/enc"] +target ["tests/bzip2", "tests/streamstack"] { LDFLAGS += "-lbz2"; } -target ["tests/deflate"] +target ["tests/deflate", "tests/enc"] { LDFLAGS += "-lz"; } diff --git a/src/base64.cpp b/src/base64.cpp index 04ca009..4d659f0 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -13,7 +13,7 @@ const char Bu::Base64::tblEnc[65] = { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" }; -Bu::Base64::Base64( Bu::Stream &rNext ) : +Bu::Base64::Base64( Bu::Stream &rNext, int iChunkSize ) : Bu::Filter( rNext ), iBPos( 0 ), iBuf( 0 ), @@ -22,7 +22,9 @@ Bu::Base64::Base64( Bu::Stream &rNext ) : bEosIn( false ), iTotalIn( 0 ), iTotalOut( 0 ), - eMode( Nothing ) + eMode( Nothing ), + iChunkSize( iChunkSize ), + iCurChunk( 0 ) { start(); @@ -62,11 +64,12 @@ Bu::Base64::~Base64() void Bu::Base64::start() { + iCurChunk = 0; } Bu::size Bu::Base64::stop() { -// if( eMode |= Encode ) + if( eMode == Encode ) { char outBuf[4]; int iBUsed = 4-(3-iBPos); @@ -80,17 +83,30 @@ Bu::size Bu::Base64::stop() { outBuf[k] = '='; } - iTotalOut += rNext.write( outBuf, 4 ); + iCurChunk += 4; + if( iChunkSize && iCurChunk >= iChunkSize ) + { + iCurChunk = iCurChunk-iChunkSize; + iTotalOut += rNext.write( outBuf, 4-iCurChunk ); + iTotalOut += rNext.write("\r\n", 2 ); + iTotalOut += rNext.write( outBuf+(4-iCurChunk), iCurChunk ); + } + else + iTotalOut += rNext.write( outBuf, 4 ); return iTotalOut; } -// else -// { + else + { return iTotalIn; -// } + } } Bu::size Bu::Base64::read( void *pBuf, Bu::size nBytes ) { + if( eMode == Encode ) + throw Bu::Base64Exception("Cannot read from an output stream."); + eMode = Decode; + if( bEosIn == true && iRPos == iChars ) return 0; Bu::size sIn = 0; @@ -157,6 +173,10 @@ Bu::size Bu::Base64::read( void *pBuf, Bu::size nBytes ) Bu::size Bu::Base64::write( const void *pBuf, Bu::size nBytes ) { + if( eMode == Decode ) + throw Bu::Base64Exception("Cannot write to an input stream."); + eMode = Encode; + Bu::size sOut = 0; char outBuf[4]; for( Bu::size j = 0; j < nBytes; j++ ) @@ -168,7 +188,16 @@ Bu::size Bu::Base64::write( const void *pBuf, Bu::size nBytes ) { outBuf[3-k] = tblEnc[(iBuf>>(6*k))&0x3f]; } - sOut += rNext.write( outBuf, 4 ); + iCurChunk += 4; + if( iChunkSize && iCurChunk >= iChunkSize ) + { + iCurChunk = iCurChunk-iChunkSize; + sOut += rNext.write( outBuf, 4-iCurChunk ); + sOut += rNext.write("\r\n", 2 ); + sOut += rNext.write( outBuf+(4-iCurChunk), iCurChunk ); + } + else + sOut += rNext.write( outBuf, 4 ); iBPos = iBuf = 0; } } diff --git a/src/base64.h b/src/base64.h index 53d7860..c081ac1 100644 --- a/src/base64.h +++ b/src/base64.h @@ -22,7 +22,7 @@ namespace Bu class Base64 : public Bu::Filter { public: - Base64( Bu::Stream &rNext ); + Base64( Bu::Stream &rNext, int iChunkSize=0 ); virtual ~Base64(); virtual void start(); @@ -51,6 +51,8 @@ namespace Bu Decode = 0x02, }; Mode eMode; + int iChunkSize; + int iCurChunk; }; }; diff --git a/src/deflate.cpp b/src/deflate.cpp index 10a9c5f..2d925a7 100644 --- a/src/deflate.cpp +++ b/src/deflate.cpp @@ -149,9 +149,6 @@ Bu::size Bu::Deflate::read( void *pData, Bu::size nBytes ) for(;;) { int ret = inflate( pState, Z_NO_FLUSH ); - printf("inflate returned %d; avail in=%d, out=%d\n", ret, - pState->avail_in, pState->avail_out ); - nReadTotal += nRead-pState->avail_out; if( ret == Z_STREAM_END ) diff --git a/src/deflate.h b/src/deflate.h index 34e8657..f835cfc 100644 --- a/src/deflate.h +++ b/src/deflate.h @@ -37,7 +37,7 @@ namespace Bu AutoGzip = 0x04|0x03 }; - Deflate( Bu::Stream &rNext, int nCompression=-1, Format eFmt=AutoRaw ); + Deflate( Bu::Stream &rNext, int nCompression=-1, Format eFmt=AutoZlib ); virtual ~Deflate(); virtual void start(); -- cgit v1.2.3