bytecompr.h

Go to the documentation of this file.

/////////////////////////////////////////////////////////////////////////////
///
/// @file bytecompr.h
///
/// Writing and reading binary data (int, double) with bzip2 compression.
/// This file contains the definition of two classes "ByteCompressor"
/// and "ByteDecompressor" which serve as simple wrappers for putting
/// binary data into a previously opened binary stream, with compression,
/// and getting the data back from a previously compressed stream.
/// This may be useful e.g. for saving a memory dump to a bz2 file,
/// or for saving and then retrieving temporary binary data structures
/// to a bz2 compressed file.
/// Note that the data is not portable, the sizes of types as well
/// as byte order may depend on a particular architecture.
/// These data structures are designed for storing temporary data
/// and re-using it from the same (binary) program. Use with caution.
///
/// @author Pawel Pilarczyk
///
/////////////////////////////////////////////////////////////////////////////
 
// Copyright (C) 1997-2014 by Pawel Pilarczyk.
//
// This file is part of my research software package. This is free software:
// you can redistribute it and/or modify it under the terms of the GNU
// General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// This software is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this software; see the file "license.txt". If not,
// please, see <https://www.gnu.org/licenses/>.
 
// Started on July 2, 2014. Last revision: July 7, 2014.
 
 
#ifndef _PWP_BYTECOMPR_H_
#define _PWP_BYTECOMPR_H_
 
#include <iostream>
#include <bzlib.h>
 
// include selected header files from the CHomP library
// (this is temporary and serves the debugging purpose only)
//#include "chomp/system/textfile.h"
//using chomp::homology::sout;
 
 
// --------------------------------------------------
// ---------------- Byte Compressor -----------------
// --------------------------------------------------
 
/// A simple wrapper for the bzip2 data compression to an output stream.
class ByteCompressor
{
public:
        /// The constructor of an output stream for storing binary data.
        /// It also initializes the bzip2 compression structure.
        ByteCompressor (std::ostream &out);
 
        /// The destructor that ends the compression procedure
        /// and releases the internal memory buffers.
        ~ByteCompressor ();
 
        /// Ends the compression and writes the remaining data to the stream.
        /// It also deletes the bzip2 compression structure.
        void close ();
 
        /// Writes the given number of bytes to the compression stream.
        /// Uses an internal buffer to avoid the inefficiency that comes
        /// from passing small portions of data to the bzip2 procedures.
        void write (const char *buf, size_t n);
 
private:
        /// The bzip2 data stream information.
        bz_stream *bzStream;
 
        /// A reference to the output stream to which the data
        /// should be written.
        std::ostream &outStream;
 
        /// An internal buffer for caching data for compression.
        char *inBuffer;
 
        /// An internal buffer for caching compressed data.
        char *outBuffer;
 
        /// The size of each of the internal buffers.
        size_t bufSize;
 
        /// The current position in the internal buffer for compression.
        size_t inPos;
 
        /// Compresses the input buffer and writes the resulting data
        /// to the output stream.
        void compress (bool finishing);
 
        // Copy constructor is not allowed.
        ByteCompressor (const ByteCompressor &);
 
        // Assignement operator is not allowed.
        ByteCompressor &operator = (const ByteCompressor &);
 
}; /* class ByteCompressor */
 
// --------------------------------------------------
 
inline ByteCompressor::ByteCompressor (std::ostream &out):
        outStream (out)
{
        bufSize = 102400;
        inBuffer = new char [bufSize];
        outBuffer = new char [bufSize];
        inPos = 0;
        bzStream = new bz_stream;
        bzStream -> next_in = inBuffer;
        bzStream -> avail_in = 0;
        bzStream -> total_in_lo32 = 0;
        bzStream -> total_in_hi32 = 0;
        bzStream -> next_out = outBuffer;
        bzStream -> avail_out = bufSize;
        bzStream -> total_out_lo32 = 0;
        bzStream -> total_out_hi32 = 0;
        bzStream -> state = 0;
        bzStream -> bzalloc = 0;
        bzStream -> bzfree = 0;
        bzStream -> opaque = 0;
        int code = BZ2_bzCompressInit (bzStream, 9, 0, 0);
        switch (code)
        {
        case BZ_CONFIG_ERROR:
                throw "BZ2_bzCompressInit config error.";
        case BZ_PARAM_ERROR:
                throw "BZ2_bzCompressInit wrong parameters.";
        case BZ_MEM_ERROR:
                throw "BZ2_bzCompressInit memory error.";
        case BZ_OK:
                break;
        default:
                throw "BZ2_bzCompressInit returned an unknown code.";
        }
        return;
} /* ByteCompressor::ByteCompressor */
 
inline void ByteCompressor::compress (bool finishing)
{
//      sout << "DEBUG: compress ().\n";
        if (!bzStream)
                return;
 
        // compress all the data gathered in the input buffer
        size_t startPos (0);
        bool finished (false);
        while ((finishing && !finished) || (startPos < inPos))
        {
        //      sout << "DEBUG: inPos = " << inPos << ".\n";
        //      sout << "DEBUG: startPos = " << startPos << ".\n";
        //      sout << "DEBUG: finishing = " << finishing << ".\n";
        //      sout << "DEBUG: finished = " << finished << ".\n";
                // prepare buffer information in the bz2 data structure
                bzStream -> next_in = inBuffer + startPos;
                bzStream -> avail_in = inPos - startPos;
                bzStream -> next_out = outBuffer;
                bzStream -> avail_out = bufSize;
 
                // compress the input buffer to the output buffer
                int code = BZ2_bzCompress (bzStream,
                        finishing ? BZ_FINISH : BZ_RUN);
                switch (code)
                {
                case BZ_SEQUENCE_ERROR:
                        throw "BZ2_bzCompress sequence error.";
                case BZ_STREAM_END:
                        finished = true;
                        if (finishing)
                                break;
                        throw "BZ2_bzCompress unexpected stream end.";
                case BZ_OK:
                case BZ_RUN_OK:
                case BZ_FINISH_OK:
                        break;
                default:
                //      sout << "DEBUG: code = " << code << ".\n";
                        throw "BZ2_bzCompress returned an unknown code.";
                }
 
                // write the output to the output stream
                size_t outSize (bzStream -> next_out - outBuffer);
                if (outSize)
                        outStream. write (outBuffer, outSize);
 
                // update the reading offset in the input buffer
                startPos = bzStream -> next_in - inBuffer;
 
                // exit the loop if this is the end of the procedure
                if (finishing && finished)
                        break;
        }
 
        // make a note that the input buffer is now empty
        inPos = 0;
 
        return;
} /* ByteCompressor::compress */
 
void ByteCompressor::write (const char *buf, size_t n)
{
//      sout << "DEBUG: write " << n << " byte(s).\n";
        if (n > bufSize)
                throw "Compression buffer size too small. Fix the code!";
        if (inPos + n > bufSize)
                compress (false);
        if (inPos + n > bufSize)
                throw "Not enough buffer space to compress data.";
        for (size_t i = 0; i < n; ++ i)
                inBuffer [inPos ++] = buf [i];
        return;
} /* ByteCompressor::write */
 
inline void ByteCompressor::close ()
{
//      sout << "DEBUG: close ().\n";
        if (!bzStream)
                return;
 
        // finish off compressing all the data
        this -> compress (true);
 
        // release all memory associated with the compression stream
        int code = BZ2_bzCompressEnd (bzStream);
        switch (code)
        {
        case BZ_PARAM_ERROR:
                throw "BZ2_bzCompressEnd: wrong bz stream data structure";
        case BZ_OK:
                break;
        default:
                throw "BZ2_bzCompressEnd returned an unknown code.";
        }
 
        // delete the compression structure
        delete bzStream;
        bzStream = 0;
 
        return;
} /* ByteCompressor::close */
 
inline ByteCompressor::~ByteCompressor ()
{
//      sout << "DEBUG: destructor.\n";
        this -> close ();
        if (bzStream)
                delete bzStream;
        if (inBuffer)
                delete [] inBuffer;
        if (outBuffer)
                delete [] outBuffer;
        return;
} /* ByteCompressor::~ByteCompressor */
 
// --------------------------------------------------
 
/// Writes a variable to a byte compressor stream.
#define DEFBYTEWRITE(var_type) \
inline ByteCompressor &operator << (ByteCompressor &c, const var_type &n) \
{ \
        c. write (reinterpret_cast<const char *> (&n), sizeof (var_type)); \
        return c; \
} /* operator << */
 
DEFBYTEWRITE(unsigned int)
DEFBYTEWRITE(signed int)
DEFBYTEWRITE(unsigned long)
DEFBYTEWRITE(signed long)
DEFBYTEWRITE(unsigned short)
DEFBYTEWRITE(signed short)
DEFBYTEWRITE(unsigned char)
DEFBYTEWRITE(char)
DEFBYTEWRITE(float)
DEFBYTEWRITE(double)
 
#undef DEFBYTEWRITE
 
 
// --------------------------------------------------
// --------------- Byte Decompressor ----------------
// --------------------------------------------------
 
/// A simple wrapper for the bzip2 data decompression from an input stream.
class ByteDecompressor
{
public:
        /// The constructor of an input stream for retrieving binary data.
        /// It also initializes the bzip2 decompression structure.
        ByteDecompressor (std::istream &in);
 
        /// The destructor that ends the decompression procedure
        /// and releases the internal memory buffers.
        ~ByteDecompressor ();
 
        /// Ends the decompression and deletes the bzip2 structure.
        void close ();
 
        /// Reads the given number of bytes from the decompressed stream.
        /// Uses an internal buffer to avoid the inefficiency that comes
        /// from taking small portions of data from the bzip2 procedures.
        /// Throws an exception if there is not enough data.
        void read (char *buf, size_t n);
 
private:
        /// The bzip2 data stream information.
        bz_stream *bzStream;
 
        /// A reference to the input stream from which the compressed data
        /// should be read.
        std::istream &inStream;
 
        /// An internal buffer for caching data read from the input stream.
        char *inBuffer;
 
        /// An internal buffer for caching decompressed data.
        char *outBuffer;
 
        /// The size of each of the internal buffers.
        size_t bufSize;
 
        /// Is the decompression procedure completed?
        bool finished;
 
        /// The current position in the internal buffer with compressed data.
        size_t inPos;
 
        /// The end of the compressed data in the internal buffer.
        size_t inPosEnd;
 
        /// The current position in the internal buffer
        /// with decompressed data.
        size_t outPos;
 
        /// The position of the end of the decompressed data
        /// in the internal buffer.
        size_t outPosEnd;
 
        /// Decompresses data from the input buffer (and the input stream)
        /// to provide more data for reading if possible.
        void decompress ();
 
        // Copy constructor is not allowed.
        ByteDecompressor (const ByteDecompressor &);
 
        // Assignement operator is not allowed.
        ByteDecompressor &operator = (const ByteDecompressor &);
 
}; /* class ByteDecompressor */
 
// --------------------------------------------------
 
inline ByteDecompressor::ByteDecompressor (std::istream &in):
        inStream (in)
{
        bufSize = 102400;
        finished = false;
        inBuffer = new char [bufSize];
        outBuffer = new char [bufSize];
        inPos = 0;
        inPosEnd = 0;
        outPos = 0;
        outPosEnd = 0;
        bzStream = new bz_stream;
        bzStream -> next_in = inBuffer;
        bzStream -> avail_in = 0;
        bzStream -> total_in_lo32 = 0;
        bzStream -> total_in_hi32 = 0;
        bzStream -> next_out = outBuffer;
        bzStream -> avail_out = bufSize;
        bzStream -> total_out_lo32 = 0;
        bzStream -> total_out_hi32 = 0;
        bzStream -> state = 0;
        bzStream -> bzalloc = 0;
        bzStream -> bzfree = 0;
        bzStream -> opaque = 0;
        int code = BZ2_bzDecompressInit (bzStream, 0, 0);
        switch (code)
        {
        case BZ_CONFIG_ERROR:
                throw "BZ2_bzDecompressInit config error.";
        case BZ_PARAM_ERROR:
                throw "BZ2_bzDecompressInit wrong parameters.";
        case BZ_MEM_ERROR:
                throw "BZ2_bzDecompressInit memory error.";
        case BZ_OK:
                break;
        default:
                throw "BZ2_bzDecompressInit returned an unknown code.";
        }
        return;
} /* ByteDecompressor::ByteDecompressor */
 
inline void ByteDecompressor::decompress ()
{
//      sout << "DEBUG: decompress ().\n";
        if (!bzStream)
                return;
 
        // if there is no data in the output buffer then make sure
        // that the writing position is reset to the buffer start
        if (outPos && (outPos == outPosEnd))
        {
                outPos = 0;
                outPosEnd = 0;
        }
 
        // if the output data is close to the end of the buffer
        // then move it to the front of the buffer to make sure
        // that there is a lot of space available for new data
        if (bufSize - outPos < (bufSize >> 8))
        {
                char *outBegin = outBuffer;
                char *outCurrent = outBuffer + outPos;
                for (size_t i = outPos; i < outPosEnd; ++ i)
                        *(outBegin ++) = *(outCurrent ++);
                outPosEnd -= outPos;
                outPos = 0;
        }
 
        while (!finished && (outPosEnd < bufSize))
        {
        //      sout << "DEBUG: inPos = " << inPos << ".\n";
        //      sout << "DEBUG: inPosEnd = " << inPosEnd << ".\n";
        //      sout << "DEBUG: finished = " << finished << ".\n";
 
                // if there is no data in the input buffer then make sure
                // that the reading position is reset to the buffer start
                if (inPos && (inPos == inPosEnd))
                {
                        inPos = 0;
                        inPosEnd = 0;
                }
 
                // if the input data is close to the end of the buffer
                // then move it to the front of the buffer to make sure
                // that there is a lot of space available for new data
                if (bufSize - inPos < (bufSize >> 8))
                {
                        char *inBegin = inBuffer;
                        char *inCurrent = inBuffer + inPos;
                        for (size_t i = inPos; i < inPosEnd; ++ i)
                                *(inBegin ++) = *(inCurrent ++);
                        inPosEnd -= inPos;
                        inPos = 0;
                }
 
                // read more data from the input stream if possible
                if (!inStream. eof () && (inPosEnd < bufSize))
                {
                        inStream. read (inBuffer + inPosEnd,
                                bufSize - inPosEnd);
                        inPosEnd += inStream. gcount ();
                }
 
                // prepare buffer information in the bz2 data structure
                bzStream -> next_in = inBuffer + inPos;
                bzStream -> avail_in = inPosEnd - inPos;
                bzStream -> next_out = outBuffer + outPosEnd;
                bzStream -> avail_out = bufSize - outPosEnd;
 
                // decompress the input buffer to the output buffer
                int code = BZ2_bzDecompress (bzStream);
                switch (code)
                {
                case BZ_PARAM_ERROR:
                        throw "BZ2_bzDecompress parameter error.";
                case BZ_DATA_ERROR:
                        throw "BZ2_bzDecompress data integrity error.";
                case BZ_DATA_ERROR_MAGIC:
                        throw "BZ2_bzDecompress magic data error.";
                case BZ_MEM_ERROR:
                        throw "BZ2_bzDecompress not enough memory available.";
                case BZ_STREAM_END:
                        finished = true;
                        break;
                case BZ_OK:
                        break;
                default:
                //      sout << "DEBUG: code = " << code << ".\n";
                        throw "BZ2_bzDecompress returned an unknown code.";
                }
 
                // update the reading offset in the input buffer
                inPos = bzStream -> next_in - inBuffer;
 
                // update the offset in the output buffer
                outPosEnd = bzStream -> next_out - outBuffer;
        }
 
        return;
} /* ByteDecompressor::decompress */
 
void ByteDecompressor::read (char *buf, size_t n)
{
//      sout << "DEBUG: read " << n << " byte(s).\n";
        if (n > bufSize)
                throw "Compression buffer size too small. Fix the code!";
        if (outPos + n > outPosEnd)
                decompress ();
        if (outPos + n > outPosEnd)
                throw "Not enough data read from the compressed stream.";
        for (size_t i = 0; i < n; ++ i)
                buf [i] = outBuffer [outPos ++];
        return;
} /* ByteDecompressor::read */
 
inline void ByteDecompressor::close ()
{
//      sout << "DEBUG: close ().\n";
        if (!bzStream)
                return;
 
        // release all memory associated with the compression stream
        int code = BZ2_bzDecompressEnd (bzStream);
        switch (code)
        {
        case BZ_PARAM_ERROR:
                throw "BZ2_bzDecompressEnd: wrong bz stream data structure";
        case BZ_OK:
                break;
        default:
                throw "BZ2_bzDecompressEnd returned an unknown code.";
        }
 
        // delete the compression structure
        delete bzStream;
        bzStream = 0;
 
        return;
} /* ByteDecompressor::close */
 
inline ByteDecompressor::~ByteDecompressor ()
{
//      sout << "DEBUG: destructor.\n";
        this -> close ();
        if (bzStream)
                delete bzStream;
        if (inBuffer)
                delete [] inBuffer;
        if (outBuffer)
                delete [] outBuffer;
        return;
} /* ByteDecompressor::~ByteDecompressor */
 
// --------------------------------------------------
 
/// Reads a variable from a byte decompressor stream.
#define DEFBYTEREAD(var_type) \
inline ByteDecompressor &operator >> (ByteDecompressor &c, var_type &n) \
{ \
        c. read (reinterpret_cast<char *> (&n), sizeof (var_type)); \
        return c; \
} /* operator >> */
 
DEFBYTEREAD(unsigned int)
DEFBYTEREAD(signed int)
DEFBYTEREAD(unsigned long)
DEFBYTEREAD(signed long)
DEFBYTEREAD(unsigned short)
DEFBYTEREAD(signed short)
DEFBYTEREAD(unsigned char)
DEFBYTEREAD(char)
DEFBYTEREAD(float)
DEFBYTEREAD(double)
 
#undef DEFBYTEREAD
 
 
#endif // _PWP_BYTECOMPR_H_