compression.h

Go to the documentation of this file.

 
#include <functional>
#include <limits>
#include <stdexcept>
#include <string>
 
#ifndef QB_IO_COMPRESSION_H
#define QB_IO_COMPRESSION_H
 
#ifndef QB_IO_WITH_ZLIB
#error "missing Z Library"
#endif
 
#include <zlib.h>
#include "qb/system/allocator/pipe.h"
 
namespace qb {
namespace compression {

enum operation_hint {
    is_last, 
    has_more 
};

struct operation_result {
    std::size_t
        input_bytes_processed; 
    std::size_t
         output_bytes_produced; 
    bool done; 
};

class compress_provider {
public:
    virtual const std::string &algorithm() const = 0;

    virtual std::size_t compress(const uint8_t *input, std::size_t input_size,
                                 uint8_t *output, std::size_t output_size,
                                 operation_hint hint, std::size_t &input_bytes_processed,
                                 bool &done) = 0;

    virtual void reset() = 0;

    virtual ~compress_provider() = default;
};

class decompress_provider {
public:
    virtual const std::string &algorithm() const = 0;

    virtual std::size_t decompress(const uint8_t *input, std::size_t input_size,
                                   uint8_t *output, std::size_t output_size,
                                   operation_hint hint,
                                   std::size_t &input_bytes_processed, bool &done) = 0;

    virtual void reset() = 0;

    virtual ~decompress_provider() = default;
};

class compress_factory {
public:
    virtual const std::string &algorithm() const = 0;

    virtual std::unique_ptr<compress_provider> make_compressor() const = 0;

    virtual ~compress_factory() = default;
};

class decompress_factory {
public:
    virtual const std::string &algorithm() const = 0;

    virtual uint16_t weight() const = 0;

    virtual std::unique_ptr<decompress_provider> make_decompressor() const = 0;

    virtual ~decompress_factory() = default;
};

namespace builtin {

bool supported();

namespace algorithm {

constexpr const char *const GZIP = "gzip";

constexpr const char *const DEFLATE = "deflate";

bool supported(const std::string &algorithm);
} // namespace algorithm

std::unique_ptr<compress_provider> make_compressor(const std::string &algorithm);

std::unique_ptr<decompress_provider> make_decompressor(const std::string &algorithm);

const std::vector<std::shared_ptr<compress_factory>> get_compress_factories();

std::shared_ptr<compress_factory> get_compress_factory(const std::string &algorithm);

const std::vector<std::shared_ptr<decompress_factory>> get_decompress_factories();

std::shared_ptr<decompress_factory> get_decompress_factory(const std::string &algorithm);

std::unique_ptr<compress_provider> make_gzip_compressor(int compressionLevel, int method,
                                                        int strategy, int memLevel);

std::unique_ptr<compress_provider>
make_deflate_compressor(int compressionLevel, int method, int strategy, int memLevel);
 
} // namespace builtin

std::shared_ptr<compress_factory> make_compress_factory(
    const std::string                                  &algorithm,
    std::function<std::unique_ptr<compress_provider>()> make_compressor);

std::shared_ptr<decompress_factory> make_decompress_factory(
    const std::string &algorithm, uint16_t weight,
    std::function<std::unique_ptr<decompress_provider>()> make_decompressor);

template <typename Output>
size_t
compress(Output &output, const char *data, std::size_t size, int level,
         int window_bits) {
#ifdef DEBUG
    // Verify if size input will fit into unsigned int, type used for zlib's avail_in
    if (size > std::numeric_limits<unsigned int>::max()) {
        throw std::runtime_error("size arg is too large to fit into unsigned int type");
    }
#endif
 
    z_stream deflate_s;
    deflate_s.zalloc   = Z_NULL;
    deflate_s.zfree    = Z_NULL;
    deflate_s.opaque   = Z_NULL;
    deflate_s.avail_in = 0;
    deflate_s.next_in  = Z_NULL;
 
    // The windowBits parameter is the base two logarithm of the window size (the size of
    // the history buffer). It should be in the range 8..15 for this version of the
    // library. Larger values of this parameter result in better compression at the
    // expense of memory usage. This range of values also changes the decoding type:
    //  -8 to -15 for raw deflate
    //  8 to 15 for zlib
    // (8 to 15) + 16 for gzip
    // (8 to 15) + 32 to automatically detect gzip/zlib header
    constexpr int mem_level = 8;
    // The memory requirements for deflate are (in bytes):
    // (1 << (window_bits+2)) +  (1 << (mem_level+9))
    // with a default value of 8 for mem_level and our window_bits of 15
    // this is 128Kb
 
    DISABLE_WARNING_PUSH
    DISABLE_WARNING_OLD_STYLE_CAST
    if (deflateInit2(&deflate_s, level, Z_DEFLATED, window_bits, mem_level,
                     Z_DEFAULT_STRATEGY) != Z_OK) {
        throw std::runtime_error("deflate init failed");
    }
    DISABLE_WARNING_POP
 
    deflate_s.next_in  = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
    deflate_s.avail_in = static_cast<unsigned int>(size);
 
    std::size_t size_compressed = 0;
    do {
        size_t increase = size / 2 + 1024;
        if (output.size() < (size_compressed + increase)) {
            output.resize(size_compressed + increase);
        }
        // There is no way we see that "increase" would not fit in an unsigned int,
        // hence we use static cast here to avoid -Wshorten-64-to-32 error
        deflate_s.avail_out = static_cast<unsigned int>(increase);
        deflate_s.next_out  = reinterpret_cast<Bytef *>((&output[0] + size_compressed));
        // From http://www.zlib.net/zlib_how.html
        // "deflate() has a return value that can indicate errors, yet we do not check it
        // here. Why not? Well, it turns out that deflate() can do no wrong here."
        // Basically only possible error is from deflateInit not working properly
        ::deflate(&deflate_s, Z_FINISH);
        size_compressed += (increase - deflate_s.avail_out);
    } while (deflate_s.avail_out == 0);
 
    deflateEnd(&deflate_s);
    output.resize(size_compressed);
    return size_compressed;
}

template <>
size_t compress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                int level, int window_bits);

template <typename Output>
std::size_t
uncompress(Output &output, const char *data, std::size_t size, std::size_t max,
           int window_bits) {
    z_stream inflate_s;
 
    inflate_s.zalloc   = Z_NULL;
    inflate_s.zfree    = Z_NULL;
    inflate_s.opaque   = Z_NULL;
    inflate_s.avail_in = 0;
    inflate_s.next_in  = Z_NULL;
 
    // The windowBits parameter is the base two logarithm of the window size (the size of
    // the history buffer). It should be in the range 8..15 for this version of the
    // library. Larger values of this parameter result in better compression at the
    // expense of memory usage. This range of values also changes the decoding type:
    //  -8 to -15 for raw deflate
    //  8 to 15 for zlib
    // (8 to 15) + 16 for gzip
    // (8 to 15) + 32 to automatically detect gzip/zlib header
 
    DISABLE_WARNING_PUSH
    DISABLE_WARNING_OLD_STYLE_CAST
    if (inflateInit2(&inflate_s, window_bits) != Z_OK) {
        throw std::runtime_error("inflate init failed");
    }
    DISABLE_WARNING_POP
    inflate_s.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
 
#ifdef DEBUG
    // Verify if size (long type) input will fit into unsigned int, type used for zlib's
    // avail_in
    std::uint64_t size_64 = size * 2;
    if (size_64 > std::numeric_limits<unsigned int>::max()) {
        inflateEnd(&inflate_s);
        throw std::runtime_error(
            "size arg is too large to fit into unsigned int type x2");
    }
#endif
    if (max && (size > max || (size * 2) > max)) {
        inflateEnd(&inflate_s);
        throw std::runtime_error(
            "size may use more memory than intended when decompressing");
    }
    inflate_s.avail_in            = static_cast<unsigned int>(size);
    std::size_t size_uncompressed = 0;
    do {
        std::size_t resize_to = size_uncompressed + 2 * size;
        if (max && resize_to > max) {
            inflateEnd(&inflate_s);
            throw std::runtime_error("size of output string will use more memory then "
                                     "intended when decompressing");
        }
        output.resize(resize_to);
        inflate_s.avail_out = static_cast<unsigned int>(2 * size);
        inflate_s.next_out  = reinterpret_cast<Bytef *>(&output[0] + size_uncompressed);
        int ret             = inflate(&inflate_s, Z_FINISH);
        if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
            std::string error_msg = inflate_s.msg;
            inflateEnd(&inflate_s);
            throw std::runtime_error(error_msg);
        }
 
        size_uncompressed += (2 * size - inflate_s.avail_out);
    } while (inflate_s.avail_out == 0);
    inflateEnd(&inflate_s);
    output.resize(size_uncompressed);
    return size_uncompressed;
}

template <>
size_t uncompress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                  std::size_t max, int window_bits);

namespace deflate {

template <typename Output>
size_t
compress(Output &output, const char *data, std::size_t size,
         int level = Z_DEFAULT_COMPRESSION) {
    constexpr int window_bits = 15; // gzip with windowbits of 15
    return compression::compress(output, data, size, level, window_bits);
}

template <>
size_t compress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                int level);

struct to_compress {
    const char *data;                          
    std::size_t size;                          
    int         level = Z_DEFAULT_COMPRESSION; 
    size_t      size_compressed;               
};

template <typename Output>
Output &
compress(Output &output, to_compress &info) {
    info.size_compressed = compress<Output>(output, info.data, info.size, info.level);
    return output;
}

std::string compress(const char *data, std::size_t size,
                     int level = Z_DEFAULT_COMPRESSION);

template <typename Output>
std::size_t
uncompress(Output &output, const char *data, std::size_t size, std::size_t max = 0) {
    constexpr int window_bits = 0; // deflate
 
    return compression::uncompress(output, data, size, max, window_bits);
}

template <>
size_t uncompress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                  std::size_t level);

struct to_uncompress {
    const char *data;              
    std::size_t size;              
    std::size_t max = 0;           
    std::size_t size_uncompressed; 
};

template <typename Output>
Output &
uncompress(Output &output, to_uncompress &info) {
    info.size_uncompressed = uncompress<Output>(output, info.data, info.size, info.max);
    return output;
}

std::string uncompress(const char *data, std::size_t size);
} // namespace deflate

namespace gzip {
inline bool
is_compressed(const char *data, std::size_t size) {
    return size > 2 && (
                           // zlib
                           (static_cast<uint8_t>(data[0]) == 0x78 &&
                            (static_cast<uint8_t>(data[1]) == 0x9C ||
                             static_cast<uint8_t>(data[1]) == 0x01 ||
                             static_cast<uint8_t>(data[1]) == 0xDA ||
                             static_cast<uint8_t>(data[1]) == 0x5E)) ||
                           // gzip
                           (static_cast<uint8_t>(data[0]) == 0x1F &&
                            static_cast<uint8_t>(data[1]) == 0x8B));
}

template <typename Output>
size_t
compress(Output &output, const char *data, std::size_t size,
         int level = Z_DEFAULT_COMPRESSION) {
    constexpr int window_bits = 15 + 16; // gzip with windowbits of 15
    return compression::compress(output, data, size, level, window_bits);
}

template <>
size_t compress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                int level);

struct to_compress {
    const char *data;                          
    std::size_t size;                          
    int         level = Z_DEFAULT_COMPRESSION; 
    size_t      size_compressed;               
};

template <typename Output>
Output &
compress(Output &output, to_compress &info) {
    info.size_compressed = compress<Output>(output, info.data, info.size, info.level);
    return output;
}

std::string compress(const char *data, std::size_t size,
                     int level = Z_DEFAULT_COMPRESSION);

template <typename Output>
std::size_t
uncompress(Output &output, const char *data, std::size_t size, std::size_t max = 0) {
    constexpr int window_bits = 15 + 32; // auto with windowbits of 15
 
    return compression::uncompress(output, data, size, max, window_bits);
}

template <>
size_t uncompress(qb::allocator::pipe<char> &output, const char *data, std::size_t size,
                  std::size_t level);

struct to_uncompress {
    const char *data;              
    std::size_t size;              
    std::size_t max = 0;           
    std::size_t size_uncompressed; 
};

template <typename Output>
Output &
uncompress(Output &output, to_uncompress &info) {
    info.size_uncompressed = uncompress<Output>(output, info.data, info.size, info.max);
    return output;
}

std::string uncompress(const char *data, std::size_t size);
} // namespace gzip
 
} // namespace compression
} // namespace qb
 
namespace qb::allocator {

template <>
pipe<char> &pipe<char>::put(qb::compression::deflate::to_compress &);

template <>
pipe<char> &pipe<char>::put(qb::compression::deflate::to_uncompress &);

template <>
pipe<char> &pipe<char>::put(qb::compression::gzip::to_compress &);

template <>
pipe<char> &pipe<char>::put(qb::compression::gzip::to_uncompress &);
 
} // namespace qb::allocator

namespace qb::gzip {
using namespace qb::compression::gzip;
}

namespace qb::deflate {
using namespace qb::compression::deflate;
}
 
#endif // QB_IO_COMPRESSION_H