2021-06-27 21:25:29 +00:00
|
|
|
/***
|
|
|
|
Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.
|
|
|
|
|
|
|
|
File: BlockDecompressor.cpp
|
|
|
|
Date: 2021-6-17
|
|
|
|
Author: Reece
|
|
|
|
***/
|
|
|
|
#include <RuntimeInternal.hpp>
|
|
|
|
#include "Compression.hpp"
|
|
|
|
#include "BlockDecompressor.hpp"
|
|
|
|
|
|
|
|
#include "bzlib.h"
|
|
|
|
#include "zstd.h"
|
|
|
|
#include "zlib.h"
|
|
|
|
#include "lz4.h"
|
|
|
|
|
|
|
|
namespace Aurora::Compression
|
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
bool BaseStream::ReadByProcessedN(void * buffer, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 10:58:08 +00:00
|
|
|
AuUInt32 read {}, len {};
|
|
|
|
|
|
|
|
if (ingestUntilEOS)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 10:58:08 +00:00
|
|
|
while (this->_outbuffer.RemainingBytes() < minimumInflated)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-07-11 10:50:57 +00:00
|
|
|
if (Ingest(4096).second == 0)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 10:58:08 +00:00
|
|
|
if (!this->_outbuffer.RemainingBytes())
|
2021-07-11 11:36:15 +00:00
|
|
|
{
|
2021-09-06 10:58:08 +00:00
|
|
|
return false;
|
2021-07-11 11:36:15 +00:00
|
|
|
}
|
2021-09-06 10:58:08 +00:00
|
|
|
|
|
|
|
break;
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
2021-09-06 10:58:08 +00:00
|
|
|
|
|
|
|
read += 4096;
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
len = this->_outbuffer.Read(buffer, len, buffer == nullptr);
|
|
|
|
pair = {read, len};
|
|
|
|
return len != 0;
|
|
|
|
}
|
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
bool BaseStream::GoBackByProcessedN(AuUInt32 offset)
|
2021-09-06 10:58:08 +00:00
|
|
|
{
|
|
|
|
return this->_outbuffer.ReaderTryGoBack(offset);
|
|
|
|
}
|
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
bool BaseStream::GoForwardByProcessedN(AuUInt32 offset)
|
2021-09-06 10:58:08 +00:00
|
|
|
{
|
|
|
|
return this->_outbuffer.ReaderTryGoForward(offset);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
bool BaseStream::Write(const void *a, AuUInt32 length)
|
|
|
|
{
|
|
|
|
auto written = this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(a),
|
|
|
|
length);
|
|
|
|
|
|
|
|
if (written != length)
|
|
|
|
{
|
|
|
|
auto increase = std::max(0, (int)length - (int)this->_outbuffer.RemainingWrite());
|
|
|
|
increase += this->_outbuffer.length;
|
|
|
|
|
|
|
|
if (increase > 64 * 1024 * 1024)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
this->_outbuffer.Resize(increase);
|
|
|
|
|
|
|
|
auto remaining = length - written;
|
|
|
|
written = this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(a) + written,
|
|
|
|
remaining);
|
|
|
|
if (written != remaining)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
AuUInt32 BaseStream::GetInternalBufferSize()
|
|
|
|
{
|
|
|
|
return this->_outbuffer.allocSize;
|
|
|
|
}
|
|
|
|
|
2021-06-27 21:25:29 +00:00
|
|
|
class ZSTDInflate : public BaseStream
|
|
|
|
{
|
|
|
|
public:
|
2021-09-06 10:58:08 +00:00
|
|
|
ZSTDInflate() : BaseStream()
|
|
|
|
{
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2021-06-27 21:25:29 +00:00
|
|
|
~ZSTDInflate()
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
if (auto dctx = std::exchange(dctx_, {}))
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
|
|
|
ZSTD_freeDCtx(dctx);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Init(Aurora::IO::IStreamReader *reader)
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
this->reader_ = reader;
|
|
|
|
this->dctx_ = ZSTD_createDCtx();
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
if (!this->dctx_)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
|
|
|
SysPushErrorGen("Couldn't create decompressor");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
2021-09-06 13:57:33 +00:00
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 13:57:33 +00:00
|
|
|
AuUInt32 length = ZSTD_DStreamInSize();
|
|
|
|
AuUInt32 outFrameLength = ZSTD_DStreamOutSize();
|
2021-07-11 10:50:57 +00:00
|
|
|
AuUInt32 done{}, read{};
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 15:47:35 +00:00
|
|
|
while (read != input)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
AuUInt32 request = std::min(input, length);
|
|
|
|
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
return AuMakePair(read, done);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
2021-09-06 14:21:39 +00:00
|
|
|
read += request;
|
|
|
|
|
|
|
|
input_ = ZSTD_inBuffer{ din_, request, 0 };
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 13:57:33 +00:00
|
|
|
while (input_.pos < input_.size)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 13:57:33 +00:00
|
|
|
ZSTD_outBuffer output = { dout_, outFrameLength, 0 };
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 13:57:33 +00:00
|
|
|
auto ret = ZSTD_decompressStream(this->dctx_, &output, &input_);
|
2021-06-27 21:25:29 +00:00
|
|
|
if (ZSTD_isError(ret))
|
|
|
|
{
|
|
|
|
SysPushErrorIO("Compression error: {}", ret);
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
2021-07-11 10:50:57 +00:00
|
|
|
done += output.pos;
|
2021-09-06 14:21:39 +00:00
|
|
|
|
|
|
|
if (!Write(reinterpret_cast<const AuUInt8 *>(output.dst),
|
|
|
|
output.pos))
|
|
|
|
{
|
|
|
|
return {};
|
|
|
|
}
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
}
|
2021-07-11 10:50:57 +00:00
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, done);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
2021-07-11 17:26:38 +00:00
|
|
|
|
|
|
|
Aurora::IO::IStreamReader *reader_;
|
|
|
|
ZSTD_DCtx *dctx_;
|
2021-09-06 14:21:39 +00:00
|
|
|
char din_[ZSTD_BLOCKSIZE_MAX + 3 /*ZSTD_BLOCKHEADERSIZE*/];
|
|
|
|
char dout_[ZSTD_BLOCKSIZE_MAX];
|
|
|
|
ZSTD_inBuffer input_;
|
2021-06-27 21:25:29 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class ZIPInflate : public BaseStream
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
~ZIPInflate()
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
if (auto ctx = std::exchange(this->init_, {}))
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
inflateEnd(&this->ctx_);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Init(Aurora::IO::IStreamReader *reader)
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
this->reader_ = reader;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
auto ret = inflateInit(&this->ctx_);
|
2021-06-27 21:25:29 +00:00
|
|
|
if (ret != Z_OK)
|
|
|
|
{
|
|
|
|
SysPushErrorMem("Error: {}", ret);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
this->init_ = true;
|
2021-06-27 21:25:29 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
|
|
|
int ret;
|
2021-07-11 10:50:57 +00:00
|
|
|
|
|
|
|
AuUInt32 done{}, read{};
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 15:47:35 +00:00
|
|
|
while (read < input)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
|
|
|
|
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
return AuMakePair(read, done);
|
|
|
|
}
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
read += request;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
this->ctx_.avail_in = request;
|
|
|
|
this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
|
2021-06-27 21:25:29 +00:00
|
|
|
|
|
|
|
do
|
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
this->ctx_.avail_out = AuArraySize(dout_); // std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
|
2021-07-11 17:26:38 +00:00
|
|
|
this->ctx_.next_out = dout_;
|
2021-07-11 10:50:57 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
if (!this->ctx_.avail_out)
|
2021-07-11 10:50:57 +00:00
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
ret = inflate(&this->ctx_, Z_NO_FLUSH);
|
2021-06-27 21:25:29 +00:00
|
|
|
if (ret != Z_OK)
|
|
|
|
{
|
|
|
|
SysPushErrorIO("Error: {}", ret);
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
auto have = AuArraySize(dout_) - this->ctx_.avail_out;
|
2021-07-11 10:50:57 +00:00
|
|
|
done += have;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
if (!Write(reinterpret_cast<const AuUInt8 *>(dout_),
|
|
|
|
have))
|
|
|
|
{
|
|
|
|
return {};
|
|
|
|
}
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
} while (this->ctx_.avail_out == 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
2021-07-11 10:50:57 +00:00
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, done);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
2021-07-11 17:26:38 +00:00
|
|
|
|
|
|
|
Aurora::IO::IStreamReader *reader_;
|
|
|
|
z_stream ctx_ {};
|
|
|
|
bool init_ {};
|
2021-07-11 10:50:57 +00:00
|
|
|
unsigned char din_[4096];
|
|
|
|
unsigned char dout_[4096];
|
2021-06-27 21:25:29 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class BZIPInflate : public BaseStream
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
~BZIPInflate()
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
if (auto ctx = std::exchange(this->init_, {}))
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
BZ2_bzDecompressEnd(&this->ctx_);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Init(Aurora::IO::IStreamReader *reader)
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
this->reader_ = reader;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
auto ret = BZ2_bzDecompressInit(&this->ctx_, 0, 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
if (ret != Z_OK)
|
|
|
|
{
|
|
|
|
SysPushErrorMem("Error: {}", ret);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
this->init_ = true;
|
2021-06-27 21:25:29 +00:00
|
|
|
return true;
|
|
|
|
}
|
2021-09-06 13:57:33 +00:00
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2021-07-11 10:50:57 +00:00
|
|
|
AuUInt32 done{}, read{};
|
2021-09-06 13:57:33 +00:00
|
|
|
while (read < input || userBound_)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
|
|
|
|
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
return AuMakePair(read, done);
|
2021-09-06 13:57:33 +00:00
|
|
|
}
|
2021-09-06 14:21:39 +00:00
|
|
|
read += request;
|
|
|
|
|
|
|
|
this->ctx_.avail_in = request;
|
|
|
|
this->ctx_.next_in = reinterpret_cast<char *>(din_);
|
2021-06-27 21:25:29 +00:00
|
|
|
|
|
|
|
do
|
|
|
|
{
|
2021-09-06 14:21:39 +00:00
|
|
|
this->ctx_.avail_out = AuArraySize(dout_);
|
2021-07-11 17:26:38 +00:00
|
|
|
this->ctx_.next_out = dout_;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
ret = BZ2_bzDecompress(&this->ctx_);
|
2021-06-27 21:25:29 +00:00
|
|
|
if (ret != Z_OK)
|
|
|
|
{
|
|
|
|
SysPushErrorIO("Error: {}", ret);
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
auto have = AuArraySize(dout_) - this->ctx_.avail_out;
|
2021-07-11 10:50:57 +00:00
|
|
|
done += have;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-09-06 14:21:39 +00:00
|
|
|
if (!Write(reinterpret_cast<const AuUInt8 *>(dout_),
|
|
|
|
have))
|
|
|
|
{
|
|
|
|
return {};
|
|
|
|
}
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
} while (this->ctx_.avail_out == 0);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
return AuMakePair(read, done);
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
2021-07-11 17:26:38 +00:00
|
|
|
|
|
|
|
Aurora::IO::IStreamReader *reader_;
|
|
|
|
bz_stream ctx_ {};
|
|
|
|
bool init_ {};
|
2021-07-11 10:50:57 +00:00
|
|
|
char dout_[4096];
|
2021-09-06 14:21:39 +00:00
|
|
|
bool userBound_ {};
|
2021-07-11 10:50:57 +00:00
|
|
|
char din_[4096];
|
2021-06-27 21:25:29 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class LZ4Inflate : public BaseStream
|
|
|
|
{
|
|
|
|
public:
|
2021-09-06 10:58:08 +00:00
|
|
|
|
|
|
|
LZ4Inflate() : BaseStream(64 * 1024 * 2)
|
|
|
|
{}
|
|
|
|
|
2021-06-27 21:25:29 +00:00
|
|
|
~LZ4Inflate()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Init(Aurora::IO::IStreamReader *reader)
|
|
|
|
{
|
2021-07-11 17:26:38 +00:00
|
|
|
this->reader_ = reader;
|
2021-06-27 21:25:29 +00:00
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
this->lz4Stream_ = LZ4_createStreamDecode();
|
|
|
|
if (!this->lz4Stream_)
|
2021-06-27 21:25:29 +00:00
|
|
|
{
|
|
|
|
SysPushErrorMem();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2021-07-11 10:50:57 +00:00
|
|
|
|
2021-09-06 10:58:08 +00:00
|
|
|
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
|
|
|
|
{
|
|
|
|
return {};
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
2021-07-11 17:26:38 +00:00
|
|
|
Aurora::IO::IStreamReader *reader_;
|
|
|
|
LZ4_streamDecode_t* lz4Stream_ {};
|
2021-06-27 21:25:29 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
AUKN_SYM ICompressionStream *DecompressorNew(IO::IStreamReader *reader, ECompresionType type)
|
|
|
|
{
|
|
|
|
BaseStream * ret{};
|
|
|
|
|
|
|
|
switch (type)
|
|
|
|
{
|
|
|
|
case ECompresionType::eZSTD:
|
|
|
|
ret = new ZSTDInflate();
|
|
|
|
break;
|
|
|
|
case ECompresionType::eBZIP2:
|
|
|
|
ret = new BZIPInflate();
|
|
|
|
break;
|
|
|
|
case ECompresionType::eLZ4:
|
|
|
|
ret = new LZ4Inflate();
|
|
|
|
break;
|
|
|
|
case ECompresionType::eDeflate:
|
|
|
|
ret = new ZIPInflate();
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
ret = nullptr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ret)
|
|
|
|
{
|
|
|
|
if (!ret->Init(reader))
|
|
|
|
{
|
|
|
|
delete ret;
|
2021-07-11 10:50:57 +00:00
|
|
|
ret = nullptr;
|
2021-06-27 21:25:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
AUKN_SYM void DecompressorRelease(ICompressionStream * stream)
|
|
|
|
{
|
|
|
|
SafeDelete<BaseStream *>(stream);
|
|
|
|
}
|
|
|
|
}
|