AuroraSupport/AuroraRuntime
2
1
Fork 0
Code Issues 29 Pull Requests Releases Wiki Activity

[*] untested possible alternative solution, expose internal buffer size, caller does not manage internal buffer size (yet)

Browse Source
This commit is contained in:
Reece Wilson 2021-09-06 15:21:39 +01:00
parent c256406053
commit 80d351472a
3 changed files with 96 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Include/Aurora/Compression/ICompressionStream.hpp
View File

@ -13,8 +13,10 @@ namespace Aurora::Compression
{
public:
virtual AuStreamReadWrittenPair_t Ingest(AuUInt32 bytesFromInputSource) = 0;
virtual bool ReadByInflatedN(void * /*opt*/, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS = true) = 0;
virtual bool GoBackByInflatedN(AuUInt32 offset) = 0;
virtual bool GoForwardByInflatedN(AuUInt32 offset) = 0;
// You should probably check this if you don't want to be DDoS'd
virtual AuUInt32 GetInternalBufferSize() = 0;
virtual bool ReadByProcessedN(void * /*opt*/, AuUInt32 minimumProcessed, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS = true) = 0;
virtual bool GoBackByProcessedN(AuUInt32 offset) = 0;
virtual bool GoForwardByProcessedN(AuUInt32 offset) = 0;
};
}

140
Source/Compression/BlockDecompressor.cpp
View File

@ -16,7 +16,7 @@
namespace Aurora::Compression
{
bool BaseStream::ReadByInflatedN(void * buffer, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS)
bool BaseStream::ReadByProcessedN(void * buffer, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS)
{
AuUInt32 read {}, len {};
@ -43,16 +43,50 @@ namespace Aurora::Compression
return len != 0;
}
bool BaseStream::GoBackByInflatedN(AuUInt32 offset)
bool BaseStream::GoBackByProcessedN(AuUInt32 offset)
{
return this->_outbuffer.ReaderTryGoBack(offset);
}
bool BaseStream::GoForwardByInflatedN(AuUInt32 offset)
bool BaseStream::GoForwardByProcessedN(AuUInt32 offset)
{
return this->_outbuffer.ReaderTryGoForward(offset);
}
bool BaseStream::Write(const void *a, AuUInt32 length)
{
auto written = this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(a),
length);
if (written != length)
{
auto increase = std::max(0, (int)length - (int)this->_outbuffer.RemainingWrite());
increase += this->_outbuffer.length;
if (increase > 64 * 1024 * 1024)
{
return false;
}
this->_outbuffer.Resize(increase);
auto remaining = length - written;
written = this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(a) + written,
remaining);
if (written != remaining)
{
return false;
}
}
return true;
}
AuUInt32 BaseStream::GetInternalBufferSize()
{
return this->_outbuffer.allocSize;
}
class ZSTDInflate : public BaseStream
{
public:
@ -83,11 +117,6 @@ namespace Aurora::Compression
return true;
}
bool userBound_ {};
char din_[ZSTD_BLOCKSIZE_MAX + 3 /*ZSTD_BLOCKHEADERSIZE*/];
char dout_[ZSTD_BLOCKSIZE_MAX];
ZSTD_inBuffer input_;
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
AuUInt32 length = ZSTD_DStreamInSize();
@ -96,17 +125,14 @@ namespace Aurora::Compression
while (read != input || userBound_)
{
if (this->userBound_)
AuUInt32 request = std::min(input, length);
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
AuUInt32 request = std::min(input, length);
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
input_ = ZSTD_inBuffer{ din_, request, 0 };
return AuMakePair(read, done);
}
read += request;
input_ = ZSTD_inBuffer{ din_, request, 0 };
while (input_.pos < input_.size)
{
@ -121,8 +147,12 @@ namespace Aurora::Compression
}
done += output.pos;
this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(output.dst),
output.pos);
if (!Write(reinterpret_cast<const AuUInt8 *>(output.dst),
output.pos))
{
return {};
}
}
this->userBound_ = input_.pos != input_.size;
@ -135,6 +165,10 @@ namespace Aurora::Compression
Aurora::IO::IStreamReader *reader_;
ZSTD_DCtx *dctx_;
bool userBound_ {};
char din_[ZSTD_BLOCKSIZE_MAX + 3 /*ZSTD_BLOCKHEADERSIZE*/];
char dout_[ZSTD_BLOCKSIZE_MAX];
ZSTD_inBuffer input_;
};
class ZIPInflate : public BaseStream
@ -163,8 +197,6 @@ namespace Aurora::Compression
return true;
}
bool userBound_ {};
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
int ret;
@ -173,23 +205,20 @@ namespace Aurora::Compression
while (read < input || userBound_)
{
if (!userBound_)
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
do
{
this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
this->ctx_.avail_out = AuArraySize(dout_); // std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
this->ctx_.next_out = dout_;
if (!this->ctx_.avail_out)
@ -207,12 +236,15 @@ namespace Aurora::Compression
auto have = AuArraySize(dout_) - this->ctx_.avail_out;
done += have;
this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(dout_),
have);
if (!Write(reinterpret_cast<const AuUInt8 *>(dout_),
have))
{
return {};
}
} while (this->ctx_.avail_out == 0);
this->userBound_ = this->ctx_.avail_in != 0;
//this->userBound_ = this->ctx_.avail_in != 0;
}
return AuMakePair(read, done);
@ -225,6 +257,7 @@ namespace Aurora::Compression
bool init_ {};
unsigned char din_[4096];
unsigned char dout_[4096];
bool userBound_ {};
};
class BZIPInflate : public BaseStream
@ -253,8 +286,6 @@ namespace Aurora::Compression
return true;
}
bool userBound_ {};
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
int ret;
@ -262,23 +293,19 @@ namespace Aurora::Compression
AuUInt32 done{}, read{};
while (read < input || userBound_)
{
if (!userBound_)
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<char *>(din_);
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<char *>(din_);
do
{
this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
this->ctx_.avail_out = AuArraySize(dout_);
this->ctx_.next_out = dout_;
ret = BZ2_bzDecompress(&this->ctx_);
@ -291,13 +318,15 @@ namespace Aurora::Compression
auto have = AuArraySize(dout_) - this->ctx_.avail_out;
done += have;
this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(dout_),
have);
if (!Write(reinterpret_cast<const AuUInt8 *>(dout_),
have))
{
return {};
}
} while (this->ctx_.avail_out == 0);
this->userBound_ = this->ctx_.avail_in != 0;
//this->userBound_ = this->ctx_.avail_in != 0;
}
return AuMakePair(read, done);
@ -309,6 +338,7 @@ namespace Aurora::Compression
bz_stream ctx_ {};
bool init_ {};
char dout_[4096];
bool userBound_ {};
char din_[4096];
};

10
Source/Compression/BlockDecompressor.hpp
View File

@ -18,10 +18,12 @@ namespace Aurora::Compression
virtual bool Init(Aurora::IO::IStreamReader *reader) = 0;
virtual bool ReadByInflatedN(void * /*opt*/, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS = true) override;
virtual bool GoBackByInflatedN(AuUInt32 offset) override;
virtual bool GoForwardByInflatedN(AuUInt32 offset) override;
virtual bool ReadByProcessedN(void * /*opt*/, AuUInt32 minimumInflated, AuStreamReadWrittenPair_t &pair, bool ingestUntilEOS = true) override;
virtual bool GoBackByProcessedN(AuUInt32 offset) override;
virtual bool GoForwardByProcessedN(AuUInt32 offset) override;
virtual AuUInt32 GetInternalBufferSize() override;
bool Write(const void *a, AuUInt32 length);
protected:
Aurora::Memory::ByteBuffer _outbuffer;