AuroraSupport/AuroraRuntime
2
1
Fork 0
Code Issues 27 Pull Requests Releases Wiki Activity

[*] Untested bug fix: large compression frames were getting dropped

Browse Source
This commit is contained in:
Reece Wilson 2021-09-06 14:57:33 +01:00
parent 35e4d4bedc
commit c256406053
3 changed files with 99 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Include/Aurora/Memory/ByteBuffer.hpp
View File

@ -397,7 +397,42 @@ namespace Aurora::Memory
}
else
{
return endAtWrite ? (writePtr - readPtr) : (length - (readPtr - base));
if (endAtWrite)
{
if (writePtr < readPtr)
{
return 0;
}
else
{
return writePtr - readPtr;
}
}
else
{
return (length - (readPtr - base));
}
}
}
inline AuUInt RemainingWrite(bool endAtRead = true)
{
if (flagCircular)
{
if ((writePtr < readPtr) && (endAtRead))
{
return length - (readPtr - writePtr);
}
else
{
auto linearOverhead = length - (writePtr - base);
auto toWriteOverhead = readPtr - base;
return linearOverhead + toWriteOverhead;
}
}
else
{
return length - (writePtr - base);
}
}

97
Source/Compression/BlockDecompressor.cpp
View File

@ -82,30 +82,38 @@ namespace Aurora::Compression
return true;
}
bool userBound_ {};
char din_[ZSTD_BLOCKSIZE_MAX + 3 /*ZSTD_BLOCKHEADERSIZE*/];
char dout_[ZSTD_BLOCKSIZE_MAX];
ZSTD_inBuffer input_;
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
AuUInt32 length = ZSTD_DStreamInSize();
void *din = alloca(length);
auto outFrameLength = ZSTD_DStreamOutSize();
void *dout = alloca(outFrameLength);
AuUInt32 length = ZSTD_DStreamInSize();
AuUInt32 outFrameLength = ZSTD_DStreamOutSize();
AuUInt32 done{}, read{};
while (read != input)
while (read != input || userBound_)
{
AuUInt32 request = std::min(input, length);
if (this->reader_->Read(din, request) != IO::EStreamError::eErrorNone)
if (this->userBound_)
{
return AuMakePair(read, done);
}
read += request;
ZSTD_inBuffer input = { din, request, 0 };
while (input.pos < input.size)
{
ZSTD_outBuffer output = { dout, outFrameLength, 0 };
AuUInt32 request = std::min(input, length);
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
auto ret = ZSTD_decompressStream(this->dctx_, &output, &input);
input_ = ZSTD_inBuffer{ din_, request, 0 };
}
while (input_.pos < input_.size)
{
auto maxWrite = std::min(outFrameLength, AuUInt32(this->_outbuffer.RemainingWrite()));
ZSTD_outBuffer output = { dout_, outFrameLength, 0 };
auto ret = ZSTD_decompressStream(this->dctx_, &output, &input_);
if (ZSTD_isError(ret))
{
SysPushErrorIO("Compression error: {}", ret);
@ -116,6 +124,8 @@ namespace Aurora::Compression
this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(output.dst),
output.pos);
}
this->userBound_ = input_.pos != input_.size;
}
return AuMakePair(read, done);
@ -153,28 +163,33 @@ namespace Aurora::Compression
return true;
}
bool userBound_ {};
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
int ret;
AuUInt32 done{}, read{};
while (read < input)
while (read < input || userBound_)
{
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
if (!userBound_)
{
return AuMakePair(read, done);
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
do
{
this->ctx_.avail_out = AuArraySize(dout_);
this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
this->ctx_.next_out = dout_;
if (!this->ctx_.avail_out)
@ -196,7 +211,8 @@ namespace Aurora::Compression
have);
} while (this->ctx_.avail_out == 0);
SysAssert(this->ctx_.avail_in == 0);
this->userBound_ = this->ctx_.avail_in != 0;
}
return AuMakePair(read, done);
@ -236,27 +252,33 @@ namespace Aurora::Compression
this->init_ = true;
return true;
}
bool userBound_ {};
AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
{
int ret;
AuUInt32 done{}, read{};
while (read < input)
while (read < input || userBound_)
{
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
if (!userBound_)
{
return AuMakePair(read, done);
}
read += request;
AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
{
return AuMakePair(read, done);
}
read += request;
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<char *>(din_);
this->ctx_.avail_in = request;
this->ctx_.next_in = reinterpret_cast<char *>(din_);
}
do
{
this->ctx_.avail_out = AuArraySize(dout_);
this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
this->ctx_.next_out = dout_;
ret = BZ2_bzDecompress(&this->ctx_);
@ -274,7 +296,8 @@ namespace Aurora::Compression
} while (this->ctx_.avail_out == 0);
SysAssert(this->ctx_.avail_in == 0);
this->userBound_ = this->ctx_.avail_in != 0;
}
return AuMakePair(read, done);

33
Source/Locale/Encoding/EncoderNSL.cpp
View File

@ -37,46 +37,19 @@ namespace Aurora::Locale::Encoding
// Convert the cpA buffer to UTF-16
MultiByteToWideChar(cpA, 0, (LPCCH)in, inLength, ret, chars);
AuUInt32 utf16Recalc = chars;
if (!utf16Recalc)
{
return {};
}
// convert the shortened string with invalid surrogates back into a cpA length
AuUInt32 read = inLength;
// always calc for now
// handle **end of stream** and error trailing bytes
#if 0
if (slowPath)
#endif
{
// Assumes A1 -> B -> A2 (A1 == A2), would GB2312 survive the round trip?
// I know edge cases don't always map 1:1 with unicode
// However, with all that struggle to maintain a codepage, would you have duplicate characters?
// googling `GB2312 duplicate characters` and how translating works specifically is taking too long
// this shoule be fine, i cope. worst case scenario, we end up with a stream jump x bytes bug
// TODO: REVIEW ME
read = WideCharToMultiByte(cpA, 0, ret, utf16Recalc, NULL, 0, NULL, NULL);
// skipping is one thing, but this would be really bad
read = std::min(AuUInt32(read), AuUInt32(inLength));
}
AuUInt32 cpLength;
if (!cpBlob)
{
cpLength = WideCharToMultiByte(cpB, 0, ret, utf16Recalc, NULL, NULL, NULL, NULL);
cpLength = WideCharToMultiByte(cpB, 0, ret, chars, NULL, NULL, NULL, NULL);
}
else
{
cpLength = WideCharToMultiByte(cpB, 0, ret, utf16Recalc, (LPSTR)cpBlob, cpLen, NULL, NULL);
cpLength = WideCharToMultiByte(cpB, 0, ret, chars, (LPSTR)cpBlob, cpLen, NULL, NULL);
}
delete[] ret;
return {read, cpLength};
return {inLength, cpLength};
#else
return {};
#endif