[*] Untested bug fix: large compression frames were getting dropped

Include/Aurora/Memory/ByteBuffer.hpp

@@ -397,7 +397,42 @@ namespace Aurora::Memory
             }
             else
             {
-                return endAtWrite ? (writePtr - readPtr) : (length - (readPtr - base));
+                if (endAtWrite)
+                {
+                    if (writePtr < readPtr)
+                    {
+                        return 0;
+                    }
+                    else
+                    {
+                        return writePtr - readPtr;
+                    }
+                }
+                else
+                {
+                    return  (length - (readPtr - base));
+                }
+            }
+        }
+
+        inline AuUInt RemainingWrite(bool endAtRead = true)
+        {
+            if (flagCircular)
+            {
+                if ((writePtr < readPtr) && (endAtRead))
+                {
+                    return length - (readPtr - writePtr);
+                }
+                else
+                {
+                    auto linearOverhead   = length - (writePtr - base);
+                    auto toWriteOverhead  = readPtr  - base;
+                    return linearOverhead + toWriteOverhead;
+                }
+            }
+            else
+            {
+                return length - (writePtr - base);
             }
         }
         

Source/Compression/BlockDecompressor.cpp

@@ -82,30 +82,38 @@ namespace Aurora::Compression
 
             return true;
         }
+        
+        bool userBound_ {};
+        char din_[ZSTD_BLOCKSIZE_MAX + 3 /*ZSTD_BLOCKHEADERSIZE*/];
+        char dout_[ZSTD_BLOCKSIZE_MAX];
+        ZSTD_inBuffer input_;
 
         AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
         {
-            AuUInt32 length = ZSTD_DStreamInSize();
+            AuUInt32 length         = ZSTD_DStreamInSize();
-            void *din = alloca(length);
+            AuUInt32 outFrameLength = ZSTD_DStreamOutSize();
-            auto outFrameLength = ZSTD_DStreamOutSize();
-            void *dout = alloca(outFrameLength);
             AuUInt32 done{}, read{};
 
-            while (read != input)
+            while (read != input || userBound_)
             {
-                AuUInt32 request = std::min(input, length);
+                if (this->userBound_)
-                if (this->reader_->Read(din, request) != IO::EStreamError::eErrorNone)
                 {
-                    return AuMakePair(read, done);
+                    AuUInt32 request = std::min(input, length);
-                }
+                    if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
-                read += request;
+                    {
-
+                        return AuMakePair(read, done);
-                ZSTD_inBuffer input = { din, request, 0 };
+                    }
-                while (input.pos < input.size)
+                    read += request;
-                {
-                    ZSTD_outBuffer output = { dout, outFrameLength, 0 };
                     
-                    auto ret = ZSTD_decompressStream(this->dctx_, &output, &input);
+                    input_ = ZSTD_inBuffer{ din_, request, 0 };
+                }
+
+                while (input_.pos < input_.size)
+                {
+                    auto maxWrite = std::min(outFrameLength, AuUInt32(this->_outbuffer.RemainingWrite()));
+                    ZSTD_outBuffer output = { dout_, outFrameLength, 0 };
+                    
+                    auto ret = ZSTD_decompressStream(this->dctx_, &output, &input_);
                     if (ZSTD_isError(ret))
                     {
                         SysPushErrorIO("Compression error: {}", ret);
@@ -116,6 +124,8 @@ namespace Aurora::Compression
                     this->_outbuffer.Write(reinterpret_cast<const AuUInt8 *>(output.dst),
                                            output.pos);
                 }
+
+                this->userBound_ = input_.pos != input_.size;
             }
             
             return AuMakePair(read, done);
@@ -153,28 +163,33 @@ namespace Aurora::Compression
             return true;
         }
 
+        bool userBound_ {};
+
         AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
         {
             int ret;
 
             AuUInt32 done{}, read{};
             
-            while (read < input)
+            while (read < input || userBound_)
             {
-                AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
+                if (!userBound_)
-                if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
                 {
-                    return AuMakePair(read, done);
+                    AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
+                    if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
+                    {
+                        return AuMakePair(read, done);
+                    }
+
+                    read += request;
+
+                    this->ctx_.avail_in = request;
+                    this->ctx_.next_in = reinterpret_cast<unsigned char *>(din_);
                 }
 
-                read += request;
-
-                this->ctx_.avail_in = request;
-                this->ctx_.next_in  = reinterpret_cast<unsigned char *>(din_);
-
                 do
                 {
-                    this->ctx_.avail_out = AuArraySize(dout_);
+                    this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
                     this->ctx_.next_out  = dout_;
 
                     if (!this->ctx_.avail_out)
@@ -196,7 +211,8 @@ namespace Aurora::Compression
                                            have);
 
                 } while (this->ctx_.avail_out == 0);
-                SysAssert(this->ctx_.avail_in == 0);
+
+                this->userBound_ = this->ctx_.avail_in != 0;
             }
             
             return AuMakePair(read, done);
@@ -236,27 +252,33 @@ namespace Aurora::Compression
             this->init_ = true;
             return true;
         }
+        
+        bool userBound_ {};
 
         AuStreamReadWrittenPair_t Ingest(AuUInt32 input) override
         {
             int ret;
             
             AuUInt32 done{}, read{};
-            while (read < input)
+            while (read < input || userBound_)
             {
-                AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
+                
-                if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
+                if (!userBound_)
                 {
-                    return AuMakePair(read, done);
+                    AuUInt32 request = std::min(input, AuUInt32(AuArraySize(din_)));
-                }
+                    if (this->reader_->Read(din_, request) != IO::EStreamError::eErrorNone)
-                read += request;
+                    {
+                        return AuMakePair(read, done);
+                    }
+                    read += request;
 
-                this->ctx_.avail_in = request;
+                    this->ctx_.avail_in = request;
-                this->ctx_.next_in  = reinterpret_cast<char *>(din_);
+                    this->ctx_.next_in = reinterpret_cast<char *>(din_);
+                }
 
                 do
                 {
-                    this->ctx_.avail_out = AuArraySize(dout_);
+                    this->ctx_.avail_out = std::min(AuUInt32(AuArraySize(dout_)), AuUInt32(this->_outbuffer.RemainingWrite()));
                     this->ctx_.next_out  = dout_;
 
                     ret = BZ2_bzDecompress(&this->ctx_);
@@ -274,7 +296,8 @@ namespace Aurora::Compression
 
 
                 } while (this->ctx_.avail_out == 0);
-                SysAssert(this->ctx_.avail_in == 0);
+
+                this->userBound_ = this->ctx_.avail_in != 0;
             }
 
             return AuMakePair(read, done);

Source/Locale/Encoding/EncoderNSL.cpp

@@ -37,46 +37,19 @@ namespace Aurora::Locale::Encoding
         // Convert the cpA buffer to UTF-16
         MultiByteToWideChar(cpA, 0, (LPCCH)in, inLength, ret, chars);
 
-        AuUInt32 utf16Recalc = chars;
-    
-        if (!utf16Recalc)
-        {
-            return {};
-        }
-
         // convert the shortened string with invalid surrogates back into a cpA length
-        AuUInt32 read = inLength;
-        
-        // always calc for now
-        // handle **end of stream** and error trailing bytes
-    #if 0
-        if (slowPath)
-    #endif
-        {
-            // Assumes A1 -> B -> A2 (A1 == A2), would GB2312 survive the round trip?
-            // I know edge cases don't always map 1:1 with unicode
-            // However, with all that struggle to maintain a codepage, would you have duplicate characters?
-            // googling `GB2312 duplicate characters` and how translating works specifically is taking too long
-            // this shoule be fine, i cope. worst case scenario, we end up with a stream jump x bytes bug
-            // TODO: REVIEW ME
-            read = WideCharToMultiByte(cpA, 0, ret, utf16Recalc, NULL, 0, NULL, NULL);
-
-            // skipping is one thing, but this would be really bad
-            read = std::min(AuUInt32(read), AuUInt32(inLength));
-        }
-
         AuUInt32 cpLength;
         if (!cpBlob)
         {
-            cpLength = WideCharToMultiByte(cpB, 0, ret, utf16Recalc, NULL, NULL, NULL, NULL);
+            cpLength = WideCharToMultiByte(cpB, 0, ret, chars, NULL, NULL, NULL, NULL);
         }
         else
         {
-            cpLength = WideCharToMultiByte(cpB, 0, ret, utf16Recalc, (LPSTR)cpBlob, cpLen, NULL, NULL);
+            cpLength = WideCharToMultiByte(cpB, 0, ret, chars, (LPSTR)cpBlob, cpLen, NULL, NULL);
         }
         
         delete[] ret;
-        return {read, cpLength};
+        return {inLength, cpLength};
     #else
         return {};
     #endif