AuroraRuntime/Source/Compression/StreamCompression.cpp

1085 lines
31 KiB
C++
Raw Normal View History

2021-06-27 21:25:29 +00:00
/***
Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.
File: StreamCompression.cpp
Date: 2021-6-17
Author: Reece
@warning **supported** shit api implementation. this is [somewhat] useless for large real world stuff outside of the utility it provides
2021-06-27 21:25:29 +00:00
***/
2021-09-30 14:57:41 +00:00
#include <Source/RuntimeInternal.hpp>
#include "AuCompression.hpp"
2021-06-27 21:25:29 +00:00
#include "StreamCompression.hpp"
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_BZIP2)
#include "bzlib.h"
#endif
2021-06-27 21:25:29 +00:00
#include "zstd.h"
#include "zlib.h"
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_LZ4)
#include "lz4.h"
#include "lz4frame.h"
#endif
2021-06-27 21:25:29 +00:00
namespace Aurora::Compression
{
static bool DecompressZSTD(const CompressionPipe &info)
{
Memory::ByteBuffer buffer;
Memory::ByteBuffer inflatedBuffer;
2022-08-20 06:53:48 +00:00
if (!info.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!info.pReadPipe)
{
return false;
}
2021-06-27 21:25:29 +00:00
auto length = ZSTD_DStreamInSize();
2021-09-06 10:58:08 +00:00
auto ok = AuTryResize(buffer, length);
2021-06-27 21:25:29 +00:00
if (!ok)
{
SysPushErrorMem("Couldn't reserve inflation buffers");
return false;
}
auto outFrameLength = ZSTD_DStreamOutSize();
2021-09-06 10:58:08 +00:00
ok = AuTryResize(inflatedBuffer, outFrameLength);
2021-06-27 21:25:29 +00:00
if (!ok)
{
SysPushErrorMem("Couldn't reserve inflation buffers");
return false;
}
auto dctx = ZSTD_createDCtx();
if (!dctx)
{
SysPushErrorGeneric("Couldn't create decompressor");
return false;
}
AuUInt outputStat{}, inputStat{};
2021-06-27 21:25:29 +00:00
AuUInt read = buffer.size();
2022-08-20 06:53:48 +00:00
while ((info.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(buffer.data(), read)) == AuIO::EStreamError::eErrorNone) && read)
2021-06-27 21:25:29 +00:00
{
inputStat += read;
2021-06-27 21:25:29 +00:00
ZSTD_inBuffer input = { buffer.data(), read, 0 };
read = buffer.size();
2021-06-27 21:25:29 +00:00
while (input.pos < input.size)
{
ZSTD_outBuffer output = { inflatedBuffer.data(), outFrameLength, 0 };
auto ret = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(ret))
{
SysPushErrorIO("Compression error: {}", ret);
ZSTD_freeDCtx(dctx);
return false;
}
{
AuUInt copy = output.pos;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(info.pWritePipe.get(), AuMemory::MemoryViewStreamRead(output.dst, copy));
outputStat += output.pos;
if (copy != output.pos)
{
SysPushErrorIO();
return false;
}
}
2021-06-27 21:25:29 +00:00
if (info.reportProgress)
{
if (!info.reportProgress(inputStat, outputStat))
2021-06-27 21:25:29 +00:00
{
ZSTD_freeDCtx(dctx);
return false;
}
}
}
}
ZSTD_freeDCtx(dctx);
return true;
}
static bool CompressZSTD(const CompressionPipe &stream, const CompressInfo &info)
2021-06-27 21:25:29 +00:00
{
Memory::ByteBuffer inflatedBuffer;
Memory::ByteBuffer deflatedBuffer;
2021-06-27 21:25:29 +00:00
size_t ret;
const auto buffInSize = ZSTD_CStreamInSize();
const auto buffOutSize = ZSTD_CStreamOutSize();
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
return false;
}
2021-06-27 21:25:29 +00:00
2021-09-06 10:58:08 +00:00
auto ok = AuTryResize(inflatedBuffer, buffInSize);
2021-06-27 21:25:29 +00:00
if (!ok)
{
SysPushErrorMem("Couldn't reserve deflation buffers");
return false;
}
2021-09-06 10:58:08 +00:00
ok = AuTryResize(deflatedBuffer, buffOutSize);
2021-06-27 21:25:29 +00:00
if (!ok)
{
SysPushErrorMem("Couldn't reserve deflation buffers. Out of memory");
return false;
}
auto cctx = ZSTD_createCCtx();
if (!cctx)
{
SysPushErrorGeneric("Couldn't create decompressor");
return false;
}
ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, info.uCompressionLevel);
2021-06-27 21:25:29 +00:00
if (ZSTD_isError(ret))
{
SysPushErrorGen("Invalid compression level");
ZSTD_freeCCtx(cctx);
return false;
}
ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
if (ZSTD_isError(ret))
{
SysPushErrorGen("Invalid option");
ZSTD_freeCCtx(cctx);
return false;
}
2022-01-19 17:08:13 +00:00
ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, AuMax(stream.threads, 1u));
2021-06-27 21:25:29 +00:00
if (ZSTD_isError(ret))
{
SysPushErrorGen();
ZSTD_freeCCtx(cctx);
return false;
}
bool status = true;
size_t const toRead = buffInSize;
AuUInt outputStat {}, inputStat {};
2021-06-27 21:25:29 +00:00
while (true)
{
AuUInt read = buffInSize;
2022-08-20 06:53:48 +00:00
if (stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(inflatedBuffer.data(), read)) != AuIO::EStreamError::eErrorNone)
{
break;
}
inputStat += read;
2021-06-27 21:25:29 +00:00
/* Select the flush mode.
* If the read may not be finished (read == toRead) we use
* ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
* Zstd optimizes the case where the first flush mode is ZSTD_e_end,
* since it knows it is compressing the entire source in one pass.
*/
int const lastChunk = (read < toRead);
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
/* Set the input buffer to what we just read.
* We compress until the input buffer is empty, each time flushing the
* output.
*/
ZSTD_inBuffer input = { inflatedBuffer.data(), read, 0 };
int finished;
do
{
/* Compress into the output buffer and write all of the output to
* the file so we can reuse the buffer next iteration.
*/
ZSTD_outBuffer output = { deflatedBuffer.data(), buffOutSize, 0 };
size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode);
if (ZSTD_isError(remaining))
{
SysPushErrorGen(": {}", ZSTD_getErrorName(remaining));
ZSTD_freeCCtx(cctx);
return false;
}
{
AuUInt copy = output.pos;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(output.dst, copy));
outputStat += output.pos;
if (copy != output.pos)
{
SysPushErrorIO();
return false;
}
}
2021-06-27 21:25:29 +00:00
if (stream.reportProgress)
{
if (!stream.reportProgress(inputStat, outputStat))
2021-06-27 21:25:29 +00:00
{
ZSTD_freeCCtx(cctx);
return false;
}
}
/* If we're on the last chunk we're finished when zstd returns 0,
* which means its consumed all the input AND finished the frame.
* Otherwise, we're finished when we've consumed all the input.
*/
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished);
SysAssertExp(input.pos == input.size);
if (lastChunk) break;
}
ZSTD_freeCCtx(cctx);
return true;
}
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
static bool CompressZLib(const CompressionPipe &stream, AuUInt8 level, AuInt8 bits)
2021-06-27 21:25:29 +00:00
{
int ret, flush;
z_stream strm {};
unsigned char in[kChunkSize];
unsigned char out[kChunkSize];
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
return false;
}
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
ret = deflateInit2(&strm, level, Z_DEFLATED, bits, 8, Z_DEFAULT_STRATEGY);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
return false;
}
AuUInt32 inputStat = 0, outputStat = 0;
do
{
AuUInt read = AuArraySize(in);
AuIO::EStreamError error;
2022-08-20 06:53:48 +00:00
if (((error = stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(in, read))) != AuIO::EStreamError::eErrorNone) &&
(error != AuIO::EStreamError::eErrorEndOfStream))
{
break;
}
2021-06-27 21:25:29 +00:00
strm.avail_in = read;
inputStat += read;
flush = (!strm.avail_in || error == AuIO::EStreamError::eErrorEndOfStream) ? Z_FINISH : Z_NO_FLUSH;
if (read && read != AuArraySize(in))
{
flush = Z_FULL_FLUSH;
}
2021-06-27 21:25:29 +00:00
strm.next_in = in;
do
{
2021-09-06 10:58:08 +00:00
strm.avail_out = AuArraySize(out);
2021-06-27 21:25:29 +00:00
strm.next_out = out;
ret = deflate(&strm, flush);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
return false;
}
2021-09-06 10:58:08 +00:00
auto have = AuArraySize(out) - strm.avail_out;
2021-06-27 21:25:29 +00:00
{
AuUInt copy = have;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(out, copy));
outputStat += copy;
if (copy != have)
{
SysPushErrorIO();
return false;
}
}
2021-06-27 21:25:29 +00:00
2021-09-06 15:47:35 +00:00
if (stream.reportProgress)
2021-06-27 21:25:29 +00:00
{
2021-09-06 15:47:35 +00:00
if (!stream.reportProgress(inputStat, outputStat))
{
deflateEnd(&strm);
return false;
}
2021-06-27 21:25:29 +00:00
}
} while (strm.avail_out == 0);
SysAssert(strm.avail_in == 0);
} while (flush != Z_FINISH);
deflateEnd(&strm);
return true;
}
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
static bool DecompressZLib(const CompressionPipe &stream, int bits)
2021-06-27 21:25:29 +00:00
{
int ret;
z_stream strm {};
unsigned char in[kChunkSize];
unsigned char out[kChunkSize];
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
return false;
}
2021-06-27 21:25:29 +00:00
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
ret = inflateInit2(&strm, bits);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
return false;
}
AuUInt32 inputStat = 0, outputStat = 0;
do
{
AuUInt read = AuArraySize(in);
2022-08-20 06:53:48 +00:00
if (stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(in, read)) != AuIO::EStreamError::eErrorNone)
{
break;
}
2021-06-27 21:25:29 +00:00
inputStat += read;
if (!read)
{
break;
}
strm.avail_in = read;
strm.next_in = in;
do
{
2021-09-06 10:58:08 +00:00
strm.avail_out = AuArraySize(out);
2021-06-27 21:25:29 +00:00
strm.next_out = out;
ret = inflate(&strm, Z_NO_FLUSH);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
inflateEnd(&strm);
2021-06-27 21:25:29 +00:00
return false;
}
2021-09-06 10:58:08 +00:00
auto have = AuArraySize(out) - strm.avail_out;
2021-06-27 21:25:29 +00:00
{
AuUInt copy = have;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(out, copy));
outputStat += copy;
if (copy != have)
{
SysPushErrorIO();
return false;
}
}
2021-06-27 21:25:29 +00:00
2021-09-06 15:47:35 +00:00
if (stream.reportProgress)
2021-06-27 21:25:29 +00:00
{
2021-09-06 15:47:35 +00:00
if (!stream.reportProgress(inputStat, outputStat))
{
inflateEnd(&strm);
return false;
}
2021-06-27 21:25:29 +00:00
}
} while (strm.avail_out == 0);
SysAssert(strm.avail_in == 0);
} while (ret != Z_STREAM_END);
inflateEnd(&strm);
return true;
}
static bool CompressBZip2(const CompressionPipe &stream, const CompressInfo &info)
2021-06-27 21:25:29 +00:00
{
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_BZIP2)
2021-06-27 21:25:29 +00:00
int ret, flush;
bz_stream strm {};
char in[kChunkSize];
char out[kChunkSize];
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
return false;
}
2021-06-27 21:25:29 +00:00
ret = BZ2_bzCompressInit(&strm, info.uCompressionLevel, 0, 0);
if (ret < BZ_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
return false;
}
AuUInt32 inputStat = 0, outputStat = 0;
do
{
AuIO::EStreamError error;
AuUInt read{ kChunkSize };
2022-08-20 06:53:48 +00:00
if (((error = stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(in, read))) != AuIO::EStreamError::eErrorNone) &&
(error != AuIO::EStreamError::eErrorEndOfStream))
{
break;
}
2021-06-27 21:25:29 +00:00
strm.avail_in = read;
inputStat += read;
flush = (!strm.avail_in || error == AuIO::EStreamError::eErrorEndOfStream) ? BZ_FINISH : BZ_RUN;
2021-06-27 21:25:29 +00:00
strm.next_in = in;
do
{
2021-09-06 10:58:08 +00:00
strm.avail_out = AuArraySize(out);
2021-06-27 21:25:29 +00:00
strm.next_out = out;
ret = BZ2_bzCompress(&strm, flush);
if (ret < BZ_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
BZ2_bzCompressEnd(&strm);
2021-06-27 21:25:29 +00:00
return false;
}
2021-09-06 10:58:08 +00:00
auto have = AuArraySize(out) - strm.avail_out;
2021-06-27 21:25:29 +00:00
{
AuUInt copy = have;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(out, copy));
outputStat += copy;
2021-06-27 21:25:29 +00:00
if (copy != have)
{
SysPushErrorIO();
return false;
}
}
2021-09-06 15:47:35 +00:00
if (stream.reportProgress)
2021-06-27 21:25:29 +00:00
{
2021-09-06 15:47:35 +00:00
if (!stream.reportProgress(inputStat, outputStat))
{
BZ2_bzCompressEnd(&strm);
return false;
}
2021-06-27 21:25:29 +00:00
}
} while (strm.avail_out == 0);
SysAssert(strm.avail_in == 0);
} while (flush != BZ_FINISH);
BZ2_bzCompressEnd(&strm);
return true;
2022-03-18 04:30:39 +00:00
#else
return false;
#endif
2021-06-27 21:25:29 +00:00
}
static bool DecompressBZip2(const CompressionPipe &stream)
{
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_BZIP2)
2021-06-27 21:25:29 +00:00
int ret;
bz_stream strm {};
char in[kChunkSize];
char out[kChunkSize];
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
return false;
}
2021-06-27 21:25:29 +00:00
ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
return false;
}
AuUInt32 inputStat = 0, outputStat = 0;
do
{
AuUInt read = AuArraySize(in);
2022-08-20 06:53:48 +00:00
if (stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(in, read)) != AuIO::EStreamError::eErrorNone)
{
break;
}
2021-06-27 21:25:29 +00:00
inputStat += read;
if (!read)
{
break;
}
strm.avail_in = read;
strm.next_in = in;
do
{
2021-09-06 10:58:08 +00:00
strm.avail_out = AuArraySize(out);
2021-06-27 21:25:29 +00:00
strm.next_out = out;
ret = BZ2_bzDecompress(&strm);
if (ret < Z_OK)
2021-06-27 21:25:29 +00:00
{
SysPushErrorIO("Error: {}", ret);
BZ2_bzDecompressEnd(&strm);
2021-06-27 21:25:29 +00:00
return false;
}
2021-09-06 10:58:08 +00:00
auto have = AuArraySize(out) - strm.avail_out;
2021-06-27 21:25:29 +00:00
{
AuUInt copy = have;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(out, copy));
outputStat += copy;
if (copy != have)
{
SysPushErrorIO();
return false;
}
}
2021-06-27 21:25:29 +00:00
2021-09-06 15:47:35 +00:00
if (stream.reportProgress)
2021-06-27 21:25:29 +00:00
{
2021-09-06 15:47:35 +00:00
if (!stream.reportProgress(inputStat, outputStat))
{
BZ2_bzDecompressEnd(&strm);
return false;
}
2021-06-27 21:25:29 +00:00
}
} while (strm.avail_out == 0);
SysAssert(strm.avail_in == 0);
} while (ret != BZ_FINISH);
BZ2_bzDecompressEnd(&strm);
return true;
2022-03-18 04:30:39 +00:00
#else
return false;
#endif
2021-06-27 21:25:29 +00:00
}
static bool CompressLZ4(const CompressionPipe &stream, const CompressInfo &info)
2021-06-27 21:25:29 +00:00
{
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_LZ4)
2021-09-06 10:58:08 +00:00
bool ret;
LZ4F_cctx *cctxPtr;
LZ4F_preferences_t pref {};
AuUInt32 inputStat = 0, outputStat = 0;
LZ4F_compressOptions_t options {};
char header[512];
2021-09-06 10:58:08 +00:00
ret = true;
pref.compressionLevel = info.uCompressionLevel;
pref.autoFlush = true;
auto maxFrameSize = info.uBlockSize ? info.uBlockSize : 64 * 1024;
auto buffer = AuSPtr<char>(new char[maxFrameSize], AuDefaultDeleter<char[]>());
2021-09-06 10:58:08 +00:00
auto maxOut = LZ4F_compressBound(maxFrameSize, &pref);
auto outBuffer = AuSPtr<char>(new char[maxOut], AuDefaultDeleter<char[]>());;
2021-06-27 21:25:29 +00:00
2021-09-06 10:58:08 +00:00
auto err = LZ4F_createCompressionContext(&cctxPtr, LZ4F_getVersion());
if (LZ4F_isError(err))
2021-06-27 21:25:29 +00:00
{
2021-09-06 10:58:08 +00:00
return false;
}
2021-07-11 17:38:16 +00:00
2021-09-06 10:58:08 +00:00
// Write header
{
auto written = LZ4F_compressBegin(cctxPtr, header, AuArraySize(header), &pref);
AuUInt copy = written;
2022-08-20 06:53:48 +00:00
stream.pWritePipe->Write(AuMemory::MemoryViewStreamRead(header, copy));
SysAssert(copy == written);
2021-09-06 10:58:08 +00:00
}
while (true)
{
AuUInt read = maxFrameSize;
AuIO::EStreamError error;
2022-08-20 06:53:48 +00:00
if (((error = stream.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(buffer.get(), read))) != AuIO::EStreamError::eErrorNone) &&
(error != AuIO::EStreamError::eErrorEndOfStream))
{
break;
}
2021-06-27 21:25:29 +00:00
bool flush = (!read || error == AuIO::EStreamError::eErrorEndOfStream);
if (!flush)
2021-09-06 15:47:35 +00:00
{
2021-09-06 15:47:35 +00:00
AuUInt32 bufferedBytes = LZ4F_compressUpdate(cctxPtr, outBuffer.get(), maxOut, buffer.get(), read, &options);
2021-06-27 21:25:29 +00:00
2021-09-06 15:47:35 +00:00
if (LZ4F_isError(bufferedBytes))
{
ret = false;
break;
}
AuUInt copy = bufferedBytes;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(outBuffer.get(), copy));
outputStat += copy;
if (copy != bufferedBytes)
{
SysPushErrorIO();
return false;
}
2021-09-06 15:47:35 +00:00
}
else
2021-06-27 21:25:29 +00:00
{
2021-09-06 15:47:35 +00:00
AuUInt32 bufferedBytes = LZ4F_compressEnd(cctxPtr, outBuffer.get(), maxOut, &options);
AuUInt copy = bufferedBytes;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(outBuffer.get(), copy));
outputStat += copy;
if (copy != bufferedBytes)
{
SysPushErrorIO();
return false;
}
2021-06-27 21:25:29 +00:00
break;
}
2021-09-06 15:47:35 +00:00
if (stream.reportProgress)
2021-09-06 10:58:08 +00:00
{
2021-09-06 15:47:35 +00:00
if (!stream.reportProgress(inputStat, outputStat))
{
ret = false;
break;
}
2021-09-06 10:58:08 +00:00
}
2021-06-27 21:25:29 +00:00
}
2021-09-06 10:58:08 +00:00
LZ4F_freeCompressionContext(cctxPtr);
2021-06-27 21:25:29 +00:00
return ret;
2022-03-18 04:30:39 +00:00
#else
return false;
#endif
2021-06-27 21:25:29 +00:00
}
static bool DecompressLZ4(const CompressionPipe &pipe, AuUInt32 frameMaxSize)
2021-06-27 21:25:29 +00:00
{
2022-03-18 04:30:39 +00:00
#if defined(_AUHAS_LZ4)
2021-06-27 21:25:29 +00:00
bool ret = true;
2021-09-06 10:58:08 +00:00
LZ4F_dctx *dctxPtr;
AuUInt32 inputStat = 0, outputStat = 0;
size_t bytesRemInFrame {};
2021-09-06 10:58:08 +00:00
LZ4F_decompressOptions_t opts {};
AuSPtr<char> bufferIn, bufferOut;
2022-08-20 06:53:48 +00:00
if (!pipe.pWritePipe)
{
return false;
}
2022-08-20 06:53:48 +00:00
if (!pipe.pReadPipe)
{
return false;
}
2021-09-06 10:58:08 +00:00
auto err = LZ4F_createDecompressionContext(&dctxPtr, LZ4F_getVersion());
if (LZ4F_isError(err))
{
return false;
}
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
bufferIn = AuMakeSharedArray<char>(frameMaxSize);
bufferOut = AuMakeSharedArray<char>(frameMaxSize);
2021-06-27 21:25:29 +00:00
while (true)
{
auto frameSize = bytesRemInFrame ? bytesRemInFrame : LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH;
auto min = frameSize;
2021-09-06 10:58:08 +00:00
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
if (min > frameMaxSize)
2021-06-27 21:25:29 +00:00
{
ret = false;
break;
2021-07-11 11:18:33 +00:00
}
if (frameSize)
2021-07-11 11:18:33 +00:00
{
AuUInt read = frameSize;
2022-08-20 06:53:48 +00:00
if (pipe.pReadPipe->Read(AuMemory::MemoryViewStreamWrite(bufferIn.get(), read)) != AuIO::EStreamError::eErrorNone)
{
break;
}
if (read != frameSize)
2021-09-06 10:58:08 +00:00
{
ret = read == 0 && outputStat;
break;
2021-09-06 10:58:08 +00:00
}
inputStat += frameSize;
2021-09-06 10:58:08 +00:00
}
2021-07-11 11:18:33 +00:00
if (frameSize)
2021-07-11 11:18:33 +00:00
{
auto mustConsume = frameSize;
size_t frameSPtr = mustConsume;
size_t frameS2Ptr = frameMaxSize;
2021-09-06 10:58:08 +00:00
bytesRemInFrame = LZ4F_decompress(dctxPtr, bufferOut.get(), &frameS2Ptr, bufferIn.get(), &frameSPtr, &opts);
if (LZ4F_isError(bytesRemInFrame))
{
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
SysPushErrorIO("{}", LZ4F_getErrorName(bytesRemInFrame));
ret = false;
break;
}
2021-09-06 10:58:08 +00:00
if (frameS2Ptr)
{
AuUInt copy = frameS2Ptr;
2022-08-20 06:53:48 +00:00
auto err = AuIO::WriteAll(pipe.pWritePipe.get(), AuMemory::MemoryViewStreamRead(bufferOut.get(), copy));
outputStat += copy;
2021-09-06 10:58:08 +00:00
if (copy != frameS2Ptr)
{
SysPushErrorIO();
return false;
}
2021-09-06 10:58:08 +00:00
}
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
if (pipe.reportProgress)
2021-09-06 10:58:08 +00:00
{
if (!pipe.reportProgress(inputStat, outputStat))
{
ret = false;
break;
}
2021-09-06 10:58:08 +00:00
}
2021-06-27 21:25:29 +00:00
}
}
2021-09-06 10:58:08 +00:00
LZ4F_freeDecompressionContext(dctxPtr);
2021-06-27 21:25:29 +00:00
return ret;
2022-03-18 04:30:39 +00:00
#else
return false;
#endif
2021-06-27 21:25:29 +00:00
}
static bool DecompressNoOldAssReferenceImpl(const CompressionPipe &stream, const DecompressInfo &meta2)
{
if (!stream.pWritePipe)
{
return false;
}
if (!stream.pReadPipe)
{
return false;
}
DecompressInfo meta3 = meta2;
meta3.uInternalStreamSize = kChunkSize * 10; // stupid, but required for BaseStream::Write to not blow up. not optimizing old apis.
auto pCompressor = DecompressorUnique(stream.pReadPipe, meta3);
if (!pCompressor)
{
SysPushErrorGeneric("No decompressor / inflator");
return false;
}
AuByteBuffer buffer(kChunkSize * 2, true);
AuStreamReadWrittenPair_t pair;
do
{
auto view = buffer.GetNextLinearRead();
view.length = AuMin(view.length, kChunkSize / 2);
auto pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), true);
while (pair.first || pair.second)
{
buffer.writePtr += pair.second;
{
AuUInt copy;
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(buffer, copy));
if (err != AuIO::EStreamError::eErrorNone)
{
return false;
}
buffer.readPtr += copy;
}
if (buffer.writePtr == buffer.readPtr)
{
buffer.readPtr = buffer.writePtr = buffer.base;
}
pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), true);
}
// debugging cope loops
#if 0
while (buffer.readPtr != buffer.writePtr)
{
{
AuUInt copy;
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(buffer, copy));
if (err != AuIO::EStreamError::eErrorNone)
{
return false;
}
buffer.readPtr += copy;
}
}
for (AuUInt i = 0;
i < 5 && (!(pair.first || pair.second));
i++)
{
pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), true);
}
#endif
}
while (pair.first || pair.second);
return true;
}
static bool CompressNoOldAssReferenceImpl(const CompressionPipe &stream, const CompressInfo &meta2)
{
if (!stream.pWritePipe)
{
return false;
}
if (!stream.pReadPipe)
{
return false;
}
CompressInfo meta3 = meta2;
meta3.uInternalStreamSize = kChunkSize / 2;
auto pCompressor = CompressorUnique(stream.pReadPipe, meta3);
if (!pCompressor)
{
SysPushErrorGeneric("No compressor / deflator");
return false;
}
AuByteBuffer buffer(kChunkSize / 2, true);
AuStreamReadWrittenPair_t pair;
do
{
auto view = buffer.GetNextLinearRead();
view.length = AuMin(view.length, kChunkSize / 2);
auto pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), true);
while (pair.first || pair.second)
{
buffer.writePtr += pair.second;
{
AuUInt copy;
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(buffer, copy));
if (err != AuIO::EStreamError::eErrorNone)
{
return false;
}
buffer.readPtr += copy;
}
if (buffer.writePtr == buffer.readPtr)
{
buffer.readPtr = buffer.writePtr = buffer.base;
}
pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), true);
}
}
while (pair.first || pair.second);
pCompressor->Flush();
pCompressor->Finish();
do
{
if (buffer.writePtr == buffer.readPtr)
{
buffer.readPtr = buffer.writePtr = buffer.base;
}
auto view = buffer.GetNextLinearRead();
view.length = AuMin(view.length, kChunkSize / 2);
auto pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), false);
while (pair.first || pair.second)
{
buffer.writePtr += pair.second;
{
AuUInt copy;
auto err = AuIO::WriteAll(stream.pWritePipe.get(), AuMemory::MemoryViewStreamRead(buffer, copy));
if (err != AuIO::EStreamError::eErrorNone)
{
return false;
}
buffer.readPtr += copy;
}
if (buffer.writePtr == buffer.readPtr)
{
buffer.readPtr = buffer.writePtr = buffer.base;
}
pair = pCompressor->ReadEx(AuMemory::MemoryViewWrite(buffer), false);
}
}
while (pair.first || pair.second);
return true;
}
AUKN_SYM bool Compress(const CompressionPipe &stream, const CompressInfo &info)
2021-06-27 21:25:29 +00:00
{
CompressInfo meta = info;
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
AuInt8 bits;
2022-08-20 06:53:48 +00:00
if (!stream.pReadPipe)
{
SysPushErrorArg();
return {};
}
2022-08-20 06:53:48 +00:00
if (!stream.pWritePipe)
{
SysPushErrorArg();
return {};
}
2021-06-27 21:25:29 +00:00
switch (info.type)
{
2022-03-30 14:20:04 +00:00
case ECompressionType::eDeflate:
case ECompressionType::eZip:
case ECompressionType::eGZip:
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
{
if (!CompressionLevelFromExternalApi(meta, bits))
{
return {};
}
return CompressZLib(stream, info.uCompressionLevel, bits);
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
}
2022-03-30 14:20:04 +00:00
case ECompressionType::eZSTD:
2021-06-27 21:25:29 +00:00
return CompressZSTD(stream, info);
2022-03-30 14:20:04 +00:00
case ECompressionType::eBZIP2:
2021-06-27 21:25:29 +00:00
return CompressBZip2(stream, info);
2022-03-30 14:20:04 +00:00
case ECompressionType::eLZ4:
2021-06-27 21:25:29 +00:00
return CompressLZ4(stream, info);
2022-03-30 14:20:04 +00:00
//case ECompressionType::eLZMA:
2021-06-27 21:25:29 +00:00
// return CompressLZMA(stream, info);
default:
return CompressNoOldAssReferenceImpl(stream, info);
2021-06-27 21:25:29 +00:00
}
}
AUKN_SYM bool Decompress(const CompressionPipe &pipe, const DecompressInfo &meta2)
2021-06-27 21:25:29 +00:00
{
DecompressInfo meta = meta2;
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
AuInt8 bits;
2022-08-20 06:53:48 +00:00
if (!pipe.pReadPipe)
{
return {};
}
2022-08-20 06:53:48 +00:00
if (!pipe.pWritePipe)
{
return {};
}
switch (meta.alg)
2021-06-27 21:25:29 +00:00
{
2022-03-30 14:20:04 +00:00
case ECompressionType::eDeflate:
case ECompressionType::eZip:
case ECompressionType::eGZip:
[*/+/-] MEGA COMMIT. ~2 weeks compressed. The intention is to quickly improve and add util apis, enhance functionality given current demands, go back to the build pipeline, finish that, publish runtime tests, and then use what we have to go back to to linux support with a more stable api. [+] AuMakeSharedArray [+] Technet ArgvQuote [+] Grug subsystem (UNIX signal thread async safe ipc + telemetry flusher + log flusher.) [+] auEndianness -> Endian swap utils [+] AuGet<N>(...) [*] AUE_DEFINE conversion for ECompresionType, EAnsiColor, EHashType, EStreamError, EHexDump [+] ConsoleMessage ByteBuffer serialization [+] CmdLine subsystem for parsing command line arguments and simple switch/flag checks [*] Split logger from console subsystem [+] StartupParameters -> A part of a clean up effort under Process [*] Refactor SysErrors header + get caller hack [+] Atomic APIs [+] popcnt [+] Ring Buffer sink [+] Added more standard errors Catch, Submission, LockError, NoAccess, ResourceMissing, ResourceLocked, MalformedData, InSandboxContext, ParseError [+] Added ErrorCategorySet, ErrorCategoryClear, GetStackTrace [+] IExitSubscriber, ETriggerLevel [*] Write bias the high performance RWLockImpl read-lock operation operation [+] ExitHandlerAdd/ExitHandlerRemove (exit subsystem) [*] Updated API style Digests [+] CpuId::CpuBitCount [+] GetUserProgramsFolder [+] GetPackagePath [*] Split IStreamReader with an inl file [*] BlobWriter/BlobReader/BlobArbitraryReader can now take shared pointers to bytebuffers. default constructor allocates a new scalable bytebuffer [+] ICharacterProvider [+] ICharacterProviderEx [+] IBufferedCharacterConsumer [+] ProviderFromSharedString [+] ProviderFromString [+] BufferConsumerFromProvider [*] Parse Subsystem uses character io bufferer [*] Rewritten NT's high perf semaphore to use userland SRW/ConVars [like mutex, based on generic semaphore] [+] ByteBuffer::ResetReadPointer [*] Bug fix bytebuffer base not reset on free and some scaling issues [+] ProcessMap -> Added kSectionNameStack, kSectionNameFile, kSectionNameHeap for Section [*] ProcessMap -> Refactor Segment to Section. I was stupid for keeping a type conflict hack API facing [+] Added 64 *byte* fast RNG seeds [+] File Advisorys/File Lock Awareness [+] Added extended IAuroraThread from OS identifier caches for debug purposes [*] Tweaked how memory is reported on Windows. Better consistency of what values mean across functions. [*] Broke AuroraUtils/Typedefs out into a separate library [*] Update build script [+] Put some more effort into adding detail to the readme before rewriting it, plus, added some media [*] Improved public API documentation [*] Bug fix `SetConsoleCtrlHandler` [+] Locale TimeDateToFileNameISO8601 [+] Console config stdOutShortTime [*] Begin using internal UTF8/16 decoders when platform support isnt available (instead of stl) [*] Bug fixes in decoders [*] Major bug fix, AuMax [+] RateLimiter [+] Binary file sink [+] Log directory sink [*] Data header usability (more operators) [+] AuRemoveRange [+] AuRemove [+] AuTryRemove [+] AuTryRemoveRange [+] auCastUtils [+] Finish NewLSWin32Source [+] AuTryFindByTupleN, AuTryRemoveByTupleN [+] Separated AuRead/Write types, now in auTypeUtils [+] Added GetPosition/SetPosition to FileWriter [*] Fix stupid AuMin in place of AuMax in SpawnThread.Unix.Cpp [*] Refactored Arbitrary readers to SeekingReaders (as in, they could be atomic and/or parallelized, and accept an arbitrary position as a work parameter -> not Seekable, as in, you can simply set the position) [*] Hack back in the sched deinit [+] File AIO loop source interop [+] Begin to prototype a LoopQueue object I had in mind for NT, untested btw [+] Stub code for networking [+] Compression BaseStream/IngestableStreamBase [*] Major: read/write locks now support write-entrant read routines. [*] Compression subsystem now uses the MemoryView concept [*] Rewrite the base stream compressions, made them less broken [*] Update hashing api [*] WriterTryGoForward and ReaderTryGoForward now revert to the previous relative index instead of panicing [+] Added new AuByteBuffer apis Trim, Pad, WriteFrom, WriteString, [TODO: ReadString] [+] Added ByteBufferPushReadState [+] Added ByteBufferPushWriteState [*] Move from USC-16 to full UTF-16. Win32 can handle full UTF-16. [*] ELogLevel is now an Aurora enum [+] Raised arbitrary limit in header to 255, the max filter buffer [+] Explicit GZip support [+] Explicit Zip support [+] Added [some] compressors et al
2022-02-17 00:11:40 +00:00
{
if (!CompressionLevelFromExternalApi(meta, bits))
{
return {};
}
return DecompressZLib(pipe, bits);
}
2022-03-30 14:20:04 +00:00
case ECompressionType::eZSTD:
2021-06-27 21:25:29 +00:00
return DecompressZSTD(pipe);
2022-03-30 14:20:04 +00:00
case ECompressionType::eBZIP2:
2021-06-27 21:25:29 +00:00
return DecompressBZip2(pipe);
2022-03-30 14:20:04 +00:00
case ECompressionType::eLZ4:
return DecompressLZ4(pipe, meta.uInternalStreamSize / 2);
2022-03-30 14:20:04 +00:00
//case ECompressionType::eLZMA:
2021-06-27 21:25:29 +00:00
// return DecompressLZMA(pipe);
default:
return DecompressNoOldAssReferenceImpl(pipe, meta2);
2021-06-27 21:25:29 +00:00
}
}
}