diff --git a/Include/Aurora/Compression/CompressionInfo.hpp b/Include/Aurora/Compression/CompressionInfo.hpp index 7751aec4..9656839d 100644 --- a/Include/Aurora/Compression/CompressionInfo.hpp +++ b/Include/Aurora/Compression/CompressionInfo.hpp @@ -9,6 +9,8 @@ namespace Aurora::Compression { + using CompressionOptions_t = AuList>; + struct CompressInfo { AU_COPY_MOVE_DEF(CompressInfo); @@ -31,12 +33,12 @@ namespace Aurora::Compression * BZIP: 0 <= x >= 9 * */ - AuInt8 uCompressionLevel {6}; + AuInt8 uCompressionLevel { 6 }; /** * @brief LZMA: 5 <= fb <= 273, default = 32 */ - AuUInt32 uFbWordSize {32}; + AuUInt32 uFbWordSize { 32 }; /** * LZMA only @@ -44,10 +46,11 @@ namespace Aurora::Compression * (1 << 12) <= dictSize <= (3 << 29) for 64-bit version * default = (1 << 24) */ - AuUInt32 uDictSize{}; + AuUInt32 uDictSize { }; /** * @brief 64KiB is a recommended "small" block size + * todo: lzma respects this */ AuUInt16 uLz4BlockSize {}; @@ -70,10 +73,14 @@ namespace Aurora::Compression */ AuUInt32 uInternalStreamSize { 512 * 1024 }; - AuUInt8 uThreads {1}; + AuUInt8 uThreads { 1 }; bool bLZ4AutoFlush {false}; + CompressionOptions_t options; + + bool bErrorCheck { true }; + inline CompressInfo(ECompressionType alg) : type(alg) { @@ -102,12 +109,18 @@ namespace Aurora::Compression /** * @brief Flag for headerless decompression streams */ - bool bHasWindowbits {true}; + bool bHasWindowbits { true }; /** * @brief Flag for headerless decompression streams */ - AuInt8 uWindowBits {15}; + AuInt8 uWindowBits { 15 }; + + AuUInt8 uThreads { 1 }; + + CompressionOptions_t options; + + bool bErrorCheck { true }; inline DecompressInfo(ECompressionType alg) : alg(alg) { diff --git a/Source/Compression/AuBlockCompressor.cpp b/Source/Compression/AuBlockCompressor.cpp index 42aa54b5..ee6ae95d 100644 --- a/Source/Compression/AuBlockCompressor.cpp +++ b/Source/Compression/AuBlockCompressor.cpp @@ -30,6 +30,10 @@ #include "Compressors/BrotliCompressor.hpp" #endif +#if defined(_AUHAS_LIBLZMA) + #include "Compressors/LZMACompressor.hpp" +#endif + namespace Aurora::Compression { AUKN_SYM ICompressionStream *CompressorNew(const AuSPtr &pReader, const CompressInfo &ref) @@ -66,6 +70,11 @@ namespace Aurora::Compression pRet = _new BrotliDeflate(info); break; #endif + #if defined(_AUHAS_LIBLZMA) + case ECompressionType::eLZMA: + pRet = _new LZMADeflate(info); + break; + #endif #if defined(_AUHAS_ZLIB) case ECompressionType::eDeflate: case ECompressionType::eZip: diff --git a/Source/Compression/AuBlockDecompressor.cpp b/Source/Compression/AuBlockDecompressor.cpp index 37417b8a..4bb43a47 100644 --- a/Source/Compression/AuBlockDecompressor.cpp +++ b/Source/Compression/AuBlockDecompressor.cpp @@ -30,6 +30,10 @@ #include "Compressors/BrotliDecompressor.hpp" #endif +#if defined(_AUHAS_LIBLZMA) + #include "Compressors/LZMADecompressor.hpp" +#endif + namespace Aurora::Compression { AUKN_SYM ICompressionStream *DecompressorNew(const AuSPtr &pReader, const DecompressInfo &ref) @@ -67,6 +71,11 @@ namespace Aurora::Compression pRet = _new BrotliInflate(info); break; #endif + #if defined(_AUHAS_LIBLZMA) + case ECompressionType::eLZMA: + pRet = _new LZMAInflate(info); + break; + #endif #if defined(_AUHAS_ZLIB) case ECompressionType::eDeflate: case ECompressionType::eZip: diff --git a/Source/Compression/Compressors/LZMACompressor.hpp b/Source/Compression/Compressors/LZMACompressor.hpp index e69de29b..a5e8ddf2 100644 --- a/Source/Compression/Compressors/LZMACompressor.hpp +++ b/Source/Compression/Compressors/LZMACompressor.hpp @@ -0,0 +1,242 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: LZMACompressor.hpp + Date: 2023-7-2 + Author: Reece +***/ +#pragma once + +#include + +namespace Aurora::Compression +{ + struct LZMADeflate : BaseStream + { + CompressInfo meta; + lzma_stream stream; + + LZMADeflate(const CompressInfo &meta) : meta(meta), BaseStream(meta.uInternalStreamSize) + { + + } + + ~LZMADeflate() + { + if (this->init_) + { + lzma_end(&this->stream); + } + } + + bool Init(const AuSPtr &pReader) override + { + this->pReader_ = pReader; + + if (!this->IsValid()) + { + SysPushErrorMem(); + return false; + } + + lzma_ret ret {}; + + if (meta.uCompressionLevel > 9 || + meta.uCompressionLevel < 0) + { + SysPushErrorArg(); + return false; + } + + if (!meta.uCompressionLevel) + { + meta.uCompressionLevel = 6; + } + + lzma_check check {}; + if (meta.bErrorCheck) + { + check = LZMA_CHECK_SHA256; + } + else + { + check = LZMA_CHECK_NONE; + } + + if (meta.uThreads > 1) + { + lzma_mt options {}; + options.block_size = meta.uLz4BlockSize; + options.preset = meta.uCompressionLevel; + options.check = check; + options.threads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads); + ret = lzma_stream_encoder_mt(&this->stream, &options); + + if (ret != LZMA_OK) + { + SysPushErrorGeneric("LZMA Coder failure: {}", (int)ret); + AuResetMember(this->pReader_); + return false; + } + } + else + { + ret = lzma_easy_encoder(&this->stream, meta.uCompressionLevel, check); + + if (ret != LZMA_OK) + { + SysPushErrorGeneric("LZMA Coder failure: {}", (int)ret); + AuResetMember(this->pReader_); + return false; + } + } + + this->init_ = true; + + this->SetArray(this->din_); + this->SetOutArray(this->dout_); + return true; + } + + AuStreamReadWrittenPair_t Ingest_s(AuUInt32 input) override + { + AuUInt32 done {}, read {}; + + if (!this->pReader_) + { + return {}; + } + + while (read < input) + { + z_stream ctx_ {}; + + read += IngestForInPointer(this->pReader_, this->stream.next_in, this->stream.avail_in, input - read, this); + + if (!this->stream.avail_in) + { + return { read, done }; + } + + do + { + auto [pMainDOut, uMainDOutLength] = this->GetDOutPair(); + this->stream.avail_out = uMainDOutLength; + this->stream.next_out = (Bytef *)pMainDOut; + + if (!this->stream.avail_out) + { + break; + } + + auto ret = lzma_code(&this->stream, LZMA_RUN); + if (ret < Z_OK) + { + SysPushErrorIO("Error: {}", int(ret)); + this->pReader_.reset(); + return AuMakePair(read, 0); + } + + auto have = uMainDOutLength - this->stream.avail_out; + done += have; + + if (!Write2(reinterpret_cast(pMainDOut), + have)) + { + this->pReader_.reset(); + SysPushErrorIO("Compression Out of Overhead"); + return AuMakePair(read, 0); + } + + } + while (this->stream.avail_out == 0); + } + + return { read, done }; + } + + bool Flush() override + { + return RunFlush(); + } + + bool Finish() override + { + return RunFlush(); + } + + bool RunFlush() + { + if (!this->pReader_) + { + return false; + } + + while (this->stream.avail_in) + { + do + { + auto [pMainDOut, uMainDOutLength] = this->GetDOutPair(); + this->stream.avail_out = uMainDOutLength; + this->stream.next_out = (Bytef *)pMainDOut; + + auto ret = lzma_code(&this->stream, LZMA_RUN); + if (ret < Z_OK) + { + SysPushErrorIO("Error: {}", int(ret)); + this->pReader_.reset(); + return false; + } + + auto have = uMainDOutLength - this->stream.avail_out; + + if (!Write2(reinterpret_cast(this->dout_), have)) + { + this->pReader_.reset(); + SysPushErrorIO("Compression Out of Overhead"); + return false; + } + } + while (this->stream.avail_out == 0); + } + + if (!this->stream.avail_in) + { + do + { + auto [pMainDOut, uMainDOutLength] = this->GetDOutPair(); + this->stream.avail_out = uMainDOutLength; + this->stream.next_out = (Bytef *)pMainDOut; + + auto ret = lzma_code(&this->stream, LZMA_FINISH); + if (ret < Z_OK) + { + SysPushErrorIO("Error: {}", zError(ret)); + this->pReader_.reset(); + return false; + } + + auto have = uMainDOutLength - this->stream.avail_out; + + if (!Write2(pMainDOut, have)) + { + this->pReader_.reset(); + SysPushErrorIO("Compression Out of Overhead"); + return false; + } + } + while (this->stream.avail_out == 0); + + return true; + } + + return true; + } + + private: + AuSPtr pReader_; + bool init_ {}; + unsigned char din_[kChunkSize]; + unsigned char dout_[kChunkSize]; + }; +} \ No newline at end of file diff --git a/Source/Compression/Compressors/LZMADecompressor.hpp b/Source/Compression/Compressors/LZMADecompressor.hpp index e69de29b..ccf1c36f 100644 --- a/Source/Compression/Compressors/LZMADecompressor.hpp +++ b/Source/Compression/Compressors/LZMADecompressor.hpp @@ -0,0 +1,150 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: LZMADecompressor.hpp + Date: 2023-7-2 + Author: Reece +***/ +#pragma once + +#include + +namespace Aurora::Compression +{ + struct LZMAInflate : BaseStream + { + DecompressInfo meta; + lzma_stream stream; + + LZMAInflate(const DecompressInfo &meta) : meta(meta), BaseStream(meta.uInternalStreamSize) + { + + } + + ~LZMAInflate() + { + if (this->init_) + { + lzma_end(&this->stream); + } + } + + bool Init(const AuSPtr &pReader) override + { + this->pReader_ = pReader; + + if (!this->IsValid()) + { + SysPushErrorMem(); + return false; + } + + lzma_ret ret {}; + + AuUInt32 uFlags {}; + if (meta.bErrorCheck) + { + uFlags = 0; + } + else + { + uFlags = LZMA_IGNORE_CHECK | LZMA_TELL_NO_CHECK; + } + + if (meta.uThreads > 1) + { + lzma_mt options {}; + options.threads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads); + options.flags = uFlags; + ret = lzma_stream_decoder_mt(&this->stream, &options); + + if (ret != LZMA_OK) + { + SysPushErrorGeneric("LZMA Decoder failure: {}", (int)ret); + AuResetMember(this->pReader_); + return false; + } + } + else + { + ret = lzma_stream_decoder(&this->stream, UINT64_MAX, uFlags); + + if (ret != LZMA_OK) + { + SysPushErrorGeneric("LZMA Decoder failure: {}", (int)ret); + AuResetMember(this->pReader_); + return false; + } + } + + this->init_ = true; + + this->SetArray(this->din_); + this->SetOutArray(this->dout_); + return true; + } + + AuStreamReadWrittenPair_t Ingest_s(AuUInt32 input) override + { + AuUInt32 done {}, read {}; + + if (!this->pReader_) + { + return {}; + } + + while (read < input) + { + z_stream ctx_ {}; + + read += IngestForInPointer(this->pReader_, this->stream.next_in, this->stream.avail_in, input - read, this); + + if (!this->stream.avail_in) + { + return { read, done }; + } + + do + { + auto [pMainDOut, uMainDOutLength] = this->GetDOutPair(); + this->stream.avail_out = uMainDOutLength; + this->stream.next_out = (Bytef *)pMainDOut; + + if (!this->stream.avail_out) + { + break; + } + + auto ret = lzma_code(&this->stream, LZMA_RUN); + if (ret < Z_OK) + { + SysPushErrorIO("Error: {}", int(ret)); + this->pReader_.reset(); + return AuMakePair(read, 0); + } + + auto have = uMainDOutLength - this->stream.avail_out; + done += have; + + if (!Write2(reinterpret_cast(pMainDOut), + have)) + { + this->pReader_.reset(); + SysPushErrorIO("Compression Out of Overhead"); + return AuMakePair(read, 0); + } + + } + while (this->stream.avail_out == 0); + } + + return { read, done }; + } + + private: + AuSPtr pReader_; + bool init_ {}; + unsigned char din_[kChunkSize]; + unsigned char dout_[kChunkSize]; + }; +} \ No newline at end of file diff --git a/Source/Compression/Compressors/ZSTDCompressor.hpp b/Source/Compression/Compressors/ZSTDCompressor.hpp index b29c9ec7..d13f338b 100644 --- a/Source/Compression/Compressors/ZSTDCompressor.hpp +++ b/Source/Compression/Compressors/ZSTDCompressor.hpp @@ -58,6 +58,8 @@ namespace Aurora::Compression return false; } + meta.uThreads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads); + uRet = ZSTD_CCtx_setParameter(this->cctx_, ZSTD_c_nbWorkers, AuMax(meta.uThreads, AuUInt8(1u))); if (ZSTD_isError(uRet)) {