AuroraSupport/AuroraRuntime
2
1
Fork 0
Code Issues 27 Pull Requests Releases Wiki Activity

[+] Reintroduce LZMA support after dropping support for the compression algorithm during the initial rewrite

Browse Source
This commit is contained in:
Reece Wilson 2023-07-05 13:56:08 +01:00
parent e9e72dce09
commit 00cb256f06
6 changed files with 431 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Include/Aurora/Compression/CompressionInfo.hpp
View File

@ -9,6 +9,8 @@
namespace Aurora::Compression
{
using CompressionOptions_t = AuList<AuPair<int, int>>;
struct CompressInfo
{
AU_COPY_MOVE_DEF(CompressInfo);
@ -31,12 +33,12 @@ namespace Aurora::Compression
* BZIP: 0 <= x >= 9
*
*/
AuInt8 uCompressionLevel {6};
AuInt8 uCompressionLevel { 6 };
/**
* @brief LZMA: 5 <= fb <= 273, default = 32
*/
AuUInt32 uFbWordSize {32};
AuUInt32 uFbWordSize { 32 };
/**
* LZMA only
@ -44,10 +46,11 @@ namespace Aurora::Compression
* (1 << 12) <= dictSize <= (3 << 29) for 64-bit version
* default = (1 << 24)
*/
AuUInt32 uDictSize{};
AuUInt32 uDictSize { };
/**
* @brief 64KiB is a recommended "small" block size
* todo: lzma respects this
*/
AuUInt16 uLz4BlockSize {};
@ -70,10 +73,14 @@ namespace Aurora::Compression
*/
AuUInt32 uInternalStreamSize { 512 * 1024 };
AuUInt8 uThreads {1};
AuUInt8 uThreads { 1 };
bool bLZ4AutoFlush {false};
CompressionOptions_t options;
bool bErrorCheck { true };
inline CompressInfo(ECompressionType alg) : type(alg)
{
@ -102,12 +109,18 @@ namespace Aurora::Compression
/**
* @brief Flag for headerless decompression streams
*/
bool bHasWindowbits {true};
bool bHasWindowbits { true };
/**
* @brief Flag for headerless decompression streams
*/
AuInt8 uWindowBits {15};
AuInt8 uWindowBits { 15 };
AuUInt8 uThreads { 1 };
CompressionOptions_t options;
bool bErrorCheck { true };
inline DecompressInfo(ECompressionType alg) : alg(alg)
{

9
Source/Compression/AuBlockCompressor.cpp
View File

@ -30,6 +30,10 @@
#include "Compressors/BrotliCompressor.hpp"
#endif
#if defined(_AUHAS_LIBLZMA)
#include "Compressors/LZMACompressor.hpp"
#endif
namespace Aurora::Compression
{
AUKN_SYM ICompressionStream *CompressorNew(const AuSPtr<IO::IStreamReader> &pReader, const CompressInfo &ref)
@ -66,6 +70,11 @@ namespace Aurora::Compression
pRet = _new BrotliDeflate(info);
break;
#endif
#if defined(_AUHAS_LIBLZMA)
case ECompressionType::eLZMA:
pRet = _new LZMADeflate(info);
break;
#endif
#if defined(_AUHAS_ZLIB)
case ECompressionType::eDeflate:
case ECompressionType::eZip:

9
Source/Compression/AuBlockDecompressor.cpp
View File

@ -30,6 +30,10 @@
#include "Compressors/BrotliDecompressor.hpp"
#endif
#if defined(_AUHAS_LIBLZMA)
#include "Compressors/LZMADecompressor.hpp"
#endif
namespace Aurora::Compression
{
AUKN_SYM ICompressionStream *DecompressorNew(const AuSPtr<IO::IStreamReader> &pReader, const DecompressInfo &ref)
@ -67,6 +71,11 @@ namespace Aurora::Compression
pRet = _new BrotliInflate(info);
break;
#endif
#if defined(_AUHAS_LIBLZMA)
case ECompressionType::eLZMA:
pRet = _new LZMAInflate(info);
break;
#endif
#if defined(_AUHAS_ZLIB)
case ECompressionType::eDeflate:
case ECompressionType::eZip:

242
Source/Compression/Compressors/LZMACompressor.hpp
View File

@ -0,0 +1,242 @@
/***
Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved.
File: LZMACompressor.hpp
Date: 2023-7-2
Author: Reece
***/
#pragma once
#include <lzma.h>
namespace Aurora::Compression
{
struct LZMADeflate : BaseStream
{
CompressInfo meta;
lzma_stream stream;
LZMADeflate(const CompressInfo &meta) : meta(meta), BaseStream(meta.uInternalStreamSize)
{
}
~LZMADeflate()
{
if (this->init_)
{
lzma_end(&this->stream);
}
}
bool Init(const AuSPtr<IO::IStreamReader> &pReader) override
{
this->pReader_ = pReader;
if (!this->IsValid())
{
SysPushErrorMem();
return false;
}
lzma_ret ret {};
if (meta.uCompressionLevel > 9 ||
meta.uCompressionLevel < 0)
{
SysPushErrorArg();
return false;
}
if (!meta.uCompressionLevel)
{
meta.uCompressionLevel = 6;
}
lzma_check check {};
if (meta.bErrorCheck)
{
check = LZMA_CHECK_SHA256;
}
else
{
check = LZMA_CHECK_NONE;
}
if (meta.uThreads > 1)
{
lzma_mt options {};
options.block_size = meta.uLz4BlockSize;
options.preset = meta.uCompressionLevel;
options.check = check;
options.threads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads);
ret = lzma_stream_encoder_mt(&this->stream, &options);
if (ret != LZMA_OK)
{
SysPushErrorGeneric("LZMA Coder failure: {}", (int)ret);
AuResetMember(this->pReader_);
return false;
}
}
else
{
ret = lzma_easy_encoder(&this->stream, meta.uCompressionLevel, check);
if (ret != LZMA_OK)
{
SysPushErrorGeneric("LZMA Coder failure: {}", (int)ret);
AuResetMember(this->pReader_);
return false;
}
}
this->init_ = true;
this->SetArray(this->din_);
this->SetOutArray(this->dout_);
return true;
}
AuStreamReadWrittenPair_t Ingest_s(AuUInt32 input) override
{
AuUInt32 done {}, read {};
if (!this->pReader_)
{
return {};
}
while (read < input)
{
z_stream ctx_ {};
read += IngestForInPointer<const uint8_t, size_t>(this->pReader_, this->stream.next_in, this->stream.avail_in, input - read, this);
if (!this->stream.avail_in)
{
return { read, done };
}
do
{
auto [pMainDOut, uMainDOutLength] = this->GetDOutPair();
this->stream.avail_out = uMainDOutLength;
this->stream.next_out = (Bytef *)pMainDOut;
if (!this->stream.avail_out)
{
break;
}
auto ret = lzma_code(&this->stream, LZMA_RUN);
if (ret < Z_OK)
{
SysPushErrorIO("Error: {}", int(ret));
this->pReader_.reset();
return AuMakePair(read, 0);
}
auto have = uMainDOutLength - this->stream.avail_out;
done += have;
if (!Write2(reinterpret_cast<const AuUInt8 *>(pMainDOut),
have))
{
this->pReader_.reset();
SysPushErrorIO("Compression Out of Overhead");
return AuMakePair(read, 0);
}
}
while (this->stream.avail_out == 0);
}
return { read, done };
}
bool Flush() override
{
return RunFlush();
}
bool Finish() override
{
return RunFlush();
}
bool RunFlush()
{
if (!this->pReader_)
{
return false;
}
while (this->stream.avail_in)
{
do
{
auto [pMainDOut, uMainDOutLength] = this->GetDOutPair();
this->stream.avail_out = uMainDOutLength;
this->stream.next_out = (Bytef *)pMainDOut;
auto ret = lzma_code(&this->stream, LZMA_RUN);
if (ret < Z_OK)
{
SysPushErrorIO("Error: {}", int(ret));
this->pReader_.reset();
return false;
}
auto have = uMainDOutLength - this->stream.avail_out;
if (!Write2(reinterpret_cast<const AuUInt8 *>(this->dout_), have))
{
this->pReader_.reset();
SysPushErrorIO("Compression Out of Overhead");
return false;
}
}
while (this->stream.avail_out == 0);
}
if (!this->stream.avail_in)
{
do
{
auto [pMainDOut, uMainDOutLength] = this->GetDOutPair();
this->stream.avail_out = uMainDOutLength;
this->stream.next_out = (Bytef *)pMainDOut;
auto ret = lzma_code(&this->stream, LZMA_FINISH);
if (ret < Z_OK)
{
SysPushErrorIO("Error: {}", zError(ret));
this->pReader_.reset();
return false;
}
auto have = uMainDOutLength - this->stream.avail_out;
if (!Write2(pMainDOut, have))
{
this->pReader_.reset();
SysPushErrorIO("Compression Out of Overhead");
return false;
}
}
while (this->stream.avail_out == 0);
return true;
}
return true;
}
private:
AuSPtr<IO::IStreamReader> pReader_;
bool init_ {};
unsigned char din_[kChunkSize];
unsigned char dout_[kChunkSize];
};
}

150
Source/Compression/Compressors/LZMADecompressor.hpp
View File

@ -0,0 +1,150 @@
/***
Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved.
File: LZMADecompressor.hpp
Date: 2023-7-2
Author: Reece
***/
#pragma once
#include <lzma.h>
namespace Aurora::Compression
{
struct LZMAInflate : BaseStream
{
DecompressInfo meta;
lzma_stream stream;
LZMAInflate(const DecompressInfo &meta) : meta(meta), BaseStream(meta.uInternalStreamSize)
{
}
~LZMAInflate()
{
if (this->init_)
{
lzma_end(&this->stream);
}
}
bool Init(const AuSPtr<IO::IStreamReader> &pReader) override
{
this->pReader_ = pReader;
if (!this->IsValid())
{
SysPushErrorMem();
return false;
}
lzma_ret ret {};
AuUInt32 uFlags {};
if (meta.bErrorCheck)
{
uFlags = 0;
}
else
{
uFlags = LZMA_IGNORE_CHECK | LZMA_TELL_NO_CHECK;
}
if (meta.uThreads > 1)
{
lzma_mt options {};
options.threads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads);
options.flags = uFlags;
ret = lzma_stream_decoder_mt(&this->stream, &options);
if (ret != LZMA_OK)
{
SysPushErrorGeneric("LZMA Decoder failure: {}", (int)ret);
AuResetMember(this->pReader_);
return false;
}
}
else
{
ret = lzma_stream_decoder(&this->stream, UINT64_MAX, uFlags);
if (ret != LZMA_OK)
{
SysPushErrorGeneric("LZMA Decoder failure: {}", (int)ret);
AuResetMember(this->pReader_);
return false;
}
}
this->init_ = true;
this->SetArray(this->din_);
this->SetOutArray(this->dout_);
return true;
}
AuStreamReadWrittenPair_t Ingest_s(AuUInt32 input) override
{
AuUInt32 done {}, read {};
if (!this->pReader_)
{
return {};
}
while (read < input)
{
z_stream ctx_ {};
read += IngestForInPointer<const uint8_t, size_t>(this->pReader_, this->stream.next_in, this->stream.avail_in, input - read, this);
if (!this->stream.avail_in)
{
return { read, done };
}
do
{
auto [pMainDOut, uMainDOutLength] = this->GetDOutPair();
this->stream.avail_out = uMainDOutLength;
this->stream.next_out = (Bytef *)pMainDOut;
if (!this->stream.avail_out)
{
break;
}
auto ret = lzma_code(&this->stream, LZMA_RUN);
if (ret < Z_OK)
{
SysPushErrorIO("Error: {}", int(ret));
this->pReader_.reset();
return AuMakePair(read, 0);
}
auto have = uMainDOutLength - this->stream.avail_out;
done += have;
if (!Write2(reinterpret_cast<const AuUInt8 *>(pMainDOut),
have))
{
this->pReader_.reset();
SysPushErrorIO("Compression Out of Overhead");
return AuMakePair(read, 0);
}
}
while (this->stream.avail_out == 0);
}
return { read, done };
}
private:
AuSPtr<IO::IStreamReader> pReader_;
bool init_ {};
unsigned char din_[kChunkSize];
unsigned char dout_[kChunkSize];
};
}

2
Source/Compression/Compressors/ZSTDCompressor.hpp
View File

@ -58,6 +58,8 @@ namespace Aurora::Compression
return false;
}
meta.uThreads = AuMin(AuHwInfo::GetCPUInfo().uThreads, meta.uThreads);
uRet = ZSTD_CCtx_setParameter(this->cctx_, ZSTD_c_nbWorkers, AuMax(meta.uThreads, AuUInt8(1u)));
if (ZSTD_isError(uRet))
{