/*** Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: FSCompress.cpp Date: 2023-1-26 Author: Reece ***/ #include #include "FS.hpp" #include "FS.Generic.hpp" #include namespace Aurora::IO::FS { static const AuUInt64 kFileCopyBlock = 0xFFFF * 4; // 64KiB static auto const kCompressionType = AuCompression::ECompressionType::eZSTD; static const AuString kStringSuffix = ".zst"; static bool CompressEx2(const AuString &path, const AuString &suffix, AuCompression::ECompressionType type, AuInt8 level, bool bCheck) { static const auto kCompressionReadChunks = kFileCopyBlock; if (suffix.empty()) { SysPushErrorInvalidArgPos2(); return {}; } if (bCheck) { if (AuEndsWith(path, suffix)) { SysPushErrorIO("File path (\"{}\") ends in {}, yet was requested to be compressed. A user or a developer is probably being stupid.", path, suffix); return false; } } auto pFileSrc = OpenReadShared(path, EFileAdvisoryLockLevel::eBlockWrite); if (!pFileSrc) { SysPushErrorIO("Couldn't open compression source path: {}", path); return {}; } auto pFileDest = OpenWriteShared(path + suffix, EFileAdvisoryLockLevel::eBlockReadWrite); if (!pFileDest) { SysPushErrorIO("Couldn't open compression destination path: {}{}", path, suffix); return {}; } if (pFileDest->GetLength()) { SysPushErrorIO("File ({}) exists", path); return {}; } auto qwLength = pFileSrc->GetLength(); auto pFileStream = AuMakeShared(pFileSrc); if (!pFileStream) { SysPushErrorMemory(); return {}; } auto pDestStream = AuMakeShared(pFileDest); if (!pDestStream) { SysPushErrorMemory(); return {}; } AuCompression::CompressInfo compress { type }; compress.uCompressionLevel = level; compress.uInternalStreamSize = kCompressionReadChunks * 2; auto pCompressor = AuCompression::CompressorUnique(pFileStream, compress); if (!pCompressor) { SysPushErrorMemory("no compressor"); return {}; } AuUInt64 qwTotalRead {}; { AuByteBuffer tempMemory(kCompressionReadChunks); if (!tempMemory) { SysPushErrorMemory(); return {}; } for (AuUInt64 i = 0; i < qwLength; i += 0 /*kCompressionReadChunks*/) { auto [read, written] = pCompressor->Ingest(kCompressionReadChunks); i += read; if (read == 0) { break; } qwTotalRead += read; bool bAnyWritten {}; while (auto uBytes = pCompressor->Read(tempMemory)) { tempMemory.writePtr += uBytes; bAnyWritten = true; AuUInt idc {}; if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) { SysPushErrorIO("AuIO::WriteAll failed"); return {}; } tempMemory.writePtr = tempMemory.base; tempMemory.readPtr = tempMemory.base; } // zstd u ok? // if i remove this, we get no data at all. // even if i try to displace this to above `->Finish()` we still get nothing // preemptively flushing fixes everything /shrug pCompressor->Flush(); } pCompressor->Finish(); while (auto uBytes = pCompressor->Read(tempMemory)) { tempMemory.writePtr += uBytes; AuUInt idc {}; if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) { SysPushErrorIO("AuIO::WriteAll failed"); return {}; } tempMemory.writePtr = tempMemory.base; tempMemory.readPtr = tempMemory.base; } } return qwTotalRead == qwLength; } AUKN_SYM bool CompressEx(const AuString &path, const AuString &suffix, AuCompression::ECompressionType type, AuInt8 level) { return CompressEx2(path, kStringSuffix, kCompressionType, level, false); } AUKN_SYM bool Compress(const AuString &path, AuInt8 level) { return CompressEx2(path, kStringSuffix, kCompressionType, level, true); } AUKN_SYM bool DecompressEx(const AuString &path, const AuString &suffix, AuCompression::ECompressionType type) { static const auto kCompressionReadChunks = kFileCopyBlock; if (suffix.empty()) { SysPushErrorInvalidArgPos2(); return {}; } auto pFileSrc = OpenReadShared(path + suffix, EFileAdvisoryLockLevel::eBlockWrite); if (!pFileSrc) { SysPushErrorIO("Couldn't open compression source path: {}{}", path, suffix); return {}; } auto pFileDest = OpenWriteShared(path, EFileAdvisoryLockLevel::eBlockReadWrite); if (!pFileDest) { SysPushErrorIO("Couldn't open decompression destination path: {}", path); return {}; } if (pFileDest->GetLength()) { SysPushErrorIO("File exists path: {}", path); return {}; } auto qwLength = pFileSrc->GetLength(); auto pFileStream = AuMakeShared(pFileSrc); if (!pFileStream) { SysPushErrorMemory(); return {}; } auto pDestStream = AuMakeShared(pFileDest); if (!pDestStream) { SysPushErrorMemory(); return {}; } AuCompression::DecompressInfo decompress{ type }; decompress.uInternalStreamSize = kCompressionReadChunks * 5; auto pDecompressor = AuCompression::DecompressorUnique(pFileStream, decompress); if (!pDecompressor) { SysPushErrorMemory("no decompressor"); return {}; } AuUInt64 qwTotalRead {}; { AuByteBuffer tempMemory(kCompressionReadChunks); if (!tempMemory) { SysPushErrorMemory(); return {}; } for (AuUInt64 i = 0; i < qwLength; i += 0 /*kCompressionReadChunks*/) { auto [read, written] = pDecompressor->Ingest(kCompressionReadChunks); i += read; if (read == 0) { break; } qwTotalRead += read; while (auto uBytes = pDecompressor->Read(tempMemory)) { tempMemory.writePtr += uBytes; AuUInt idc {}; if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) { SysPushErrorIO("AuIO::WriteAll failed"); return {}; } tempMemory.writePtr = tempMemory.base; tempMemory.readPtr = tempMemory.base; } } } return qwTotalRead == qwLength; } AUKN_SYM bool Decompress(const AuString &path) { return DecompressEx(path, kStringSuffix, kCompressionType); } }