diff --git a/Aurora.json b/Aurora.json index 1252ce56..8adc2b95 100644 --- a/Aurora.json +++ b/Aurora.json @@ -20,6 +20,7 @@ "wxwidgets", "glm", "bzip2", + "liblzma", "lz4" ], "depends": [ diff --git a/Include/Aurora/IO/FS/FS.hpp b/Include/Aurora/IO/FS/FS.hpp index f7662197..ad44a384 100644 --- a/Include/Aurora/IO/FS/FS.hpp +++ b/Include/Aurora/IO/FS/FS.hpp @@ -44,6 +44,14 @@ namespace Aurora::IO::FS */ AUKN_SYM bool DirDeleter(const AuString &string); + /** + * @brief + * @param string + * @param failingPaths + * @return + */ + AUKN_SYM bool DirDeleterEx(const AuString &string, AuList &failingPaths); + /** * @brief Writes a blob into a file chunk-by-chunk. * The directory structure may or may not exist for the write operation to succeed. @@ -143,18 +151,33 @@ namespace Aurora::IO::FS AUKN_SYM bool BlockFile(const AuString &path); /** - * @brief Specifies generic level of trust + * @brief Specifies generic local-system/trusted level of trust * @param path * @return */ AUKN_SYM bool UnblockFile(const AuString &path); /** - * @brief Specifies user/internal level trust of a file + * @brief Specifies user/executable level trust of a file + * @warning This routine is intended to enable execution of files + * on both UNIX and NT based systems. UnblockFile + * will be enough for resources and some powershell scripts; + * however, this is required for unblocking / `mode |= 0111`ing + * executable files. * @param path * @return */ AUKN_SYM bool TrustFile(const AuString &path); + + /** + * @brief Transfers the contents of the specified filepath through a + * zstandard compression pipe to an ending path + ".zst" file. + * @warning This file API does not relate to file-system level compression + * @param path = ur mother + */ + AUKN_SYM bool Compress(const AuString &path, AuInt8 level = 17); + + AUKN_SYM bool Decompress(const AuString &path); /** * @brief Normalizes an arbitrary string of in @@ -197,4 +220,5 @@ namespace Aurora::IO::FS #include "IAsyncFileStream.hpp" #include "Async.hpp" #include "Watcher.hpp" -#include "IReadDir.hpp" \ No newline at end of file +#include "IReadDir.hpp" +#include "Overlapped.hpp" \ No newline at end of file diff --git a/Include/Aurora/IO/FS/Overlapped.hpp b/Include/Aurora/IO/FS/Overlapped.hpp new file mode 100644 index 00000000..c554bdb3 --- /dev/null +++ b/Include/Aurora/IO/FS/Overlapped.hpp @@ -0,0 +1,69 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: Overlapped.hpp + Date: 2023-1-26 + Author: Reece + Note: Defer to Async.hpp for creation of overlapped stream objects + This header defines FS.hpp-like APIs that run overlapped or on a + worker thread or two. +***/ +#pragma once + +namespace Aurora::IO::Loop +{ + struct ILoopSource; +} + +namespace Aurora::IO::FS +{ + struct IOverlappedOperationBase; + + AUKN_INTERFACE(IOverlappedCallback, + AUI_METHOD(void, OnSuccess, (AuSPtr, pOriginator)), + AUI_METHOD(void, OnFailure, (AuSPtr, pOriginator)) + ); + + struct IOverlappedOperationBase + { + virtual AuSPtr ToWaitable() = 0; + + virtual AuSPtr SetCallback(AuSPtr pCallback) = 0; + + virtual bool IsOperationComplete() = 0; + virtual bool IsOperationSuccessful() = 0; + }; + + struct IOverlappedStatOperation : IOverlappedOperationBase + { + virtual AuSPtr GetStats() = 0; + }; + + struct IOverlappedReadOperation : IOverlappedOperationBase + { + virtual AuSPtr GetByteBuffer() = 0; + virtual AuSPtr GetReadView() = 0; + }; + + /** + * @brief If running under AuAsync runner, we use the local thread for the entire IO operation. + * In the future, whence I can be bothered to finish the APC api, *all* overlapped operations + * will be processed on the local thread. + * Specifying true forces all IO to occour on a preallocated pool of IO workers + * @warning this api is thread-local + * @warning this api does not change the behaviour of overlapped file streamss + */ + AUKN_SYM bool OverlappedForceDelegatedIO(bool bForceIOWorkerThreads); + + AUKN_SYM AuSPtr OverlappedCompress(const AuString &path, AuInt8 iLevel = 17); + AUKN_SYM AuSPtr OverlappedDecompress(const AuString &path); + AUKN_SYM AuSPtr OverlappedWrite(const AuString &path, AuSPtr pMemoryView); + AUKN_SYM AuSPtr OverlappedRead(const AuString &path); + AUKN_SYM AuSPtr OverlappedStat(const AuString &path); + AUKN_SYM AuSPtr OverlappedCopy(const AuString &path, const AuString &dest); + AUKN_SYM AuSPtr OverlappedRelink(const AuString &path, const AuString &dest); + AUKN_SYM AuSPtr OverlappedTrustFile(const AuString &path); + AUKN_SYM AuSPtr OverlappedBlockFile(const AuString &path); + AUKN_SYM AuSPtr OverlappedUnblockFile(const AuString &path); + AUKN_SYM AuSPtr OverlappedDelete(const AuString &path); +} \ No newline at end of file diff --git a/Include/Aurora/IO/Net/INetSrvSockets.hpp b/Include/Aurora/IO/Net/INetSrvSockets.hpp index beb0beb3..94df1374 100644 --- a/Include/Aurora/IO/Net/INetSrvSockets.hpp +++ b/Include/Aurora/IO/Net/INetSrvSockets.hpp @@ -11,9 +11,22 @@ namespace Aurora::IO::Net { struct ISocketServer; + struct NetSocketConnectByHost + { + AuOptionalEx netHostname; + AuOptionalEx uPort; + AuOptionalEx protocol; + }; + struct NetSocketConnect { - NetEndpoint endpoint; + // connect by protocol, address:port + AuOptionalEx endpoint; + + // or connect by [string/ip family, ip address], port + NetSocketConnectByHost byHost; + + // AuSPtr pDriver; AuUInt32 uMaxConnectTimeMs {}; }; diff --git a/Include/Aurora/IO/Net/NetHostname.hpp b/Include/Aurora/IO/Net/NetHostname.hpp index 5732a936..3a548072 100644 --- a/Include/Aurora/IO/Net/NetHostname.hpp +++ b/Include/Aurora/IO/Net/NetHostname.hpp @@ -11,12 +11,16 @@ namespace Aurora::IO::Net { struct AUKN_SYM NetHostname { + NetHostname(const NetHostname &cpy); + NetHostname(); NetHostname(const AuString &hostname); NetHostname(const IPAddress &ipAddress); - const EHostnameType type; - const AuString hostname; - const IPAddress address; + EHostnameType type; + AuString hostname; + IPAddress address; + + NetHostname &operator =(const NetHostname &other); bool operator ==(const NetHostname &other) const; const AuUInt HashCode() const; diff --git a/Include/Aurora/Logging/Sinks.hpp b/Include/Aurora/Logging/Sinks.hpp index 17ec5bf8..05ec8422 100644 --- a/Include/Aurora/Logging/Sinks.hpp +++ b/Include/Aurora/Logging/Sinks.hpp @@ -16,8 +16,11 @@ namespace Aurora::Logging { struct DirectoryLogger { - AuUInt32 maxLogsOrZero {}; - AuUInt32 maxFileSizeOrZero {}; // MB + AuUInt32 uMaxLogsOrZeroBeforeDelete {}; + AuUInt32 uMaxCumulativeFileSizeInMiBOrZeroBeforeDelete {}; + AuUInt32 uMaxCumulativeFileSizeInMiBOrZeroBeforeCompress {}; + AuUInt32 uMaxFileTimeInDeltaMSOrZeroBeforeCompress {}; + AuUInt32 uMaxFileTimeInDeltaMSOrZeroBeforeDelete {}; }; /** diff --git a/Include/Aurora/Memory/ByteBuffer.hpp b/Include/Aurora/Memory/ByteBuffer.hpp index 8c24ce7c..f3b40a89 100644 --- a/Include/Aurora/Memory/ByteBuffer.hpp +++ b/Include/Aurora/Memory/ByteBuffer.hpp @@ -116,7 +116,13 @@ namespace Aurora::Memory */ inline ByteBuffer(const ByteBuffer &buffer, bool preservePointers = true) { - this->base = FAlloc(buffer.length); + if (buffer.length) + { + this->base = FAlloc(buffer.length); + } + this->scaleSize = buffer.scaleSize; + this->flagCircular = buffer.flagCircular; + this->flagExpandable = buffer.flagExpandable; if (!this->base) { Reset(); @@ -135,9 +141,6 @@ namespace Aurora::Memory this->readPtr = this->base; } AuMemcpy(this->base, buffer.base, this->length); - this->flagCircular = buffer.flagCircular; - this->flagExpandable = buffer.flagExpandable; - this->scaleSize = buffer.scaleSize; } /** @@ -149,7 +152,8 @@ namespace Aurora::Memory */ inline ByteBuffer(const void *in, AuUInt length, bool circular = false, bool expandable = false) : flagCircular(circular), flagExpandable(expandable), flagReadError(0), flagWriteError(0) { - this->base = FAlloc(length); + this->scaleSize = kBufferInitialPower; + this->base = length ? FAlloc(length) : nullptr; if (!this->base) { Reset(); @@ -160,12 +164,12 @@ namespace Aurora::Memory this->readPtr = this->base; this->writePtr = this->readPtr + this->length; AuMemcpy(this->base, in, this->length); - this->scaleSize = kBufferInitialPower; } inline ByteBuffer(const AuList &vector, bool circular = false, bool expandable = false) : flagCircular(circular), flagExpandable(expandable), flagReadError(0), flagWriteError(0) { - this->base = FAlloc(vector.size()); + this->scaleSize = kBufferInitialPower; + this->base = vector.size() ? FAlloc(vector.size()) : nullptr; if (!this->base) { Reset(); @@ -176,11 +180,11 @@ namespace Aurora::Memory this->readPtr = this->base; this->writePtr = this->readPtr + this->length; AuMemcpy(this->base, vector.data(), this->length); - this->scaleSize = kBufferInitialPower; } inline ByteBuffer(AuUInt length, bool circular = false, bool expandable = false) : flagCircular(circular), flagExpandable(expandable), flagReadError(0), flagWriteError(0) { + this->scaleSize = kBufferInitialPower; if (!length) { Reset(); @@ -196,7 +200,6 @@ namespace Aurora::Memory this->allocSize = length; this->readPtr = this->base; this->writePtr = this->base; - this->scaleSize = kBufferInitialPower; } inline ByteBuffer(AuUInt length, AuUInt alignment, bool circular = false, bool expandable = false) : flagCircular(circular), flagExpandable(expandable), flagReadError(0), flagWriteError(0) @@ -206,6 +209,7 @@ namespace Aurora::Memory Reset(); return; } + this->scaleSize = kBufferInitialPower; this->base = ZAlloc(length, alignment); if (!this->base) { @@ -216,14 +220,14 @@ namespace Aurora::Memory this->allocSize = length; this->readPtr = this->base; this->writePtr = this->base; - this->scaleSize = kBufferInitialPower; } template ByteBuffer(T *base, T *end, bool circular = false, bool expandable = false) : flagCircular(circular), flagExpandable(expandable), flagReadError(0), flagWriteError(0) { auto length = static_cast(end - base) * sizeof(T); - this->base = ZAlloc(length); + this->base = length ? ZAlloc(length) : nullptr; + this->scaleSize = kBufferInitialPower; if (!this->base) { Reset(); @@ -233,7 +237,6 @@ namespace Aurora::Memory this->allocSize = length; this->readPtr = this->base; this->writePtr = this->base + length; - this->scaleSize = kBufferInitialPower; AuMemcpy(this->base, base, length); } @@ -285,14 +288,14 @@ namespace Aurora::Memory * @warning writers should use ::GetLinearWriteable(uDesiredLength) or ::GetOrAllocateLinearWriteable(...) * @return */ - inline auline AuUInt8 * begin() const; + inline auline const AuUInt8 * begin() const; /** * @brief linear read end * @warning writers should use ::GetLinearWriteable(uDesiredLength) or ::GetOrAllocateLinearWriteable(...) * @return */ - inline auline AuUInt8 * end() const; + inline auline const AuUInt8 * end() const; inline auline bool empty() const; diff --git a/Include/Aurora/Memory/ByteBuffer_Memory.inl b/Include/Aurora/Memory/ByteBuffer_Memory.inl index fbaa670d..e887ba8b 100644 --- a/Include/Aurora/Memory/ByteBuffer_Memory.inl +++ b/Include/Aurora/Memory/ByteBuffer_Memory.inl @@ -16,6 +16,16 @@ namespace Aurora::Memory Free(this->base); this->base = nullptr; } + + if (!length) + { + this->length = length; + this->allocSize = length; + this->readPtr = this->base; + this->writePtr = this->base; + return true; + } + this->base = fast ? FAlloc(length) : ZAlloc(length); if (!this->base) { @@ -35,11 +45,22 @@ namespace Aurora::Memory Free(this->base); this->base = nullptr; } + + if (!length) + { + this->length = length; + this->allocSize = length; + this->readPtr = this->base; + this->writePtr = this->base; + return true; + } + this->base = fast ? FAlloc(length, alignment) : ZAlloc(length, alignment); if (!this->base) { return false; } + this->length = length; this->allocSize = length; this->readPtr = this->base; @@ -119,9 +140,9 @@ namespace Aurora::Memory if (length == 0) { - this->length = 0; + this->length = 0; this->writePtr = this->base; - this->readPtr = this->base; + this->readPtr = this->base; return true; } diff --git a/Include/Aurora/Memory/ByteBuffer_TypedReadWrite.inl b/Include/Aurora/Memory/ByteBuffer_TypedReadWrite.inl index 168fb816..1efd3b89 100644 --- a/Include/Aurora/Memory/ByteBuffer_TypedReadWrite.inl +++ b/Include/Aurora/Memory/ByteBuffer_TypedReadWrite.inl @@ -9,6 +9,8 @@ namespace Aurora::Memory { + static const auto kMaxSaneElementsForAuMemory = 0xFFFFF; + template bool ByteBuffer::Read(T &out) { @@ -22,10 +24,16 @@ namespace Aurora::Memory return false; } - auto len = Read(); - out.resize(len); + auto uLength = Read(); + if (uLength > kMaxSaneElementsForAuMemory) + { + this->flagReadError = true; + return false; + } - for (auto i = 0u; i < len; i++) + out.resize(uLength); + + for (auto i = 0u; i < uLength; i++) { Read(out[i]); } @@ -34,7 +42,13 @@ namespace Aurora::Memory } else if constexpr (AuIsSame_v, AuString>) { - out.resize(Read()); + auto uLength = Read(); + if (uLength > kMaxSaneElementsForAuMemory) + { + this->flagReadError = true; + return false; + } + out.resize(uLength); Read(out.data(), out.size()); return !this->flagReadError; } diff --git a/Include/Aurora/Memory/ByteBuffer_Utils.inl b/Include/Aurora/Memory/ByteBuffer_Utils.inl index 51c6d82d..8c5f60d2 100644 --- a/Include/Aurora/Memory/ByteBuffer_Utils.inl +++ b/Include/Aurora/Memory/ByteBuffer_Utils.inl @@ -41,7 +41,7 @@ namespace Aurora::Memory AuUInt8 &ByteBuffer::operator [](AuUInt idx) const { - auto pBegin = this->begin(); + auto pBegin = (AuUInt8 *)this->begin(); // intentionally returning a volatile reference auto pEnd = this->end(); SysAssert(idx < (AuUInt)(pEnd - pBegin)); return *(pBegin + idx); @@ -62,7 +62,7 @@ namespace Aurora::Memory return IsEmpty(); } - AuUInt8 *ByteBuffer::begin() const + const AuUInt8 *ByteBuffer::begin() const { if (this->flagCircular) { @@ -75,7 +75,7 @@ namespace Aurora::Memory return this->readPtr; } - AuUInt8 *ByteBuffer::end() const + const AuUInt8 *ByteBuffer::end() const { AuUInt8 *pBase {}; AuUInt uCount {}; diff --git a/Include/Aurora/Runtime.hpp b/Include/Aurora/Runtime.hpp index c0fdba4d..e8c0e4a2 100644 --- a/Include/Aurora/Runtime.hpp +++ b/Include/Aurora/Runtime.hpp @@ -270,7 +270,9 @@ namespace Aurora { /// You can bypass branding by assigning an empty string to 'defaultBrand' AuString defaultBrand = "Aurora SDK Sample"; - }; + bool bForceOverlappedUtilsToDelegatedThreadPool { false }; + AuUInt32 uOverlappedUtilsThreadPoolSize { 2 }; // note: this does not relate to the overlapped aio apis + }; // these threads are only spawned as a fallback for AuFS::Overlapped*** apis struct DebugConfig { diff --git a/Source/Compression/Compressors/LZMACompressor.hpp b/Source/Compression/Compressors/LZMACompressor.hpp new file mode 100644 index 00000000..e69de29b diff --git a/Source/Compression/Compressors/LZMADecompressor.hpp b/Source/Compression/Compressors/LZMADecompressor.hpp new file mode 100644 index 00000000..e69de29b diff --git a/Source/IO/Character/BufferedLineReader.cpp b/Source/IO/Character/BufferedLineReader.cpp index a39abd71..d44ab7c2 100644 --- a/Source/IO/Character/BufferedLineReader.cpp +++ b/Source/IO/Character/BufferedLineReader.cpp @@ -26,7 +26,7 @@ namespace Aurora::IO::Character } AuUInt bytesRead; - if (this->inputStream_->Read(AuMemoryViewStreamWrite(AuMemoryViewWrite(this->buffer_.writePtr, this->buffer_.end()), bytesRead)) != + if (this->inputStream_->Read(AuMemoryViewStreamWrite(AuMemoryViewWrite(this->buffer_.writePtr, this->buffer_.writePtr + this->buffer_.length), bytesRead)) != EStreamError::eErrorNone) { SysPushErrorIO(); diff --git a/Source/IO/FS/DirDeleter.cpp b/Source/IO/FS/DirDeleter.cpp index 157e1fee..a047f315 100644 --- a/Source/IO/FS/DirDeleter.cpp +++ b/Source/IO/FS/DirDeleter.cpp @@ -17,18 +17,19 @@ namespace Aurora::IO::FS AuList nextLevel2; AuString curPath; AuString curSubDir; + AuList failedPaths; bool OpenDir(const AuString &str) { - curPath = str; - pDir = ReadDir(str); + this->curPath = str; + this->pDir = ReadDir(str); return bool(pDir); } bool OpenNext(const AuString &str) { - curPath = str; - pDir = ReadDir(str); + this->curPath = str; + this->pDir = ReadDir(str); return bool(pDir); } @@ -36,15 +37,15 @@ namespace Aurora::IO::FS { this->pDir.reset(); - if (!nextLevel.size()) + if (!this->nextLevel.size()) { return; } - auto a = nextLevel[0]; - nextLevel.erase(nextLevel.begin()); - this->pDir = ReadDir(curPath + "/" + a); - curSubDir = a; + auto a = this->nextLevel[0]; + this->nextLevel.erase(this->nextLevel.begin()); + this->pDir = ReadDir(this->curPath + "/" + a); + this->curSubDir = a; } virtual StatEx *Next() override @@ -79,7 +80,10 @@ namespace Aurora::IO::FS } else { - AuFS::Remove(pNext->path); + if (!AuFS::Remove(pNext->path)) + { + this->failedPaths.push_back(pNext->fileName); + } } return pNext; @@ -87,9 +91,13 @@ namespace Aurora::IO::FS void RemoveDirs() { - for (auto itr = nextLevel2.rbegin(); itr != nextLevel2.rend(); itr++) + for (auto itr = this->nextLevel2.rbegin(); itr != this->nextLevel2.rend(); itr++) { - AuFS::Remove(curPath + "/" + itr->c_str()); + auto dir = this->curPath + "/" + itr->c_str(); + if (!AuFS::Remove(dir)) + { + this->failedPaths.push_back(itr->c_str()); + } } } }; @@ -118,4 +126,44 @@ namespace Aurora::IO::FS AuFS::Remove(string); return !AuFS::DirExists(string); } + + AUKN_SYM bool DirDeleterEx(const AuString &string, AuList &failingPaths) + { + auto pObj = AuMakeShared(); + if (!pObj) + { + SysPushErrorMem(); + return {}; + } + + if (!pObj->OpenDir(string)) + { + return {}; + } + + while (pObj->Next()) + { + + } + + pObj->RemoveDirs(); + + AuFS::Remove(string); + + if (AuFS::DirExists(string)) + { + for (const auto &str : pObj->failedPaths) + { + if (AuFS::FileExists(str) || + AuFS::DirExists(str)) + { + failingPaths.push_back(str); + } + } + + return false; + } + + return true; + } } \ No newline at end of file diff --git a/Source/IO/FS/FS.NT.cpp b/Source/IO/FS/FS.NT.cpp index 6105046d..3a9183f3 100644 --- a/Source/IO/FS/FS.NT.cpp +++ b/Source/IO/FS/FS.NT.cpp @@ -192,7 +192,18 @@ namespace Aurora::IO::FS CreateDirectories(pathNormalized, true); - fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NULL, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE | GENERIC_READ, NULL, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (fileHandle == INVALID_HANDLE_VALUE) + { + fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NULL, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + } + + if (fileHandle == INVALID_HANDLE_VALUE) + { + AuThreading::ContextYield(); + fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE | GENERIC_READ, NULL, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + } + if (fileHandle == INVALID_HANDLE_VALUE) { SysPushErrorIO("Couldn't open handle: {}", path); diff --git a/Source/IO/FS/FS.Unix.cpp b/Source/IO/FS/FS.Unix.cpp index 1b88be40..12bdf9bb 100755 --- a/Source/IO/FS/FS.Unix.cpp +++ b/Source/IO/FS/FS.Unix.cpp @@ -250,7 +250,7 @@ namespace Aurora::IO::FS // NOTE: Linux filesystems are such a cluster fuck of unimplemented interfaces and half-assed drivers // It's not unusual for these "files" to not support the required seek operations across NIX-like oses. - if (len == 0) + if (qwLength == 0) { if (bIsStupidFD) { @@ -291,7 +291,7 @@ namespace Aurora::IO::FS static bool UnixExists(const AuString &path, bool dir) { struct stat s; - int err = stat(path.c_str(), &s); + int err = ::stat(path.c_str(), &s); if (-1 == err) { SysAssert(ENOENT == errno, "General File IO Error, path {}", path); diff --git a/Source/IO/FS/FSCompress.cpp b/Source/IO/FS/FSCompress.cpp new file mode 100644 index 00000000..7bdf6d2e --- /dev/null +++ b/Source/IO/FS/FSCompress.cpp @@ -0,0 +1,224 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: FSCompress.cpp + Date: 2023-1-26 + Author: Reece +***/ +#include +#include "FS.hpp" +#include "FS.Generic.hpp" +#include + +namespace Aurora::IO::FS +{ + static const AuUInt64 kFileCopyBlock = 0xFFFF * 4; // 64KiB + static auto const kCompressionType = AuCompression::ECompressionType::eZSTD; + static const AuString kStringSuffix = ".zst"; + + AUKN_SYM bool Compress(const AuString &path, AuInt8 level) + { + static const auto kCompressionReadChunks = kFileCopyBlock; + + auto pFileSrc = OpenReadShared(path, EFileAdvisoryLockLevel::eBlockWrite); + if (!pFileSrc) + { + SysPushErrorIO("Couldn't open compression source path: {}", path); + return {}; + } + + auto pFileDest = OpenWriteShared(path + kStringSuffix, EFileAdvisoryLockLevel::eBlockReadWrite); + if (!pFileDest) + { + SysPushErrorIO("Couldn't open compression destination path: {}.{}", path, kStringSuffix); + return {}; + } + + if (pFileDest->GetLength()) + { + SysPushErrorIO("File exists"); + return {}; + } + + auto qwLength = pFileSrc->GetLength(); + + auto pFileStream = AuMakeShared(pFileSrc); + if (!pFileStream) + { + SysPushErrorMemory(); + return {}; + } + + auto pDestStream = AuMakeShared(pFileDest); + if (!pDestStream) + { + SysPushErrorMemory(); + return {}; + } + + AuCompression::CompressInfo compress { kCompressionType }; + compress.uCompressionLevel = level; + compress.uInternalStreamSize = kCompressionReadChunks * 2; + auto pCompressor = AuCompression::CompressorUnique(pFileStream, compress); + if (!pCompressor) + { + SysPushErrorMemory("no compressor"); + return {}; + } + + AuUInt64 qwTotalRead {}; + { + AuByteBuffer tempMemory(kCompressionReadChunks); + if (!tempMemory) + { + SysPushErrorMemory(); + return {}; + } + + for (AuUInt64 i = 0; i < qwLength; i += 0 /*kCompressionReadChunks*/) + { + auto [read, written] = pCompressor->Ingest(kCompressionReadChunks); + i += read; + + if (read == 0) + { + break; + } + + qwTotalRead += read; + + bool bAnyWritten {}; + while (auto uBytes = pCompressor->Read(tempMemory)) + { + tempMemory.writePtr += uBytes; + bAnyWritten = true; + + AuUInt idc {}; + if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) + { + SysPushErrorIO("AuIO::WriteAll failed"); + return {}; + } + + tempMemory.writePtr = tempMemory.base; + tempMemory.readPtr = tempMemory.base; + } + + // zstd u ok? + // if i remove this, we get no data at all. + // even if i try to displace this to above `->Finish()` we still get nothing + // preemptively flushing fixes everything /shrug + pCompressor->Flush(); + } + + pCompressor->Finish(); + + while (auto uBytes = pCompressor->Read(tempMemory)) + { + tempMemory.writePtr += uBytes; + + AuUInt idc {}; + if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) + { + SysPushErrorIO("AuIO::WriteAll failed"); + return {}; + } + + tempMemory.writePtr = tempMemory.base; + tempMemory.readPtr = tempMemory.base; + } + } + + return qwTotalRead == qwLength; + } + + AUKN_SYM bool Decompress(const AuString &path) + { + static const auto kCompressionReadChunks = kFileCopyBlock; + + auto pFileSrc = OpenReadShared(path + kStringSuffix, EFileAdvisoryLockLevel::eBlockWrite); + if (!pFileSrc) + { + SysPushErrorIO("Couldn't open compression source path: {}.{}", path, kStringSuffix); + return {}; + } + + auto pFileDest = OpenWriteShared(path, EFileAdvisoryLockLevel::eBlockReadWrite); + if (!pFileDest) + { + SysPushErrorIO("Couldn't open decompression destination path: {}", path); + return {}; + } + + if (pFileDest->GetLength()) + { + SysPushErrorIO("File exists"); + return {}; + } + + auto qwLength = pFileSrc->GetLength(); + + auto pFileStream = AuMakeShared(pFileSrc); + if (!pFileStream) + { + SysPushErrorMemory(); + return {}; + } + + auto pDestStream = AuMakeShared(pFileDest); + if (!pDestStream) + { + SysPushErrorMemory(); + return {}; + } + + AuCompression::DecompressInfo decompress{ kCompressionType }; + decompress.uInternalStreamSize = kCompressionReadChunks * 5; + auto pDecompressor = AuCompression::DecompressorUnique(pFileStream, decompress); + if (!pDecompressor) + { + SysPushErrorMemory("no decompressor"); + return {}; + } + + AuUInt64 qwTotalRead {}; + { + AuByteBuffer tempMemory(kCompressionReadChunks); + if (!tempMemory) + { + SysPushErrorMemory(); + return {}; + } + + for (AuUInt64 i = 0; i < qwLength; i += 0 /*kCompressionReadChunks*/) + { + auto [read, written] = pDecompressor->Ingest(kCompressionReadChunks); + i += read; + + if (read == 0) + { + break; + } + + qwTotalRead += read; + + while (auto uBytes = pDecompressor->Read(tempMemory)) + { + tempMemory.writePtr += uBytes; + + AuUInt idc {}; + if (AuIO::WriteAll(pDestStream.get(), { tempMemory, idc }) != AuIO::EStreamError::eErrorNone) + { + SysPushErrorIO("AuIO::WriteAll failed"); + return {}; + } + + tempMemory.writePtr = tempMemory.base; + tempMemory.readPtr = tempMemory.base; + } + } + } + + return qwTotalRead == qwLength; + } +} \ No newline at end of file diff --git a/Source/IO/FS/FSCompress.hpp b/Source/IO/FS/FSCompress.hpp new file mode 100644 index 00000000..18478a71 --- /dev/null +++ b/Source/IO/FS/FSCompress.hpp @@ -0,0 +1,13 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: FSCompress.hpp + Date: 2023-1-26 + Author: Reece +***/ +#pragma once + +namespace Aurora::IO::FS +{ + +} \ No newline at end of file diff --git a/Source/IO/FS/FSDefaultOverlappedWorkerThread.cpp b/Source/IO/FS/FSDefaultOverlappedWorkerThread.cpp new file mode 100644 index 00000000..e69de29b diff --git a/Source/IO/FS/FSDefaultOverlappedWorkerThread.hpp b/Source/IO/FS/FSDefaultOverlappedWorkerThread.hpp new file mode 100644 index 00000000..e69de29b diff --git a/Source/IO/FS/FSOverlappedUtilities.cpp b/Source/IO/FS/FSOverlappedUtilities.cpp new file mode 100644 index 00000000..542e0d49 --- /dev/null +++ b/Source/IO/FS/FSOverlappedUtilities.cpp @@ -0,0 +1,93 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: FSOverlappedUtilities.cpp + Date: 2023-1-26 + Author: Reece +***/ +#include +#include "FS.hpp" +#include "FSOverlappedUtilities.hpp" + +namespace Aurora::IO::FS +{ + struct OverlappedStatOperation : + virtual BaseOverlappedOperation, + IOverlappedStatOperation + { + Stat stat; + + virtual AuSPtr GetStats() override; + }; + + struct OverlappedReadOperation : + virtual BaseOverlappedOperation, + IOverlappedReadOperation + { + AuMemoryViewWrite write; + AuByteBuffer buffer; + + virtual AuSPtr GetByteBuffer() override; + virtual AuSPtr GetReadView() override; + }; + + AUKN_SYM bool OverlappedForceDelegatedIO(bool bForceIOWorkerThreads) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedCompress(const AuString &path, AuInt8 iLevel) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedDecompress(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedWrite(const AuString &path, AuSPtr pMemoryView) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedRead(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedStat(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedCopy(const AuString &path, const AuString &dest) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedRelink(const AuString &path, const AuString &dest) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedTrustFile(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedBlockFile(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedUnblockFile(const AuString &path) + { + return {}; + } + + AUKN_SYM AuSPtr OverlappedDelete(const AuString &path) + { + return {}; + } +} \ No newline at end of file diff --git a/Source/IO/FS/FSOverlappedUtilities.hpp b/Source/IO/FS/FSOverlappedUtilities.hpp new file mode 100644 index 00000000..7f48b10e --- /dev/null +++ b/Source/IO/FS/FSOverlappedUtilities.hpp @@ -0,0 +1,30 @@ +/*** + Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: FSOverlappedUtilities.hpp + Date: 2023-1-26 + Author: Reece +***/ +#pragma once + +namespace Aurora::IO::FS +{ + struct BaseOverlappedOperation : virtual IOverlappedOperationBase + { + BaseOverlappedOperation(); + + virtual AuSPtr ToWaitable() override; + + virtual AuSPtr SetCallback(AuSPtr pCallback) override; + + virtual bool IsOperationComplete() override; + virtual bool IsOperationSuccessful() override; + + void Complete(); + void Fail(); + + private: + AuSPtr pWaitable; + AuSPtr pCallback; + }; +} \ No newline at end of file diff --git a/Source/IO/FS/FileStream.NT.cpp b/Source/IO/FS/FileStream.NT.cpp index fbd198a7..aa39c9c3 100644 --- a/Source/IO/FS/FileStream.NT.cpp +++ b/Source/IO/FS/FileStream.NT.cpp @@ -250,7 +250,7 @@ namespace Aurora::IO::FS fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE | GENERIC_READ, NtLockAdvisoryToShare(lock), NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (fileHandle == INVALID_HANDLE_VALUE) { - fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NtLockAdvisoryToShare(lock), NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NtLockAdvisoryToShare(lock), NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, NULL); } break; } @@ -260,7 +260,7 @@ namespace Aurora::IO::FS fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE | FILE_READ_ATTRIBUTES, NtLockAdvisoryToShare(lock), NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (fileHandle == INVALID_HANDLE_VALUE) { - fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NtLockAdvisoryToShare(lock), NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + fileHandle = CreateFileW(win32Path.c_str(), GENERIC_WRITE, NtLockAdvisoryToShare(lock), NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, NULL); } break; } diff --git a/Source/IO/Net/AuNetHostname.cpp b/Source/IO/Net/AuNetHostname.cpp index ed012346..0365a8b6 100644 --- a/Source/IO/Net/AuNetHostname.cpp +++ b/Source/IO/Net/AuNetHostname.cpp @@ -10,6 +10,19 @@ namespace Aurora::IO::Net { + NetHostname::NetHostname() + { + + } + + NetHostname::NetHostname(const NetHostname &cpy) : + type(cpy.type), + hostname(cpy.hostname), + address(cpy.address) + { + + } + NetHostname::NetHostname(const AuString &hostname) : type(EHostnameType::eHostByDns), hostname(hostname), @@ -32,6 +45,14 @@ namespace Aurora::IO::Net this->address == other.address; } + NetHostname &NetHostname::operator =(const NetHostname &other) + { + this->type = other.type; + this->hostname = other.hostname; + this->address = other.address; + return *this; + } + const AuUInt NetHostname::HashCode() const { return AuHashCode(this->type) ^ diff --git a/Source/IO/Net/AuNetResolver.NT.cpp b/Source/IO/Net/AuNetResolver.NT.cpp index fd162bc6..07cd0fb3 100644 --- a/Source/IO/Net/AuNetResolver.NT.cpp +++ b/Source/IO/Net/AuNetResolver.NT.cpp @@ -34,7 +34,12 @@ namespace Aurora::IO::Net ADDRINFOEXW infoEx { 0 }; int iInfoEx { 0 }; - if (this->bA && !this->bAAAA) + if (this->bA && this->bAAAA) + { + infoEx.ai_family = AF_UNSPEC; + //infoEx.ai_flags = AI_ALL; + } + else if (this->bA && !this->bAAAA) { infoEx.ai_family = AF_INET; } @@ -43,11 +48,6 @@ namespace Aurora::IO::Net infoEx.ai_family = AF_INET6; //infoEx.ai_flags = AI_V4MAPPED; // beats returning nothing... } - else if (this->bA && this->bAAAA) - { - infoEx.ai_family = AF_UNSPEC; - //infoEx.ai_flags = AI_ALL; - } else { return false; diff --git a/Source/IO/Net/AuNetResolver.Unix.cpp b/Source/IO/Net/AuNetResolver.Unix.cpp index 480d828e..53b1feb3 100644 --- a/Source/IO/Net/AuNetResolver.Unix.cpp +++ b/Source/IO/Net/AuNetResolver.Unix.cpp @@ -113,7 +113,11 @@ namespace Aurora::IO::Net { int iInfoEx { 0 }; - if (this->bA && !this->bAAAA) + if (this->bA && this->bAAAA) + { + infoEx.ai_family = AF_UNSPEC; + } + else if (this->bA && !this->bAAAA) { infoEx.ai_family = AF_INET; } @@ -121,10 +125,6 @@ namespace Aurora::IO::Net { infoEx.ai_family = AF_INET6; } - else if (this->bA && this->bAAAA) - { - infoEx.ai_family = AF_UNSPEC; - } else { return false; diff --git a/Source/IO/Net/AuNetSocket.NT.cpp b/Source/IO/Net/AuNetSocket.NT.cpp index 232632af..6d0df8f3 100644 --- a/Source/IO/Net/AuNetSocket.NT.cpp +++ b/Source/IO/Net/AuNetSocket.NT.cpp @@ -29,13 +29,24 @@ namespace Aurora::IO::Net { } - + + Socket::Socket(struct NetInterface *pInterface, + struct NetWorker *pWorker, + const AuSPtr &pSocketDriver, + const NetEndpoint &endpoint) : + SocketBase(pInterface, pWorker, pSocketDriver, endpoint) + { + + } + Socket::Socket(struct NetInterface *pInterface, struct NetWorker *pWorker, const AuSPtr &pSocketDriver, - const NetEndpoint &endpoint) : - SocketBase(pInterface, pWorker, pSocketDriver, endpoint) + const AuPair &endpoint, + AuNet::ETransportProtocol eProtocol) : + SocketBase(pInterface, pWorker, pSocketDriver, endpoint, eProtocol) { + } Socket::Socket(NetInterface *pInterface, @@ -44,7 +55,7 @@ namespace Aurora::IO::Net const NetSocketConnectMany &connectMany) : SocketBase(pInterface, pWorker, pSocketDriver, connectMany) { - + } Socket::~Socket() @@ -62,7 +73,7 @@ namespace Aurora::IO::Net } } - void Socket::FinishConstructAsync() + void Socket::RenewSocket() { if (!this->SendPreestablish()) { @@ -70,6 +81,18 @@ namespace Aurora::IO::Net return; } + if (this->bHasRemoteMany_ && this->connectMany_.ips.size()) + { + this->remoteEndpoint_ = this->connectMany_.ips[0]; + } + + if (this->osHandle_ && + this->osHandle_ != -1) + { + ::closesocket(this->osHandle_); + this->osHandle_ = 0; + } + this->osHandle_ = ::WSASocketW( IPToDomain(this->remoteEndpoint_), TransportToPlatformType(this->remoteEndpoint_), @@ -79,7 +102,6 @@ namespace Aurora::IO::Net WSA_FLAG_OVERLAPPED ); - if (this->osHandle_ == -1) { this->SendErrorNoStream(GetLastNetError()); @@ -93,10 +115,34 @@ namespace Aurora::IO::Net } #if !defined(AURORA_IS_MODERNNT_DERIVED) + + this->osHandleOwner_ = AuMakeShared(); + if (!this->osHandle_) + { + this->SendErrorNoStream(GetLastNetError()); + return; + } + this->osHandleOwner_->Init((int)this->osHandle_, (int)this->osHandle_); #endif } + void Socket::FinishConstructAsync() + { + if (this->resolveLater.size() || this->bResolving_) + { + if (!this->TryStartResolve()) + { + this->SendErrorNoStream(GetLastNetError()); + return; + } + + return; + } + + RenewSocket(); + } + bool Socket::PrepareConnectOperations() { ::setsockopt(this->osHandle_, diff --git a/Source/IO/Net/AuNetSocket.NT.hpp b/Source/IO/Net/AuNetSocket.NT.hpp index 3d1df7a1..7573479b 100644 --- a/Source/IO/Net/AuNetSocket.NT.hpp +++ b/Source/IO/Net/AuNetSocket.NT.hpp @@ -29,6 +29,12 @@ namespace Aurora::IO::Net const AuSPtr &pSocketDriver, AuUInt osHandle); + Socket(struct NetInterface *pInterface, + struct NetWorker *pWorker, + const AuSPtr &pSocketDriver, + const AuPair &endpoint, + AuNet::ETransportProtocol eProtocol); + Socket(struct NetInterface *pInterface, struct NetWorker *pWorker, const AuSPtr &pSocketDriver, @@ -61,5 +67,6 @@ namespace Aurora::IO::Net virtual void Shutdown(bool bNow) override; virtual void CloseSocket() override; + virtual void RenewSocket() override; }; } \ No newline at end of file diff --git a/Source/IO/Net/AuNetSocket.cpp b/Source/IO/Net/AuNetSocket.cpp index ceecdf11..932b3068 100644 --- a/Source/IO/Net/AuNetSocket.cpp +++ b/Source/IO/Net/AuNetSocket.cpp @@ -12,6 +12,7 @@ #include "AuIPAddress.hpp" #include "AuNetError.hpp" #include "AuNetSocketServer.hpp" +#include "AuNetInterface.hpp" #if defined(AURORA_IS_MODERNNT_DERIVED) #include "AuNetStream.NT.hpp" @@ -54,9 +55,9 @@ namespace Aurora::IO::Net } SocketBase::SocketBase(struct NetInterface *pInterface, - struct NetWorker *pWorker, - const AuSPtr &pSocketDriver, - const NetEndpoint &endpoint) : + struct NetWorker *pWorker, + const AuSPtr &pSocketDriver, + const NetEndpoint &endpoint) : connectOperation(this), socketChannel_(this), pInterface_(pInterface), @@ -72,6 +73,45 @@ namespace Aurora::IO::Net this->pWorker_->AddSocket(this); } + SocketBase::SocketBase(struct NetInterface *pInterface, + struct NetWorker *pWorker, + const AuSPtr &pSocketDriver, + const AuPair &endpoint, + AuNet::ETransportProtocol eProtocol) : + connectOperation(this), + socketChannel_(this), + pInterface_(pInterface), + pWorker_(pWorker), + pSocketDriver_(pSocketDriver) + { + auto &[host, uPort] = endpoint; + + if (host.type == AuNet::EHostnameType::eHostByIp) + { + this->remoteEndpoint_.ip = host.address; + this->remoteEndpoint_.uPort = uPort; + this->remoteEndpoint_.transportProtocol = eProtocol; + OptimizeEndpoint(this->remoteEndpoint_); + } + else + { + this->resolveLater = host.hostname; + this->remoteEndpoint_.uPort = uPort; + this->remoteEndpoint_.transportProtocol = eProtocol; + this->connectMany_.uPort = uPort; + this->connectMany_.protocol = eProtocol; + } + + this->osHandleOwner_ = AuMakeShared(); + if (!this->osHandle_) + { + return; + } + + this->pWorker_->AddSocket(this); + } + + SocketBase::SocketBase(NetInterface *pInterface, NetWorker *pWorker, const AuSPtr &pSocketDriver, @@ -100,13 +140,48 @@ namespace Aurora::IO::Net bool SocketBase::IsValid() { - return bool(this->osHandleOwner_) && + return (this->resolveLater.size()) || + bool(this->osHandleOwner_) && bool(this->connectOperation.IsValid()) && bool(this->osHandle_ != 0) && bool(this->osHandle_ != -1) && bool(!this->bForceFailConstruct_); } + bool SocketBase::TryStartResolve() + { + auto pThat = this->SharedFromThis(); + + if (this->bResolving_) + { + return true; + } + + this->bResolving_ = true; + auto address = this->resolveLater; + this->resolveLater.clear(); + + auto pResolver = this->pInterface_->GetResolveService()->SimpleAllResolve(address, + AuMakeSharedThrow, + AuNet::NetError>>( + [=](const AuSPtr> &ips) + { + + pThat->bResolving_ = false; + pThat->connectMany_.uPort = pThat->remoteEndpoint_.uPort; + pThat->connectMany_.ips.insert(pThat->connectMany_.ips.end(), ips->begin(), ips->end()); + pThat->bHasRemoteMany_ = true; + pThat->RenewSocket(); + pThat->ConnectNext(); + }, + [=](const AuSPtr &error) + { + pThat->SendErrorNoStream(error ? *error.get() : AuNet::NetError {}); + })); + + return bool(pResolver); + } + bool SocketBase::ConnectNext() { if (this->connectMany_.ips.empty()) @@ -126,6 +201,12 @@ namespace Aurora::IO::Net bool SocketBase::Connect(const NetEndpoint &endpoint) { + if (!this->IsValid()) + { + this->SendErrorNoStream({}); + return false; + } + this->remoteEndpoint_ = endpoint; this->endpointSize_ = OptimizeEndpoint(this->remoteEndpoint_); this->localEndpoint_.transportProtocol = this->remoteEndpoint_.transportProtocol; @@ -272,7 +353,7 @@ namespace Aurora::IO::Net void SocketBase::SendOnData() { auto pReadableBuffer = this->socketChannel_.AsReadableByteBuffer(); - auto pStartOffset = pReadableBuffer->readPtr; + auto pStartOffset = pReadableBuffer ? pReadableBuffer->readPtr : nullptr; if (this->bHasFinalized_) { @@ -305,7 +386,7 @@ namespace Aurora::IO::Net this->ToChannel()->ScheduleOutOfFrameWrite(); - auto uHeadDelta = pReadableBuffer->readPtr - pStartOffset; + auto uHeadDelta = pReadableBuffer ? (pReadableBuffer->readPtr - pStartOffset) : 0; this->socketChannel_.GetRecvStatsEx().AddBytes(uHeadDelta); } @@ -344,6 +425,11 @@ namespace Aurora::IO::Net bool SocketBase::SendPreestablish(SocketServer *pServer) { + if (this->bHasPreestablished_) + { + return true; + } + if (pServer && pServer->uDefaultInputStreamSize) { this->socketChannel_.uBytesInputBuffer = pServer->uDefaultInputStreamSize; @@ -365,7 +451,7 @@ namespace Aurora::IO::Net } } - return true; + return this->bHasPreestablished_ = true; } void SocketBase::SendEnd() diff --git a/Source/IO/Net/AuNetSocket.hpp b/Source/IO/Net/AuNetSocket.hpp index 2f810650..77ac2e77 100644 --- a/Source/IO/Net/AuNetSocket.hpp +++ b/Source/IO/Net/AuNetSocket.hpp @@ -37,6 +37,12 @@ namespace Aurora::IO::Net const AuSPtr &pSocketDriver, const NetEndpoint &endpoint); + SocketBase(struct NetInterface *pInterface, + struct NetWorker *pWorker, + const AuSPtr &pSocketDriver, + const AuPair &endpoint, + AuNet::ETransportProtocol eProtocol); + SocketBase(struct NetInterface *pInterface, struct NetWorker *pWorker, const AuSPtr &pSocketDriver, @@ -44,6 +50,7 @@ namespace Aurora::IO::Net virtual ~SocketBase(); + bool TryStartResolve(); bool ConnectNext(); bool Connect(const NetEndpoint &endpoint); @@ -100,16 +107,18 @@ namespace Aurora::IO::Net virtual bool ConnectOverlapped() = 0; virtual bool ConnectNonblocking() = 0; virtual bool ConnectBlocking() = 0; - + virtual void CloseSocket() = 0; + virtual void RenewSocket() = 0; AuUInt endpointSize_ {}; bool bHasFinalized_ {}; bool bHasEnded {}; + bool bResolving_ {}; protected: - AuUInt osHandle_; + AuUInt osHandle_ {}; NetEndpoint remoteEndpoint_; NetEndpoint localEndpoint_; @@ -128,6 +137,8 @@ namespace Aurora::IO::Net bool bHasErrored_ {}; bool bHasConnected_ {}; + bool bHasPreestablished_ {}; + AuString resolveLater {}; }; } diff --git a/Source/IO/Net/AuNetSrvSockets.cpp b/Source/IO/Net/AuNetSrvSockets.cpp index fed44b6b..a6e3d566 100644 --- a/Source/IO/Net/AuNetSrvSockets.cpp +++ b/Source/IO/Net/AuNetSrvSockets.cpp @@ -25,12 +25,6 @@ namespace Aurora::IO::Net AuSPtr NetSrvSockets::Connect(const NetSocketConnect &netConnect) { - if (netConnect.endpoint.transportProtocol != ETransportProtocol::eProtocolTCP) - { - SysPushErrorNet("Invalid transport protocol. Hint: Use ConnectManyEx for UDP."); - return {}; - } - auto pWorker = this->pParent_->TryScheduleEx(); if (!pWorker) { @@ -38,10 +32,48 @@ namespace Aurora::IO::Net return {}; } - auto pSocket = AuMakeShared(this->pParent_, - pWorker.get(), - netConnect.pDriver, - netConnect.endpoint); + AuSPtr pSocket; + + if (netConnect.endpoint) + { + if (netConnect.endpoint.Value().transportProtocol != ETransportProtocol::eProtocolTCP) + { + SysPushErrorNet("Invalid transport protocol. Hint: Use ConnectManyEx for UDP."); + return {}; + } + + pSocket = AuMakeShared(this->pParent_, + pWorker.get(), + netConnect.pDriver, + netConnect.endpoint.value()); + } + else + { + if (!netConnect.byHost.netHostname) + { + SysPushErrorArg("Missing hostname or endpoint"); + return {}; + } + + if (!netConnect.byHost.protocol) + { + SysPushErrorArg("Missing protocol"); + return {}; + } + + if (!netConnect.byHost.uPort) + { + SysPushErrorArg("Missing port"); + return {}; + } + + pSocket = AuMakeShared(this->pParent_, + pWorker.get(), + netConnect.pDriver, + AuMakePair(netConnect.byHost.netHostname.value(), netConnect.byHost.uPort.value()), + netConnect.byHost.protocol.value()); + } + if (!pSocket) { SysPushErrorNet("No Memory"); @@ -52,6 +84,11 @@ namespace Aurora::IO::Net { pSocket->FinishConstructAsync(); + if (pSocket->bResolving_) + { + return; + } + if (!pSocket->IsValid()) { pSocket->SendErrorNoStream({}); diff --git a/Source/Logging/Sinks/DirLogArchive.cpp b/Source/Logging/Sinks/DirLogArchive.cpp index 60fa37d5..1323b9ce 100644 --- a/Source/Logging/Sinks/DirLogArchive.cpp +++ b/Source/Logging/Sinks/DirLogArchive.cpp @@ -11,54 +11,153 @@ namespace Aurora::Logging::Sinks { - static void EraseFilesByTimestamp(AuUInt32 maxLogs, - const AuString &path, /*const its not worth reallocation*/ AuList &files) - { - if (files.size() <= maxLogs) - { - return; - } - - // our filenames are usually prefixed by an ISO 8601 timestamp where the most significant bits are last (YYYY-MM-DD?HH-MM-SS) - // a quick ghetto sort should be all we need. no need to waste time parsing timestamps - std::sort(files.begin(), files.end()); - - auto amount = files.size() - maxLogs; - for (auto x = 0, i = 0; ((x < amount) && (i > files.size())); i++) - { - try - { - if (AuIOFS::Remove(path + "/" + files[i])) - { - x++; - } - } - catch (...) - { - - } - } - } - static void CleanupOldLogs(DirectoryLogger logger, const AuString &baseLogPath) { AuList files; - AuBST fileMeta; - AuUInt32 size {}; + AuList> fileMeta; + AuUInt64 qwSize {}; - AuIOFS::FilesInDirectory(baseLogPath, files); - - for (const auto &file : files) + auto doScan = [&]() { - AuIOFS::Stat stat; - AuIOFS::StatFile(baseLogPath + "/" + file, stat); - fileMeta[file] = stat; - size += stat.uSize; + files.clear(); + fileMeta.clear(); + qwSize = 0; + + AuIOFS::FilesInDirectory(baseLogPath, files); + + for (const auto &file : files) + { + AuIOFS::Stat stat; + AuIOFS::StatFile(baseLogPath + "/" + file, stat); + fileMeta.push_back(AuMakePair(file, stat)); + qwSize += stat.uSize; + } + + std::sort(fileMeta.begin(), fileMeta.end(), [](AuPair a, AuPair b) + { + return AuGet<1>(a).modified < AuGet<1>(b).modified; + }); + }; + + doScan(); + + if (logger.uMaxFileTimeInDeltaMSOrZeroBeforeCompress) + { + auto qwMaxTime = AuTime::CurrentClockMS() - logger.uMaxFileTimeInDeltaMSOrZeroBeforeCompress; + bool bRescan {}; + + for (AuUInt i = 0; i < fileMeta.size(); i++) + { + if (qwMaxTime > fileMeta[i].second.modified) + { + auto path = baseLogPath + "/" + fileMeta[i].first; + if (AuIOFS::Compress(path)) + { + AuIOFS::Remove(path); + } + + bRescan = true; + } + } + + if (bRescan) + { + doScan(); + } } - EraseFilesByTimestamp(logger.maxLogsOrZero, baseLogPath, files); - // TODO: erase when size >= maxFileSizeOrZero * 1024 - // Didn't auRuntime v1.0 have this? + if (logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeCompress) + { + bool bRescan {}; + + if (qwSize > logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeCompress) + { + std::sort(fileMeta.begin(), fileMeta.end(), [](AuPair a, AuPair b) + { + return AuGet<1>(a).uSize > AuGet<1>(b).uSize; + }); + + for (AuUInt i = 0; i < fileMeta.size(); i++) + { + if (qwSize <= logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeCompress) + { + break; + } + + auto path = baseLogPath + "/" + fileMeta[i].first; + if (AuIOFS::Compress(path)) + { + AuIOFS::Remove(path); + } + + bRescan = true; + } + } + + if (bRescan) + { + doScan(); + } + } + + if (logger.uMaxLogsOrZeroBeforeDelete) + { + if (fileMeta.size() > logger.uMaxLogsOrZeroBeforeDelete) + { + for (AuUInt i = logger.uMaxLogsOrZeroBeforeDelete; i < fileMeta.size(); i++) + { + AuIOFS::Remove(baseLogPath + "/" + fileMeta[i].first); + } + + doScan(); + } + } + + if (logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeDelete) + { + bool bRescan {}; + + if (qwSize > logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeDelete) + { + std::sort(fileMeta.begin(), fileMeta.end(), [](AuPair a, AuPair b) + { + return AuGet<1>(a).uSize > AuGet<1>(b).uSize; + }); + + for (AuUInt i = 0; i < fileMeta.size(); i++) + { + if (qwSize <= logger.uMaxCumulativeFileSizeInMiBOrZeroBeforeDelete) + { + break; + } + + if (AuIOFS::Remove(baseLogPath + "/" + fileMeta[i].first)) + { + qwSize -= fileMeta[i].second.uSize; + bRescan = true; + } + } + + } + + if (bRescan) + { + doScan(); + } + } + + if (logger.uMaxFileTimeInDeltaMSOrZeroBeforeDelete) + { + auto qwMaxTime = AuTime::CurrentClockMS() - logger.uMaxFileTimeInDeltaMSOrZeroBeforeDelete; + + for (AuUInt i = 0; i < fileMeta.size(); i++) + { + if (qwMaxTime > fileMeta[i].second.modified) + { + AuIOFS::Remove(baseLogPath + "/" + fileMeta[i].first); + } + } + } } IBasicSink *NewDirectoryLoggerNew(const AuString &baseDirectory, @@ -68,10 +167,11 @@ namespace Aurora::Logging::Sinks AuString path; auto tm = Time::ToCivilTime(Time::CurrentClockMS()); - path = fmt::format("{}/{:04}-{:02}-{:02}T{:02}-{:02}-{:02}Z.txt", + path = fmt::format("{}/{:04}-{:02}-{:02}T{:02}-{:02}-{:02}Z.{}", baseDirectory, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec); + tm.tm_hour, tm.tm_min, tm.tm_sec, + bBinary ? ".log" : ".txt"); CleanupOldLogs(meta, baseDirectory);