From f166849e9f35b2e62011b800570cfa342174a417 Mon Sep 17 00:00:00 2001 From: Reece Date: Tue, 14 Jun 2022 02:14:51 +0100 Subject: [PATCH] [TODO: BLOCK 2 PROGRESS] --- Include/Aurora/Exit/README.md | 9 ++ Include/Aurora/IO/IIOPipeProcessor.hpp | 107 +++++++++++--- Include/Aurora/Memory/README.md | 12 ++ Include/Aurora/Processes/EStreamForward.hpp | 17 +++ Include/Aurora/Processes/Processes.hpp | 1 + Include/Aurora/Processes/StartupParmaters.hpp | 15 +- Include/Aurora/RNG/README.md | 12 ++ Source/IO/IOProcessor.cpp | 47 +++++- Source/IO/IOProcessor.hpp | 4 +- Source/IO/IOProcessorItems.cpp | 4 + Source/IO/IOProcessorItems.hpp | 2 + Source/Processes/Process.NT.cpp | 136 ++++++++++++++---- Source/Processes/Process.NT.hpp | 8 ++ Source/Processes/Process.Unix.cpp | 6 +- 14 files changed, 316 insertions(+), 64 deletions(-) create mode 100644 Include/Aurora/Exit/README.md create mode 100644 Include/Aurora/Memory/README.md create mode 100644 Include/Aurora/Processes/EStreamForward.hpp create mode 100644 Include/Aurora/RNG/README.md diff --git a/Include/Aurora/Exit/README.md b/Include/Aurora/Exit/README.md new file mode 100644 index 00000000..6eef6880 --- /dev/null +++ b/Include/Aurora/Exit/README.md @@ -0,0 +1,9 @@ +# AuExit + +Provides termination events, usually on a worker thread, when application termination is likely. Event categories include: control c, an exception was thrown, application shutdown. + +Trigger Levels: +* eSafeTermination (safe unload) +* eFatalException (a fatal exception, math, or memory error occurred) +* eSigTerminate (control + c) +* eProblematicEvent (an exception was thrown or some other fatal condition. should probably flush & sync streams or invalidate caches.) \ No newline at end of file diff --git a/Include/Aurora/IO/IIOPipeProcessor.hpp b/Include/Aurora/IO/IIOPipeProcessor.hpp index ad8774e3..3381c605 100644 --- a/Include/Aurora/IO/IIOPipeProcessor.hpp +++ b/Include/Aurora/IO/IIOPipeProcessor.hpp @@ -11,62 +11,129 @@ namespace Aurora::IO { struct IIOPipeInterceptor; + struct IPipeFrames + { + /** + * @brief pipe starting hook + */ + virtual void OnStart() = 0; + + /** + * @brief This function is called once the stream reader returns zero + * You should use this opportunity to schedule the next waitable item state (eg, initiate async read, set event high, etc) + * You can return false to soft-fail the pipe to indicate EoS + * You should otherwise return true in order to continue yield until the next waitable item state change + * Note, an EoS event may also occur during the next alert state change should the stream reader return an error + */ + virtual void OnEndPump() = 0; + + /** + * @brief pipe shutdown hook + */ + virtual void OnEnd(bool fatal) = 0; + }; + struct IOPipeData { + /** + * @brief IO events + */ AuSPtr watchItem; + + /** + * @brief Input source + */ AuSPtr reader; + + /** + * @brief Output drain + */ AuSPtr writer; + + /** + * @brief Callbacks + */ + AuSPtr frames; + + /** + * @brief Enables aggressive stream consumption, allowing for bias towards clients if they were to send a lot of data (including dos) + * Instead of reading the input stream once, so long as the output stream written paremeter yields a non-zero number, bytes + * will continue to pump through the writer. Breakdown of the pipe is expected on reader oversaturation as defined by the + * IOPipeRequest + */ + bool bShouldReadUntilZero {}; }; struct IOPipeRequest { /** - * @brief The two streams to join and an invokable object - */ - IOPipeData data; - - /** - * @brief Amount of bytes to transfer + * @brief Amount of bytes to transfer or zero if stream */ AuUInt32 lengthOrZero {}; + /** + * @brief internal frame size or zero if fallback + */ + AuUInt32 pageLengthOrZero {}; + /** * @brief event listener */ AuSPtr listener; /** - * @brief Used as the buffer size for streams of length 0 + * @brief Used as the buffer size for streams of page length 0 */ AuUInt32 fallbackPageSize {4096 * 50}; AuSPtr processor; }; + struct IOPipeRequestBasic : IOPipeRequest + { + /** + * @brief The two streams to join and an invokable object + */ + IOPipeData data; + }; + + struct IOPipeRequestAIO : IOPipeRequest + { + AuSPtr asyncTransaction; + + AuSPtr writer; + + bool shouldReadUntilZero; + }; + + struct IIOPipeWork + { + /** + * @brief + * @return false should no IIOPipeEventListener event ever be fired. you should generally expect callback based failure. + */ + virtual bool Start() = 0; + + virtual bool End() = 0; + }; + /** * @brief Different operating systems implement high level stream copy abstraction between network, file, and/or file descriptors. - * This interface connects arbitrary stream objects to one another by piping data under an iprocessor tick; or delegates + * This interface connects arbitrary stream objects to one another by piping data under an iprocessor tick; or delegates * such task to the operating system, if possible. */ struct IIOPipeProcessor { /** - * @brief - * @param request - * @return + * @brief + * @param request + * @return */ - virtual bool BeginPipe(const IOPipeRequest &request) = 0; - - /** - * @brief - * @param itm - */ - virtual void EndByWatch(const AuSPtr &itm) = 0; + virtual AuSPtr NewBasicPipe(const IOPipeRequestBasic &request) = 0; /** * @brief - * @param itm */ - virtual void EndByListener(const AuSPtr &itm) = 0; + virtual AuSPtr NewAIOPipe(const IOPipeRequestAIO &request) = 0; }; } \ No newline at end of file diff --git a/Include/Aurora/Memory/README.md b/Include/Aurora/Memory/README.md new file mode 100644 index 00000000..baa1e184 --- /dev/null +++ b/Include/Aurora/Memory/README.md @@ -0,0 +1,12 @@ +# AuMemory + +Example Usage: \ + File: HelloAurora/master/Tests/Public/15. Hello ByteBuffer/Main.cpp + + +# Features +* ByteBuffer +* Optimize cache [heavy precache] +* O(1) Heap Allocator (based on a hacked up external lib) +* Purge Executable Memory Cache +* Pin virtual memory into physical memory / SwapLock \ No newline at end of file diff --git a/Include/Aurora/Processes/EStreamForward.hpp b/Include/Aurora/Processes/EStreamForward.hpp new file mode 100644 index 00000000..46849058 --- /dev/null +++ b/Include/Aurora/Processes/EStreamForward.hpp @@ -0,0 +1,17 @@ +/*** + Copyright (C) 2022 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: EStreamForward.hpp + Date: 2022-6-13 + Author: Reece +***/ +#pragma once + +namespace Aurora::Processes +{ + AUE_DEFINE(EStreamForward, ( + eNull, + eCurrentProcess, + eAsyncPipe + )); +} \ No newline at end of file diff --git a/Include/Aurora/Processes/Processes.hpp b/Include/Aurora/Processes/Processes.hpp index 692c0620..581838b0 100644 --- a/Include/Aurora/Processes/Processes.hpp +++ b/Include/Aurora/Processes/Processes.hpp @@ -9,6 +9,7 @@ #include "ESpawnType.hpp" #include "EStandardHandle.hpp" +#include "EStreamForward.hpp" #include "IProcess.hpp" #include "StartupParmaters.hpp" #include "Spawn.hpp" diff --git a/Include/Aurora/Processes/StartupParmaters.hpp b/Include/Aurora/Processes/StartupParmaters.hpp index de614847..de755386 100644 --- a/Include/Aurora/Processes/StartupParmaters.hpp +++ b/Include/Aurora/Processes/StartupParmaters.hpp @@ -31,23 +31,30 @@ namespace Aurora::Processes /** * @brief Enables stdout ipc to parent */ - bool fwdOut {}; + EStreamForward fwdOut {}; /** * @brief Enables stderr ipc to parent */ - bool fwdErr {}; + EStreamForward fwdErr {}; /** * @brief Enables stdin ipc from parent (us) to child */ - bool fwdIn {}; + EStreamForward fwdIn {}; /** * @brief Effectively is user facing. * Under Windows GUI, this means hides conhost. * Under ConsoleApps, this means blank the childs output handles. */ - bool noShowConsole {}; + bool bNoShowConsole {}; + + /** + * @brief Starts the process in a resumable suspension mode + */ + bool bInDebugMode {}; + + AuOptional workingDirectory; }; } \ No newline at end of file diff --git a/Include/Aurora/RNG/README.md b/Include/Aurora/RNG/README.md new file mode 100644 index 00000000..807b26bd --- /dev/null +++ b/Include/Aurora/RNG/README.md @@ -0,0 +1,12 @@ +# AuRng + +Example Usage: HelloAurora/master/Tests/Public/14. Hello RNG/Main.cpp + +# Features +* Static and IRandomDevice RNG routines +* Partial template support for arrays and strings +* Various numeric operations (int range, 0-1 decimal, string of character set, next word, etc) + +# Backends +* System provided CSRNG backend +* WELL fast backend; seeded by U32's, U64's, and larger WELL blob's diff --git a/Source/IO/IOProcessor.cpp b/Source/IO/IOProcessor.cpp index a6ac3cfd..91b48706 100644 --- a/Source/IO/IOProcessor.cpp +++ b/Source/IO/IOProcessor.cpp @@ -27,6 +27,11 @@ namespace Aurora::IO bool IOProcessor::Init() { + if (!this->ToQueue()) + { + return {}; + } + if (!this->items.Init()) { SysPushErrorNested(); @@ -39,6 +44,8 @@ namespace Aurora::IO return false; } + this->ToQueue()->SourceAdd(this->items.cvEvent); + return true; } @@ -119,6 +126,7 @@ namespace Aurora::IO AU_LOCK_GUARD(this->items.mutex); FrameStart(); FrameWaitForAny(0); + FramePumpWaitingBlocked(); FrameRunThreadIO(); FrameRunCheckLSes(); return FrameRunEpilogue(); @@ -425,17 +433,21 @@ namespace Aurora::IO } } + void IOProcessor::FramePumpWaitingBlocked() + { + auto blocked = this->items.GetBlockedSignals(); + if (blocked.size()) + { + this->items.workSignaled.insert(this->items.workSignaled.end(), blocked.begin(), blocked.end()); + } + } + void IOProcessor::FrameStart() { ReportState(EIOProcessorEventStage::eFrameStartOfFrame); this->bFrameStart = true; - auto blocked = this->items.GetBlockedSignals(); - if (blocked.size()) - { - AU_LOCK_GUARD(this->items.mutex); - this->items.workSignaled.insert(this->items.workSignaled.end(), blocked.begin(), blocked.end()); - } + FramePumpWaitingBlocked(); } void IOProcessor::FrameRunThreadIO() @@ -544,13 +556,24 @@ namespace Aurora::IO void IOProcessor::CancelWorkItem() { + if (!this->workItem) + { + return; + } + this->workItem->Cancel(); this->workItem.reset(); } void IOProcessor::RemoveLSTimer() { - this->ToQueue()->SourceRemove(this->timers.lsTicker); + auto queue = this->ToQueue(); + if (!queue) + { + return; + } + + queue->SourceRemove(this->timers.lsTicker); } bool IOProcessor::IsAsync() @@ -658,7 +681,17 @@ namespace Aurora::IO void IOProcessor::ReleaseAllWatches() { + RemoveTimer(); + auto queue = ToQueue(); + if (queue) + { + if (this->items.cvEvent) + { + queue->SourceRemove(this->items.cvEvent); + } + queue->Commit(); + } } bool IOProcessor::HasItems() diff --git a/Source/IO/IOProcessor.hpp b/Source/IO/IOProcessor.hpp index 169ce706..e1792e03 100644 --- a/Source/IO/IOProcessor.hpp +++ b/Source/IO/IOProcessor.hpp @@ -33,7 +33,8 @@ namespace Aurora::IO bool AddEventListener(const AuSPtr &eventListener) override; void RemoveEventListener(const AuSPtr &eventListener) override; - void FrameStart(); + void FrameStart(); + void FramePumpWaitingBlocked(); bool FrameWaitForAny(AuUInt32 msMax); AuUInt FrameRunEpilogue(); void FrameRunThreadIO(); @@ -102,6 +103,7 @@ namespace Aurora::IO AuThreadPrimitives::SpinLock listenersSpinLock; AuList> listeners; + bool bFrameStart {}; }; } \ No newline at end of file diff --git a/Source/IO/IOProcessorItems.cpp b/Source/IO/IOProcessorItems.cpp index bf657485..b1e0ee90 100644 --- a/Source/IO/IOProcessorItems.cpp +++ b/Source/IO/IOProcessorItems.cpp @@ -9,6 +9,7 @@ #include #include "IOProcessorItem.hpp" #include "IOProcessorItems.hpp" +#include "IOProcessor.hpp" namespace Aurora::IO { @@ -17,6 +18,8 @@ namespace Aurora::IO this->mutex = AuThreadPrimitives::CriticalSectionUnique(); this->mutex2 = AuThreadPrimitives::CriticalSectionUnique(); + this->cvEvent = AuLoop::NewLSEvent(false, true, true); + return bool(this->mutex) && bool(this->mutex2); } @@ -31,6 +34,7 @@ namespace Aurora::IO if (!AddFrameTemp(item)) { AU_LOCK_GUARD(this->mutex2); + this->cvEvent->Set(); return AuTryInsert(this->workSignaled2, item); } diff --git a/Source/IO/IOProcessorItems.hpp b/Source/IO/IOProcessorItems.hpp index 5eee303d..e9c898fc 100644 --- a/Source/IO/IOProcessorItems.hpp +++ b/Source/IO/IOProcessorItems.hpp @@ -14,6 +14,8 @@ namespace Aurora::IO struct IOProcessorItems { + AuSPtr cvEvent; + AuList> allItems; AuList> onTickReceivers; AuList> onOtherReceivers; diff --git a/Source/Processes/Process.NT.cpp b/Source/Processes/Process.NT.cpp index c335799e..f6a9386d 100644 --- a/Source/Processes/Process.NT.cpp +++ b/Source/Processes/Process.NT.cpp @@ -146,9 +146,7 @@ namespace Aurora::Processes void ProcessImpl::ShutdownPipes() { - AuWin32CloseHandle(this->pipeStdOutWrite_); - AuWin32CloseHandle(this->pipeStdErrWrite_); - AuWin32CloseHandle(this->pipeStdInRead_); + RelOtherHandles(); //AuWin32CloseHandle(this->pipeStdOutRead_); //AuWin32CloseHandle(this->pipeStdErrRead_); //AuWin32CloseHandle(this->pipeStdInWrite_); @@ -238,9 +236,14 @@ namespace Aurora::Processes } this->exitCode_ = 0x10110100; - if (this->startup_.fwdOut) + if (this->startup_.fwdOut == EStreamForward::eAsyncPipe) { - if (!CreatePipeEx(&pipeStdOutRead_, &pipeStdOutWrite_, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0)) + if (!CreatePipeEx(&this->pipeStdOutRead_, + &this->pipeStdOutWrite_, + &saAttr, + 0, + FILE_FLAG_OVERLAPPED, + 0)) { return false; } @@ -250,36 +253,81 @@ namespace Aurora::Processes return false; } } - - if (this->startup_.fwdErr) + else if (this->startup_.fwdOut == EStreamForward::eCurrentProcess) { - if (!CreatePipeEx(&pipeStdErrRead_, &pipeStdErrWrite_, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0)) + HANDLE handle = GetStdHandle(STD_OUTPUT_HANDLE); + + if (handle != INVALID_HANDLE_VALUE && handle) + { + SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1); + + this->bDontRelOut_ = true; + this->pipeStdOutWrite_ = handle; + } + } + + if (this->startup_.fwdErr == EStreamForward::eAsyncPipe) + { + if (!CreatePipeEx(&this->pipeStdErrRead_, + &this->pipeStdErrWrite_, + &saAttr, + 0, + FILE_FLAG_OVERLAPPED, + 0)) { return false; } - if (!SetHandleInformation(pipeStdErrRead_, HANDLE_FLAG_INHERIT, 0)) + if (!SetHandleInformation(this->pipeStdErrRead_, HANDLE_FLAG_INHERIT, 0)) { return false; } } - - if (this->startup_.fwdIn) + else if (this->startup_.fwdErr == EStreamForward::eCurrentProcess) { - if (!CreatePipeEx(&pipeStdInRead_, &pipeStdInWrite_, &saAttr, 0, 0, FILE_FLAG_OVERLAPPED)) + HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); + + if (handle != INVALID_HANDLE_VALUE && handle) + { + SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1); + + this->bDontRelErr_ = true; + this->pipeStdErrWrite_ = handle; + } + } + + if (this->startup_.fwdIn == EStreamForward::eAsyncPipe) + { + if (!CreatePipeEx(&this->pipeStdInRead_, + &this->pipeStdInWrite_, + &saAttr, + 0, + 0, + FILE_FLAG_OVERLAPPED)) { return false; } - if (!SetHandleInformation(pipeStdInWrite_, HANDLE_FLAG_INHERIT, 0)) + if (!SetHandleInformation(this->pipeStdInWrite_, HANDLE_FLAG_INHERIT, 0)) { return false; } } - - if (this->startup_.noShowConsole) + else if (this->startup_.fwdIn == EStreamForward::eCurrentProcess) { - HANDLE nulFile = INVALID_HANDLE_VALUE; + HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); + + if (handle != INVALID_HANDLE_VALUE && handle) + { + SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1); + + this->bDontRelIn_ = true; + this->pipeStdInRead_ = handle; + } + } + + { + HANDLE nulFile; #define NEW_NULL_HANDLE \ { \ @@ -306,7 +354,7 @@ namespace Aurora::Processes } } - if (this->startup_.fwdIn || this->startup_.fwdOut) + if (this->startup_.fwdIn == EStreamForward::eAsyncPipe || this->startup_.fwdOut == EStreamForward::eAsyncPipe) { this->fsHandle_ = AuMakeShared(); if (!this->fsHandle_) @@ -324,7 +372,7 @@ namespace Aurora::Processes this->fsStream_->Init(this->fsHandle_); } - if (this->startup_.fwdErr) + if (this->startup_.fwdErr == EStreamForward::eAsyncPipe) { this->fsErrorHandle_ = AuMakeShared(); if (!this->fsErrorHandle_) @@ -374,18 +422,29 @@ namespace Aurora::Processes STARTUPINFOW startupInfo = { 0 }; startupInfo.cb = sizeof(startupInfo); - bool inheritHandles = this->startup_.fwdIn || this->startup_.fwdErr || this->startup_.fwdOut || this->startup_.noShowConsole; + startupInfo.hStdInput = this->pipeStdInRead_; + startupInfo.hStdError = this->pipeStdErrWrite_; + startupInfo.hStdOutput = this->pipeStdOutWrite_; + startupInfo.dwFlags |= STARTF_USESTDHANDLES; - startupInfo.hStdInput = pipeStdInRead_; - startupInfo.hStdError = pipeStdErrWrite_; - startupInfo.hStdOutput = pipeStdOutWrite_; - startupInfo.dwFlags |= (inheritHandles ? STARTF_USESTDHANDLES : 0); + auto cwd = this->startup_.workingDirectory; + std::wstring wcwd; + if (cwd) + { + wcwd = Locale::ConvertFromUTF8(this->windowsCli_); + if (!wcwd.size()) + { + SysPushErrorMem(); + return false; + } + } auto result = CreateProcessW(Locale::ConvertFromUTF8(this->startup_.process).c_str(), Locale::ConvertFromUTF8(this->windowsCli_).data(), - NULL, NULL, inheritHandles, - this->startup_.noShowConsole ? CREATE_NO_WINDOW : NULL, // yea we can keep CREATE_NO_WINDOW on for non-console apps. its legal -> https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags - NULL, NULL, &startupInfo, &processInfo); + NULL, NULL, true, + (this->startup_.bNoShowConsole ? CREATE_NO_WINDOW : NULL) | // yea we can keep CREATE_NO_WINDOW on for non-console apps. its legal -> https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + (this->startup_.bInDebugMode ? CREATE_SUSPENDED : NULL), + NULL, wcwd.size() ? wcwd.data() : nullptr, &startupInfo, &processInfo); if (!result) { @@ -397,9 +456,7 @@ namespace Aurora::Processes this->process_ = processInfo.hProcess; this->hthread_ = processInfo.hThread; - AuWin32CloseHandle(this->pipeStdOutWrite_); - AuWin32CloseHandle(this->pipeStdErrWrite_); - AuWin32CloseHandle(this->pipeStdInRead_); + RelOtherHandles(); if (this->type_ == ESpawnType::eSpawnChildProcessWorker) { @@ -409,7 +466,7 @@ namespace Aurora::Processes } } - // TODO: delegate to a singular worker thread + // TODO: delegate to a singular worker thread / SetThreadPoolWait auto a = [=]() { WaitForSingleObject(processInfo.hProcess, INFINITE); @@ -439,6 +496,25 @@ namespace Aurora::Processes return true; } + + void ProcessImpl::RelOtherHandles() + { + if (!this->bDontRelOut_) + { + AuWin32CloseHandle(this->pipeStdOutWrite_); + } + + if (!this->bDontRelErr_) + { + AuWin32CloseHandle(this->pipeStdErrWrite_); + } + + if (!this->bDontRelIn_) + { + AuWin32CloseHandle(this->pipeStdInRead_); + } + } + AUKN_SYM IProcess *SpawnNew(const StartupParmaters ¶ms) { try diff --git a/Source/Processes/Process.NT.hpp b/Source/Processes/Process.NT.hpp index 4e414b54..a5489f3f 100644 --- a/Source/Processes/Process.NT.hpp +++ b/Source/Processes/Process.NT.hpp @@ -40,6 +40,8 @@ namespace Aurora::Processes bool Init(); + void RelOtherHandles(); + private: HANDLE pipeStdOutRead_ {INVALID_HANDLE_VALUE}; @@ -49,6 +51,12 @@ namespace Aurora::Processes HANDLE pipeStdInRead_ {INVALID_HANDLE_VALUE}; HANDLE pipeStdInWrite_ {INVALID_HANDLE_VALUE}; + + bool bDontRelOut_ {}; + bool bDontRelIn_ {}; + bool bDontRelErr_ {}; + + AuSPtr loopSource_; AuSPtr fsHandle_; diff --git a/Source/Processes/Process.Unix.cpp b/Source/Processes/Process.Unix.cpp index 16365800..4ccecb53 100644 --- a/Source/Processes/Process.Unix.cpp +++ b/Source/Processes/Process.Unix.cpp @@ -339,6 +339,7 @@ namespace Aurora::Processes pid_t pid; { + // TODO: clone without CLONE_FS and CLONE_THREAD (if non-sid path) pid = ::fork(); if (pid == 0) @@ -369,7 +370,8 @@ namespace Aurora::Processes { ::setsid(); } - + + // TODO: pthread_chdir_np on OSX ::execv(this->startup_.process.c_str(), (char * const *)this->cargs_.data()); // https://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html SysPushErrorGen("execv didn't overwrite the process map. Launch: {} ({})", this->startup_.process, this->debug_); @@ -396,7 +398,7 @@ namespace Aurora::Processes this->alive_ = true; { - AU_LOCK_GUARD(gRWLock->AsWritable()); + AU_LOCK_GUARD(gRWLock->AsWritable());\ SysAssert(AuTryInsert(gPidLookupMap, pid, this)); }