[TODO: BLOCK 2 PROGRESS]

Include/Aurora/Exit/README.md

@@ -0,0 +1,9 @@
+# AuExit
+
+Provides termination events, usually on a worker thread, when application termination is likely. Event categories include: control c, an exception was thrown, application shutdown.
+
+Trigger Levels:
+* eSafeTermination (safe unload)
+* eFatalException (a fatal exception, math, or memory error occurred) 
+* eSigTerminate (control + c)
+* eProblematicEvent (an exception was thrown or some other fatal condition. should probably flush & sync streams or invalidate caches.)

Include/Aurora/IO/IIOPipeProcessor.hpp

@@ -11,62 +11,129 @@ namespace Aurora::IO
 {
     struct IIOPipeInterceptor;
 
+    struct IPipeFrames
+    {
+        /**
+         * @brief pipe starting hook
+        */
+        virtual void OnStart() = 0;
+
+        /**
+         * @brief This function is called once the stream reader returns zero
+         *        You should use this opportunity to schedule the next waitable item state (eg, initiate async read, set event high, etc)
+         *        You can return false to soft-fail the pipe to indicate EoS
+         *        You should otherwise return true in order to continue yield until the next waitable item state change
+         *        Note, an EoS event may also occur during the next alert state change should the stream reader return an error
+        */
+        virtual void OnEndPump() = 0;
+
+        /**
+         * @brief pipe shutdown hook
+        */
+        virtual void OnEnd(bool fatal) = 0;
+    };
+
     struct IOPipeData
     {
+        /**
+         * @brief IO events
+        */
         AuSPtr<IIOWaitableItem> watchItem;
+
+        /**
+         * @brief Input source
+        */
         AuSPtr<IStreamReader> reader;
+
+        /**
+         * @brief Output drain
+        */
         AuSPtr<IStreamWriter> writer;
+
+        /**
+         * @brief Callbacks
+        */
+        AuSPtr<IPipeFrames> frames;
+
+        /**
+         * @brief Enables aggressive stream consumption, allowing for bias towards clients if they were to send a lot of data (including dos)
+         *        Instead of reading the input stream once, so long as the output stream written paremeter yields a non-zero number, bytes
+         *        will continue to pump through the writer. Breakdown of the pipe is expected on reader oversaturation as defined by the
+         *        IOPipeRequest
+        */
+        bool bShouldReadUntilZero {};
     };
 
     struct IOPipeRequest
     {
         /**
-         * @brief The two streams to join and an invokable object
+         * @brief Amount of bytes to transfer or zero if stream
-        */
-        IOPipeData data;
-
-        /**
-         * @brief Amount of bytes to transfer
         */
         AuUInt32 lengthOrZero {};
 
+        /**
+         * @brief internal frame size or zero if fallback
+        */
+        AuUInt32 pageLengthOrZero {};
+
         /**
          * @brief event listener
         */
         AuSPtr<IIOPipeEventListener> listener;
 
         /**
-         * @brief Used as the buffer size for streams of length 0
+         * @brief Used as the buffer size for streams of page length 0
         */
         AuUInt32 fallbackPageSize {4096 * 50};
 
         AuSPtr<IIOPipeInterceptor> processor;
     };
 
+    struct IOPipeRequestBasic : IOPipeRequest
+    {
+        /**
+         * @brief The two streams to join and an invokable object
+        */
+        IOPipeData data;
+    };
+
+    struct IOPipeRequestAIO : IOPipeRequest
+    {
+        AuSPtr<IAsyncTransaction> asyncTransaction;
+
+        AuSPtr<IStreamWriter> writer;
+
+        bool shouldReadUntilZero;
+    };
+
+    struct IIOPipeWork
+    {
+        /**
+         * @brief 
+         * @return false should no IIOPipeEventListener event ever be fired. you should generally expect callback based failure.
+        */
+        virtual bool Start() = 0;
+
+        virtual bool End() = 0;
+    };
+
     /**
      * @brief Different operating systems implement high level stream copy abstraction between network, file, and/or file descriptors.
-     *        This interface connects arbitrary stream objects to one another by piping data under an iprocessor tick; or delegates 
+     *        This interface connects arbitrary stream objects to one another by piping data under an iprocessor tick; or delegates
      *        such task to the operating system, if possible.
     */
     struct IIOPipeProcessor
     {
         /**
-         * @brief 
+         * @brief
-         * @param request 
+         * @param request
-         * @return 
+         * @return
         */
-        virtual bool BeginPipe(const IOPipeRequest &request) = 0;
+        virtual AuSPtr<IIOPipeWork> NewBasicPipe(const IOPipeRequestBasic &request) = 0;
-        
-        /**
-         * @brief 
-         * @param itm 
-        */
-        virtual void EndByWatch(const AuSPtr<IIOWaitableItem> &itm) = 0;
 
         /**
          * @brief 
-         * @param itm 
         */
-        virtual void EndByListener(const AuSPtr<IIOPipeEventListener> &itm) = 0;
+        virtual AuSPtr<IIOPipeWork> NewAIOPipe(const IOPipeRequestAIO &request) = 0;
     };
 }

Include/Aurora/Memory/README.md

@@ -0,0 +1,12 @@
+# AuMemory
+
+Example Usage: \
+ File: HelloAurora/master/Tests/Public/15. Hello ByteBuffer/Main.cpp
+
+
+# Features
+* ByteBuffer
+* Optimize cache [heavy precache]
+* O(1) Heap Allocator (based on a hacked up external lib)
+* Purge Executable Memory Cache
+* Pin virtual memory into physical memory / SwapLock

Include/Aurora/Processes/EStreamForward.hpp

@@ -0,0 +1,17 @@
+/***
+    Copyright (C) 2022 J Reece Wilson (a/k/a "Reece"). All rights reserved.
+
+    File: EStreamForward.hpp
+    Date: 2022-6-13
+    Author: Reece
+***/
+#pragma once
+
+namespace Aurora::Processes
+{
+    AUE_DEFINE(EStreamForward, (
+        eNull,
+        eCurrentProcess,
+        eAsyncPipe
+    ));
+}

Include/Aurora/Processes/Processes.hpp

@@ -9,6 +9,7 @@
 
 #include "ESpawnType.hpp"
 #include "EStandardHandle.hpp"
+#include "EStreamForward.hpp"
 #include "IProcess.hpp"
 #include "StartupParmaters.hpp"
 #include "Spawn.hpp"

Include/Aurora/Processes/StartupParmaters.hpp

@@ -31,23 +31,30 @@ namespace Aurora::Processes
         /**
          * @brief Enables stdout ipc to parent
         */
-        bool fwdOut {};
+        EStreamForward fwdOut {};
 
         /**
          * @brief Enables stderr ipc to parent
         */
-        bool fwdErr {};
+        EStreamForward fwdErr {};
 
         /**
          * @brief Enables stdin ipc from parent (us) to child
         */
-        bool fwdIn {};
+        EStreamForward fwdIn {};
 
         /**
          * @brief Effectively is user facing. 
          *        Under Windows GUI, this means hides conhost. 
          *        Under ConsoleApps, this means blank the childs output handles.
         */
-        bool noShowConsole {};
+        bool bNoShowConsole {};
+
+        /**
+         * @brief Starts the process in a resumable suspension mode
+        */
+        bool bInDebugMode {};
+
+        AuOptional<AuString> workingDirectory;
     };
 }

Include/Aurora/RNG/README.md

@@ -0,0 +1,12 @@
+# AuRng
+
+Example Usage: HelloAurora/master/Tests/Public/14. Hello RNG/Main.cpp
+
+# Features
+* Static and IRandomDevice RNG routines
+* Partial template support for arrays and strings
+* Various numeric operations (int range, 0-1 decimal, string of character set, next word, etc)
+
+# Backends
+* System provided CSRNG backend
+* WELL fast backend; seeded by U32's, U64's, and larger WELL blob's

Source/IO/IOProcessor.cpp

@@ -27,6 +27,11 @@ namespace Aurora::IO
 
     bool IOProcessor::Init()
     {
+        if (!this->ToQueue())
+        {
+            return {};
+        }
+
         if (!this->items.Init())
         {
             SysPushErrorNested();
@@ -39,6 +44,8 @@ namespace Aurora::IO
             return false;
         }
 
+        this->ToQueue()->SourceAdd(this->items.cvEvent);
+
         return true;
     }
 
@@ -119,6 +126,7 @@ namespace Aurora::IO
         AU_LOCK_GUARD(this->items.mutex);
         FrameStart();
         FrameWaitForAny(0);
+        FramePumpWaitingBlocked();
         FrameRunThreadIO();
         FrameRunCheckLSes();
         return FrameRunEpilogue();
@@ -425,17 +433,21 @@ namespace Aurora::IO
         }
     }
 
+    void IOProcessor::FramePumpWaitingBlocked()
+    {
+        auto blocked = this->items.GetBlockedSignals();
+        if (blocked.size())
+        {
+            this->items.workSignaled.insert(this->items.workSignaled.end(), blocked.begin(), blocked.end());
+        }
+    }
+
     void IOProcessor::FrameStart()
     {
         ReportState(EIOProcessorEventStage::eFrameStartOfFrame);
         this->bFrameStart = true;
 
-        auto blocked = this->items.GetBlockedSignals();
+        FramePumpWaitingBlocked();
-        if (blocked.size())
-        {
-            AU_LOCK_GUARD(this->items.mutex);
-            this->items.workSignaled.insert(this->items.workSignaled.end(), blocked.begin(), blocked.end());
-        }
     }
 
     void IOProcessor::FrameRunThreadIO()
@@ -544,13 +556,24 @@ namespace Aurora::IO
 
     void IOProcessor::CancelWorkItem()
     {
+        if (!this->workItem)
+        {
+            return;
+        }
+
         this->workItem->Cancel();
         this->workItem.reset();
     }
 
     void IOProcessor::RemoveLSTimer()
     {
-        this->ToQueue()->SourceRemove(this->timers.lsTicker);
+        auto queue = this->ToQueue();
+        if (!queue)
+        {
+            return;
+        }
+
+        queue->SourceRemove(this->timers.lsTicker);
     }
 
     bool IOProcessor::IsAsync()
@@ -658,7 +681,17 @@ namespace Aurora::IO
 
     void IOProcessor::ReleaseAllWatches()
     {
+        RemoveTimer();
+        auto queue = ToQueue();
+        if (queue)
+        {
+            if (this->items.cvEvent)
+            {
+                queue->SourceRemove(this->items.cvEvent);
+            }
 
+            queue->Commit();
+        }
     }
 
     bool IOProcessor::HasItems()

Source/IO/IOProcessor.hpp

@@ -33,7 +33,8 @@ namespace Aurora::IO
         bool AddEventListener(const AuSPtr<IIOProcessorEventListener> &eventListener) override;
         void RemoveEventListener(const AuSPtr<IIOProcessorEventListener> &eventListener) override;
 
-        void FrameStart();
+        void FrameStart(); 
+        void FramePumpWaitingBlocked();
         bool FrameWaitForAny(AuUInt32 msMax);
         AuUInt FrameRunEpilogue();
         void FrameRunThreadIO();
@@ -102,6 +103,7 @@ namespace Aurora::IO
         AuThreadPrimitives::SpinLock listenersSpinLock;
         AuList<AuSPtr<IIOProcessorEventListener>> listeners;
 
+
         bool bFrameStart {};
     };
 }

Source/IO/IOProcessorItems.cpp

@@ -9,6 +9,7 @@
 #include <Aurora/IO/IOExperimental.hpp>
 #include "IOProcessorItem.hpp"
 #include "IOProcessorItems.hpp"
+#include "IOProcessor.hpp"
 
 namespace Aurora::IO
 {
@@ -17,6 +18,8 @@ namespace Aurora::IO
         this->mutex = AuThreadPrimitives::CriticalSectionUnique();
         this->mutex2 = AuThreadPrimitives::CriticalSectionUnique();
 
+        this->cvEvent = AuLoop::NewLSEvent(false, true, true);
+
         return bool(this->mutex) && bool(this->mutex2);
     }
 
@@ -31,6 +34,7 @@ namespace Aurora::IO
         if (!AddFrameTemp(item))
         {
             AU_LOCK_GUARD(this->mutex2);
+            this->cvEvent->Set();
             return AuTryInsert(this->workSignaled2, item);
         }
 

Source/IO/IOProcessorItems.hpp

@@ -14,6 +14,8 @@ namespace Aurora::IO
 
     struct IOProcessorItems
     {
+        AuSPtr<AuLoop::ILSEvent> cvEvent;
+
         AuList<AuSPtr<IOProcessorItem>> allItems;
         AuList<AuSPtr<IOProcessorItem>> onTickReceivers;
         AuList<AuSPtr<IOProcessorItem>> onOtherReceivers;

Source/Processes/Process.NT.cpp

@@ -146,9 +146,7 @@ namespace Aurora::Processes
 
     void ProcessImpl::ShutdownPipes()
     {
-        AuWin32CloseHandle(this->pipeStdOutWrite_);
+        RelOtherHandles();
-        AuWin32CloseHandle(this->pipeStdErrWrite_);
-        AuWin32CloseHandle(this->pipeStdInRead_);
         //AuWin32CloseHandle(this->pipeStdOutRead_);
         //AuWin32CloseHandle(this->pipeStdErrRead_);
         //AuWin32CloseHandle(this->pipeStdInWrite_);
@@ -238,9 +236,14 @@ namespace Aurora::Processes
         }
 
         this->exitCode_ = 0x10110100;
-        if (this->startup_.fwdOut)
+        if (this->startup_.fwdOut == EStreamForward::eAsyncPipe)
         {
-            if (!CreatePipeEx(&pipeStdOutRead_, &pipeStdOutWrite_, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0))
+            if (!CreatePipeEx(&this->pipeStdOutRead_,
+                              &this->pipeStdOutWrite_,
+                              &saAttr,
+                              0,
+                              FILE_FLAG_OVERLAPPED,
+                              0))
             {
                 return false;
             }
@@ -250,36 +253,81 @@ namespace Aurora::Processes
                 return false;
             }
         }
-
+        else if (this->startup_.fwdOut == EStreamForward::eCurrentProcess)
-        if (this->startup_.fwdErr)
         {
-            if (!CreatePipeEx(&pipeStdErrRead_, &pipeStdErrWrite_, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0))
+            HANDLE handle = GetStdHandle(STD_OUTPUT_HANDLE);
+
+            if (handle != INVALID_HANDLE_VALUE && handle)
+            {
+                SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1);
+
+                this->bDontRelOut_ = true;
+                this->pipeStdOutWrite_ = handle;
+            }
+        }
+
+        if (this->startup_.fwdErr == EStreamForward::eAsyncPipe)
+        {
+            if (!CreatePipeEx(&this->pipeStdErrRead_,
+                              &this->pipeStdErrWrite_,
+                              &saAttr,
+                              0,
+                              FILE_FLAG_OVERLAPPED,
+                              0))
             {
                 return false;
             }
 
-            if (!SetHandleInformation(pipeStdErrRead_, HANDLE_FLAG_INHERIT, 0))
+            if (!SetHandleInformation(this->pipeStdErrRead_, HANDLE_FLAG_INHERIT, 0))
             {
                 return false;
             }
         }
-
+        else if (this->startup_.fwdErr == EStreamForward::eCurrentProcess)
-        if (this->startup_.fwdIn)
         {
-            if (!CreatePipeEx(&pipeStdInRead_, &pipeStdInWrite_, &saAttr, 0, 0, FILE_FLAG_OVERLAPPED))
+            HANDLE handle = GetStdHandle(STD_ERROR_HANDLE);
+
+            if (handle != INVALID_HANDLE_VALUE && handle)
+            {
+                SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1);
+
+                this->bDontRelErr_ = true;
+                this->pipeStdErrWrite_ = handle;
+            }
+        }
+
+        if (this->startup_.fwdIn == EStreamForward::eAsyncPipe)
+        {
+            if (!CreatePipeEx(&this->pipeStdInRead_,
+                              &this->pipeStdInWrite_,
+                              &saAttr,
+                              0,
+                              0,
+                              FILE_FLAG_OVERLAPPED))
             {
                 return false;
             }
 
-            if (!SetHandleInformation(pipeStdInWrite_, HANDLE_FLAG_INHERIT, 0))
+            if (!SetHandleInformation(this->pipeStdInWrite_, HANDLE_FLAG_INHERIT, 0))
             {
                 return false;
             }
         }
-
+        else if (this->startup_.fwdIn == EStreamForward::eCurrentProcess)
-        if (this->startup_.noShowConsole)
         {
-            HANDLE nulFile = INVALID_HANDLE_VALUE;
+            HANDLE handle = GetStdHandle(STD_ERROR_HANDLE);
+
+            if (handle != INVALID_HANDLE_VALUE && handle)
+            {
+                SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 1);
+
+                this->bDontRelIn_ = true;
+                this->pipeStdInRead_ = handle;
+            }
+        }
+
+        {
+            HANDLE nulFile;
 
         #define NEW_NULL_HANDLE                                                                                                                \
             {                                                                                                                                  \
@@ -306,7 +354,7 @@ namespace Aurora::Processes
             }
         }
 
-        if (this->startup_.fwdIn || this->startup_.fwdOut)
+        if (this->startup_.fwdIn == EStreamForward::eAsyncPipe || this->startup_.fwdOut == EStreamForward::eAsyncPipe)
         {
             this->fsHandle_ = AuMakeShared<IO::FS::FileHandle>();
             if (!this->fsHandle_)
@@ -324,7 +372,7 @@ namespace Aurora::Processes
             this->fsStream_->Init(this->fsHandle_);
         }
 
-        if (this->startup_.fwdErr)
+        if (this->startup_.fwdErr == EStreamForward::eAsyncPipe)
         {
             this->fsErrorHandle_ = AuMakeShared<IO::FS::FileHandle>();
             if (!this->fsErrorHandle_)
@@ -374,18 +422,29 @@ namespace Aurora::Processes
             STARTUPINFOW startupInfo = { 0 };
             startupInfo.cb = sizeof(startupInfo);
             
-            bool inheritHandles = this->startup_.fwdIn || this->startup_.fwdErr || this->startup_.fwdOut || this->startup_.noShowConsole;
+            startupInfo.hStdInput  = this->pipeStdInRead_;
+            startupInfo.hStdError  = this->pipeStdErrWrite_;
+            startupInfo.hStdOutput = this->pipeStdOutWrite_;
+            startupInfo.dwFlags |= STARTF_USESTDHANDLES;
 
-            startupInfo.hStdInput  = pipeStdInRead_;
+            auto cwd = this->startup_.workingDirectory;
-            startupInfo.hStdError  = pipeStdErrWrite_;
+            std::wstring wcwd;
-            startupInfo.hStdOutput = pipeStdOutWrite_;
+            if (cwd)
-            startupInfo.dwFlags   |= (inheritHandles ? STARTF_USESTDHANDLES : 0);
+            {
+                wcwd = Locale::ConvertFromUTF8(this->windowsCli_);
+                if (!wcwd.size())
+                {
+                    SysPushErrorMem();
+                    return false;
+                }
+            }
 
             auto result = CreateProcessW(Locale::ConvertFromUTF8(this->startup_.process).c_str(),
                                          Locale::ConvertFromUTF8(this->windowsCli_).data(),
-                                         NULL, NULL, inheritHandles,
+                                         NULL, NULL, true,
-                                         this->startup_.noShowConsole ? CREATE_NO_WINDOW : NULL, // yea we can keep CREATE_NO_WINDOW on for non-console apps. its legal -> https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags
+                                         (this->startup_.bNoShowConsole ? CREATE_NO_WINDOW : NULL) | // yea we can keep CREATE_NO_WINDOW on for non-console apps. its legal -> https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags
-                                         NULL, NULL, &startupInfo, &processInfo);
+                                         (this->startup_.bInDebugMode ? CREATE_SUSPENDED : NULL),
+                                         NULL, wcwd.size() ? wcwd.data() : nullptr, &startupInfo, &processInfo);
 
             if (!result)
             {
@@ -397,9 +456,7 @@ namespace Aurora::Processes
             this->process_ = processInfo.hProcess;
             this->hthread_ = processInfo.hThread;
 
-            AuWin32CloseHandle(this->pipeStdOutWrite_);
+            RelOtherHandles();
-            AuWin32CloseHandle(this->pipeStdErrWrite_);
-            AuWin32CloseHandle(this->pipeStdInRead_);
 
             if (this->type_ == ESpawnType::eSpawnChildProcessWorker)
             {
@@ -409,7 +466,7 @@ namespace Aurora::Processes
             }
         }
 
-        // TODO: delegate to a singular worker thread
+        // TODO: delegate to a singular worker thread / SetThreadPoolWait
         auto a = [=]()
         {
             WaitForSingleObject(processInfo.hProcess, INFINITE);
@@ -439,6 +496,25 @@ namespace Aurora::Processes
         return true;
     }
 
+
+    void ProcessImpl::RelOtherHandles()
+    {
+        if (!this->bDontRelOut_)
+        {
+            AuWin32CloseHandle(this->pipeStdOutWrite_);
+        }
+
+        if (!this->bDontRelErr_)
+        {
+            AuWin32CloseHandle(this->pipeStdErrWrite_);
+        }
+
+        if (!this->bDontRelIn_)
+        {
+            AuWin32CloseHandle(this->pipeStdInRead_);
+        }
+    }
+
     AUKN_SYM IProcess *SpawnNew(const StartupParmaters &params)
     {
         try

Source/Processes/Process.NT.hpp

@@ -40,6 +40,8 @@ namespace Aurora::Processes
 
         bool Init();
 
+        void RelOtherHandles();
+
     private:
         
         HANDLE pipeStdOutRead_  {INVALID_HANDLE_VALUE};
@@ -49,6 +51,12 @@ namespace Aurora::Processes
         HANDLE pipeStdInRead_   {INVALID_HANDLE_VALUE};
         HANDLE pipeStdInWrite_  {INVALID_HANDLE_VALUE};
 
+
+        bool bDontRelOut_ {};
+        bool bDontRelIn_ {};
+        bool bDontRelErr_ {};
+
+
         AuSPtr<AuLoop::ILoopSource> loopSource_;
 
         AuSPtr<IO::FS::FileHandle> fsHandle_;

Source/Processes/Process.Unix.cpp

@@ -339,6 +339,7 @@ namespace Aurora::Processes
 
         pid_t pid;
         {
+            // TODO: clone without CLONE_FS and CLONE_THREAD (if non-sid path) 
             pid = ::fork();
 
             if (pid == 0)
@@ -369,7 +370,8 @@ namespace Aurora::Processes
                 {
                     ::setsid();
                 }
-
+                
+                // TODO: pthread_chdir_np on OSX
                 ::execv(this->startup_.process.c_str(), (char * const *)this->cargs_.data()); // https://pubs.opengroup.org/onlinepubs/9699919799/functions/exec.html
                 
                 SysPushErrorGen("execv didn't overwrite the process map. Launch: {} ({})", this->startup_.process, this->debug_);
@@ -396,7 +398,7 @@ namespace Aurora::Processes
                 this->alive_ = true;
 
                 {
-                    AU_LOCK_GUARD(gRWLock->AsWritable());
+                    AU_LOCK_GUARD(gRWLock->AsWritable());\
                     SysAssert(AuTryInsert(gPidLookupMap, pid, this));
                 }