From 99c5e1fa653ccb07073d0ce19f95c2539d68d6d9 Mon Sep 17 00:00:00 2001 From: Reece Date: Fri, 5 Nov 2021 17:34:23 +0000 Subject: [PATCH] A pretty large patch not worth breaking up into separate commits [*] Split up Aurora Async [*] Split Async app into seperate ThreadPool concept [*] Fix various OSThread bugs and tls transfer issues [*] Set default affinity to 0xFFFFFFFF [*] Update Build script [+] Add AuTuplePopFront [+] New Network Interface (unimplemented) [*] Stub out the interfaces required for a better logger [*] Fix Win32 ShellExecute bug; windows 11 struggles without explicit com init per the docs - now deferring to thread pool [*] Update gitignore [*] Follow XDG home standard [*] Refactor some namespaces to use the shorthand aliases [*] Various stability fixes --- .gitignore | 192 ++- Aurora.json | 6 +- Include/Aurora/Async/Async.hpp | 610 +--------- Include/Aurora/Async/AsyncTypes.hpp | 75 ++ Include/Aurora/Async/IAsyncApp.hpp | 25 + Include/Aurora/Async/IThreadPool.hpp | 70 ++ Include/Aurora/Async/IWorkItem.hpp | 54 + Include/Aurora/Async/IWorkItemHandler.hpp | 47 + Include/Aurora/Async/JobFrom.hpp | 186 +++ Include/Aurora/Async/Jobs.hpp | 33 + Include/Aurora/Async/TaskFrom.hpp | 87 ++ Include/Aurora/Async/Tasks.hpp | 32 + Include/Aurora/Async/WorkBasic.hpp | 118 ++ Include/Aurora/Async/WorkPairImpl.hpp | 372 ++++++ Include/Aurora/Console/Commands/Commands.hpp | 15 +- Include/Aurora/Console/Console.hpp | 15 +- Include/Aurora/Console/Hooks/Hooks.hpp | 10 + .../ITextLineSubscriber.hpp | 2 +- Include/Aurora/Console/Logging/IBasicSink.hpp | 17 + .../Aurora/Console/Logging/IBasicSinkRB.hpp | 17 + Include/Aurora/Console/Logging/ILogger.hpp | 84 ++ Include/Aurora/Console/Logging/Logging.hpp | 14 +- Include/Aurora/Console/Logging/Sinks.hpp | 22 + Include/Aurora/Console/README.md | 15 + Include/Aurora/Data/Data.hpp | 2 +- Include/Aurora/IO/Net/Net.hpp | 561 ++++++--- Include/Aurora/Locale/ECodePage.hpp | 2 +- Include/Aurora/Locale/Encoding/Encoding.hpp | 32 +- Include/Aurora/Memory/ByteBuffer.hpp | 1 - Include/Aurora/Process/Process.hpp | 15 +- Include/Aurora/Processes/IProcess.hpp | 1 + Include/Aurora/RNG/IRandomDevice.hpp | 2 +- Include/Aurora/Runtime.hpp | 48 +- .../Aurora/Threading/Threads/EThreadPrio.hpp | 6 +- .../Aurora/Threading/Threads/TLSVariable.hpp | 5 + .../Aurora/Threading/Threads/ThreadInfo.hpp | 6 +- Include/AuroraTypedefs.hpp | 6 +- Include/AuroraUtils.hpp | 15 + Source/Async/Async.cpp | 3 + Source/Async/Async.hpp | 37 +- Source/Async/AsyncApp.cpp | 1048 ++-------------- Source/Async/AsyncApp.hpp | 99 +- Source/Async/AsyncRunnable.hpp | 75 ++ Source/Async/GroupState.cpp | 35 + Source/Async/GroupState.hpp | 34 + Source/Async/Schedular.cpp | 50 +- Source/Async/Schedular.hpp | 6 +- Source/Async/ThreadPool.cpp | 1061 +++++++++++++++++ Source/Async/ThreadPool.hpp | 127 ++ Source/Async/ThreadState.hpp | 56 + Source/Async/WorkItem.cpp | 126 +- Source/Async/WorkItem.hpp | 12 +- Source/Console/Commands/Commands.cpp | 46 +- Source/Console/Console.cpp | 13 +- Source/Console/ConsoleFIO/ConsoleFIO.cpp | 6 +- Source/Console/ConsoleStd/ConsoleStd.cpp | 20 +- .../ConsoleWxWidgets/ConsoleWxWidgets.cpp | 21 +- Source/Console/Flusher.cpp | 14 +- Source/Console/Hooks/Hooks.cpp | 7 +- Source/Console/Hooks/Hooks.hpp | 2 + Source/Debug/ExceptionWatcher.Win32.cpp | 18 +- Source/Entrypoint.cpp | 72 ++ Source/HWInfo/CpuInfo.cpp | 8 +- Source/IO/FS/Async.NT.cpp | 2 +- Source/IO/FS/Resources.cpp | 60 +- Source/IO/Net/Net.cpp | 4 +- Source/IO/Net/Net.hpp | 5 +- Source/IO/Net/SocketStatAverageBps.hpp | 86 ++ Source/IO/Net/SocketStatChannel.hpp | 36 + Source/Locale/Encoding/ConvertInternal.cpp | 5 - Source/Locale/Encoding/ConvertInternal.hpp | 1 - Source/Locale/Encoding/EncoderAdapter.cpp | 74 +- Source/Locale/Encoding/EncoderAdapter.hpp | 4 +- Source/Locale/Encoding/EncoderNSL.cpp | 42 - Source/Locale/Encoding/EncoderNSL.hpp | 1 - Source/Locale/Encoding/Encoding.cpp | 92 +- Source/Locale/Encoding/Encoding.hpp | 140 +-- Source/Locale/Encoding/GBK/GBK.hpp | 36 + Source/Locale/Encoding/SJIS/SJIS.hpp | 67 ++ Source/Locale/Encoding/UTFn/AuUTF16.hpp | 53 + Source/Locale/Encoding/UTFn/AuUTF32.hpp | 188 +++ Source/Locale/Encoding/UTFn/AuUTF8.hpp | 13 + Source/Locale/Locale.cpp | 2 +- Source/Memory/Heap.cpp | 22 +- Source/Process/Process.cpp | 6 +- Source/Processes/Open.Linux.cpp | 3 +- Source/Processes/Open.Win32.cpp | 8 +- Source/Processes/Process.Win32.cpp | 16 +- Source/Threading/Threads/OSThread.cpp | 68 +- Source/Threading/Threads/OSThread.hpp | 1 + 90 files changed, 4593 insertions(+), 2258 deletions(-) create mode 100644 Include/Aurora/Async/AsyncTypes.hpp create mode 100644 Include/Aurora/Async/IAsyncApp.hpp create mode 100644 Include/Aurora/Async/IThreadPool.hpp create mode 100644 Include/Aurora/Async/IWorkItem.hpp create mode 100644 Include/Aurora/Async/IWorkItemHandler.hpp create mode 100644 Include/Aurora/Async/JobFrom.hpp create mode 100644 Include/Aurora/Async/Jobs.hpp create mode 100644 Include/Aurora/Async/TaskFrom.hpp create mode 100644 Include/Aurora/Async/Tasks.hpp create mode 100644 Include/Aurora/Async/WorkBasic.hpp create mode 100644 Include/Aurora/Async/WorkPairImpl.hpp rename Include/Aurora/Console/{Commands => Hooks}/ITextLineSubscriber.hpp (88%) create mode 100644 Include/Aurora/Console/Logging/IBasicSink.hpp create mode 100644 Include/Aurora/Console/Logging/IBasicSinkRB.hpp create mode 100644 Include/Aurora/Console/Logging/ILogger.hpp create mode 100644 Include/Aurora/Console/Logging/Sinks.hpp create mode 100644 Include/Aurora/Console/README.md create mode 100644 Source/Async/AsyncRunnable.hpp create mode 100644 Source/Async/GroupState.cpp create mode 100644 Source/Async/GroupState.hpp create mode 100644 Source/Async/ThreadPool.cpp create mode 100644 Source/Async/ThreadPool.hpp create mode 100644 Source/Async/ThreadState.hpp create mode 100644 Source/IO/Net/SocketStatAverageBps.hpp create mode 100644 Source/IO/Net/SocketStatChannel.hpp create mode 100644 Source/Locale/Encoding/GBK/GBK.hpp create mode 100644 Source/Locale/Encoding/SJIS/SJIS.hpp create mode 100644 Source/Locale/Encoding/UTFn/AuUTF16.hpp create mode 100644 Source/Locale/Encoding/UTFn/AuUTF32.hpp create mode 100644 Source/Locale/Encoding/UTFn/AuUTF8.hpp diff --git a/.gitignore b/.gitignore index 694a112b..acad8877 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,208 @@ +# Aurora's general purpose JS/TS/C/C++/Go vs/vscode/intellij/codelite gitignore reference +# Almost usable for Java and Qt + +# Aurora build configuration Build_CompilerWorkingDirectory/* Build_Developers/* Build_Ship/* Build_Internal/* Build_Develop/* -*.vcxproj -*.vcxproj.filters -*.vcxproj.user +Build_Stage/* +Build_Ship/* +Build_Workspace/* + +# License Headers VS extension *.licenseheader + +# Binaries / object files *.dll *.exe *.obj *.so +*.so.* +*.la +*.lai +*.pdb +*.idb +*.exe~ +*.obj *.dynlib +*.dylib *.lib *.d *.o +*.a +*.la +*.slo +*.lo *.out -.vs +# go unit test +*.test + +# Autogenerated project files compile_flags.txt *.mk *.project *cmake +Makefile +*.vcxproj +*.xcodeproj + +# IDE trash +.vscode +.vs +/*.gcno .intellij .clion -Makefile +*.vcxproj.filters +*.vcxproj.user +*.tlog + +# OSX +.DS_Store +.AppleDouble +.LSOverride +xcuserdata/ + +# Win32 +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.lnk + +# Linux is trash and cant hotswap like NT +.nfs* +.fuse_hidden* + +# Ninja +.ninja_deps +.ninja_log + +# PID locks +*.pid +*.pid.lock + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# JetBrains +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-*/ + +# Android Studio +.idea/caches/build_file_checksums.ser + +# why would we ever ship this dir? +.idea/caches/* + +# NodeJS +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# node-waf configuration +.lock-wscript + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# VS Code Extensions +.vscode-test + +# Qt unit tests +target_wrapper.* + +# QtCreator +*.autosave + +# QtCreator Qml +*.qmlproject.user +*.qmlproject.user.* + +# QtCreator CMake +CMakeLists.txt.user* + +# QtCreator 4.8< compilation database +compile_commands.json + +# QtCreator local machine specific files for imported projects +*creator.user* + +*_qmlcache.qrc + +# QT cache and user files +/.qmake.cache +/.qmake.stash +*.pro.user +*.pro.user.* +*.qbs.user +*.qbs.user.* +*.moc + +# Java trash +hs_err_pid* +.gradle +gradle-app.setting +!gradle-wrapper.jar +.gradletasknamecache +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/Aurora.json b/Aurora.json index 3a29c7da..c1ed4a30 100644 --- a/Aurora.json +++ b/Aurora.json @@ -7,8 +7,8 @@ "staticImpDefines": "AURORA_ENGINE_KERNEL_STATIC", "defines": [], "soft-depends": ["wxwidgets", "glm"], - "depends": ["AuroraInterfaces", "mimalloc", "uuid", "fmt", "json", "bzip2", "ltc", "o1heap", "zstd", "zlib", "lz4", "mbedtls"], - "include-depends": ["fmt", "uuid", "AuroraInterfaces"], + "depends": ["AuroraInterfaces", "AuroraEnum", "mimalloc", "uuid", "fmt", "json", "bzip2", "ltc", "o1heap", "zstd", "zlib", "lz4", "mbedtls"], + "include-depends": ["fmt", "uuid", "AuroraInterfaces", "AuroraEnum"], "features": ["guess-platform-code"], "linkSources": "Source/Alloc.cpp", "actions": [ @@ -19,4 +19,4 @@ } } ] -} \ No newline at end of file +} diff --git a/Include/Aurora/Async/Async.hpp b/Include/Aurora/Async/Async.hpp index baf1eb55..6103202d 100644 --- a/Include/Aurora/Async/Async.hpp +++ b/Include/Aurora/Async/Async.hpp @@ -7,305 +7,52 @@ ***/ #pragma once -namespace Aurora::Loop -{ - class ILoopSource; -} +#include "AsyncTypes.hpp" + +#include "IWorkItem.hpp" +#include "IWorkItemHandler.hpp" +#include "IThreadPool.hpp" +#include "IAsyncApp.hpp" + +#include "Jobs.hpp" +#include "Tasks.hpp" + namespace Aurora::Async { - class IWorkItem; - class IAsyncApp; + AUKN_SYM IAsyncApp *GetAsyncApp(); - struct AVoid - { - AuUInt8 x[1]; - }; - - AUKN_SYM IAsyncApp *GetAsyncApp(); - - /// ThreadGroup_t: - /// 0 = system main thread - /// 1+ = user defined - using ThreadGroup_t = AuUInt8; - - /// ThreadId_t: - /// -1 = invalid - /// index = tid/runner id - using ThreadId_t = AuUInt16; - - static const ThreadId_t kThreadIdAny = -1; - - struct WorkerId_t : AuPair - { - WorkerId_t() : AuPair(0, 0) - {} - - WorkerId_t(ThreadGroup_t group) : AuPair(group, kThreadIdAny) - {} - - WorkerId_t(ThreadGroup_t group, ThreadId_t id) : AuPair(group, id) - {} - - WorkerId_t(const WorkerId_t &cpy) : AuPair(cpy.first, cpy.second) - {} - }; - - struct WorkPriv - { - AuUInt32 magic; - }; - - struct IWorkItemHandler - { - enum class EProcessNext - { - eInvalid = -1, - eFinished = 0, - eRerun, - eSchedule, - eFailed - }; - - struct ProcessInfo - { - ProcessInfo(bool finished) : type(finished ? EProcessNext::eFinished : EProcessNext::eFailed) {} - ProcessInfo(EProcessNext type) : type(type) {} - ProcessInfo(const AuList> &blockedBy) : type(EProcessNext::eSchedule), waitFor(blockedBy) {} - // ... - - EProcessNext type; - AuList> waitFor; - AuUInt32 reschedMs; - AuUInt64 reschedNs; - }; - - virtual void DispatchFrame(ProcessInfo &info) = 0; - - /// A really terrible name for the overloadable method that serves as the critical failure callback - /// You have a 'shutdown'/free function you can overload, it's called the dtor - /// Don't moan about the shitty naming of this, im not refactoring it - virtual void Shutdown() = 0; - - virtual void *GetPrivateData() { return nullptr; } - }; - - template - struct FJob - { - std::function onSuccess = 0; - std::function onFailure = 0; - }; - - template - static inline FJob JobFromConsumer(const AuConsumer &onSuccess) - { - FJob ret; - ret.onSuccess = [=](const Info_t &in, const Result_t &a) - { - onSuccess(in, a); - }; - return ret; - } - - template - static inline FJob JobFromConsumer(const AuConsumer &onSuccess, const AuConsumer &onFailure) - - { - FJob ret; - ret.onSuccess = [=](const Info_t &in, const Result_t &a) - { - onSuccess(in, a); - }; - ret.onFailure = [=](const Info_t &a) - { - onFailure(a, true); - }; - return ret; - } - - template - static inline FJob JobFromConsumer(const AuConsumer &onSuccess, const AuConsumer &onFailure) - - { - FJob ret; - ret.onSuccess = [=](const Info_t &in, const Result_t &a) - { - onSuccess(in, a); - }; - ret.onFailure = [=](const Info_t &a) - { - onFailure(a); - }; - return ret; - } - - - template - static inline FJob JobFromDerivedJob(const FJob &reference) - { - FJob ret; - ret.onSuccess = [=](const Info_t &in, const Result_t &a) - { - if (reference.onSuccess) - { - reference.onSuccess(in, a); - } - }; - ret.onFailure = [=](const Info_t &a) - { - if (reference.onFailure) - { - reference.onSuccess(a); - } - }; - return ret; - } - - using FVoidJob = FJob; - - template - struct CJob - { - void(* onSuccess)(const Info_t &, const Result_t &); - void(* onFailure)(const Info_t &); - }; - - template - struct FTask - { - std::function onFrame = 0; - }; - - using FVoidTask = FTask; - - template - FTask TaskFromConsumerRefT(ClazzImpl &&func) - { - FTask ret; - ret.onFrame = [callable = func](const Info_t &in) -> Out_t - { - if constexpr (std::is_same_v) - { - callable(in); - return {}; - } - else - { - return callable(in); - } - }; - return ret; - } - - template - FTask TaskFromVoidVoid(const AuVoidFunc &func) - { - FTask ret; - ret.onFrame = [callable = func](const Info_t &in) -> Out_t - { - callable(); - return {}; - }; - return ret; - } - - template - struct CTask - { - Result_t(* onFrame)(const Info_t &); - }; - - class IWorkItem - { - public: - virtual AuSPtr WaitFor(const AuSPtr &workItem) = 0; - virtual AuSPtr WaitFor(const AuList> &workItem) = 0; - - // ms = time relative to the current time - virtual AuSPtr SetSchedTime(AuUInt32 ms) = 0; - - // ns = time relative to the current time - virtual AuSPtr SetSchedTimeNs(AuUInt64 ns) = 0; - - // ms = time relative to the time at which the work item would otherwise dispatch - virtual AuSPtr AddDelayTime(AuUInt32 ms) = 0; - - // ns = time relative to the time at which the work item would otherwise dispatch - virtual AuSPtr AddDelayTimeNs(AuUInt64 ns) = 0; - - virtual AuSPtr Then(const AuSPtr &next) = 0; - - virtual AuSPtr Dispatch() = 0; - - virtual bool BlockUntilComplete() = 0; - virtual bool HasFinished() = 0; - virtual bool HasFailed() = 0; - virtual void Cancel() = 0; - - virtual void *GetPrivateData() = 0; - virtual AuOptional ToWorkResultT() = 0; - }; + /// + AUKN_SYM WorkerPId_t GetCurrentWorkerPId(); + /// Async app only | Thread pools must use the IThreadPool::NewFence function AUKN_SYM AuSPtr NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking = false); + + AUKN_SYM AuSPtr NewWorkItem(const WorkerPId_t &worker, const AuSPtr &task, bool supportsBlocking = false); + + /// Async app only | Thread pools must use the IThreadPool::NewFence function AUKN_SYM AuSPtr NewFence(); - class IAsyncApp - { - public: - // Main thread logic - virtual void Start() = 0; - virtual void Main() = 0; - virtual void Shutdown() = 0; - virtual bool Exiting() = 0; - virtual void SetConsoleCommandDispatcher(WorkerId_t id) = 0; + /// Allocates a new thread pool for usage + AUKN_SYM AuSPtr NewThreadPool(); - // Spawning - virtual bool Spawn(WorkerId_t workerId) = 0; - - // Event runner threads release upon encountering a zero work condition allowing for a clean exit of event driven apps without the headache of carefully chaining together exit callbacks - // Applications that aren't designed around an event driven model should set callerOwns to true to keep the around during global work exhaustion - virtual void SetWorkerIdIsThreadRunner(WorkerId_t, bool runner) = 0; - - virtual Threading::Threads::ThreadShared_t ResolveHandle(WorkerId_t) = 0; - virtual AuBST> GetThreads() = 0; - virtual WorkerId_t GetCurrentThread() = 0; - - // Synchronization - // Note: syncing to yourself will nullify requireSignal to prevent deadlock - virtual bool Sync(WorkerId_t group, AuUInt32 timeoutMs = 0, bool requireSignal = false) = 0; - virtual void Signal(WorkerId_t group) = 0; - virtual void SyncAllSafe() = 0; - - // Features - virtual void AddFeature(WorkerId_t id, AuSPtr feature, bool async = false) = 0; - - // Debug - virtual void AssertInThreadGroup(ThreadGroup_t thread) = 0; - virtual void AssertWorker(WorkerId_t id) = 0; - - virtual bool Poll(bool block) = 0; - - virtual bool ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) = 0; - }; - - + // TODO: move the following trash out of here #pragma region EASE_OF_READING struct BasicWorkStdFunc : IWorkItemHandler { - std::function callback; - std::function shutdown; // error + AuFunction callback; + AuFunction shutdown; // error - BasicWorkStdFunc(std::function &&callback, std::function &&shutdown) : callback(std::move(callback)), shutdown(std::move(shutdown)) + BasicWorkStdFunc(AuFunction &&callback, AuFunction &&shutdown) : callback(std::move(callback)), shutdown(std::move(shutdown)) {} - BasicWorkStdFunc(std::function &&callback) : callback(std::move(callback)) + BasicWorkStdFunc(AuFunction &&callback) : callback(std::move(callback)) {} - BasicWorkStdFunc(const std::function &callback) : callback(callback) + BasicWorkStdFunc(const AuFunction &callback) : callback(callback) {} - BasicWorkStdFunc(const std::function &callback, const std::function &shutdown) : callback(callback), shutdown(shutdown) + BasicWorkStdFunc(const AuFunction &callback, const AuFunction &shutdown) : callback(callback), shutdown(shutdown) {} private: @@ -315,7 +62,7 @@ namespace Aurora::Async try { callback(); - } + } catch (...) { Debug::PrintError(); @@ -330,7 +77,7 @@ namespace Aurora::Async { shutdown(); } - } + } catch (...) { Debug::PrintError(); @@ -341,214 +88,9 @@ namespace Aurora::Async #if !defined(_CPPSHARP) + /// @hideinitializer - struct BasicWorkCtx : WorkPriv - { - BasicWorkCtx() - { - magic = AuConvertMagicTag32("BWOT"); - opt = nullptr; - } - void *opt; - }; - - /// @hideinitializer - template, typename Job_t = FJob> - struct BasicWorkCallback : IWorkItemHandler, std::enable_shared_from_this - { - BasicWorkCallback() - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(Task_t &&task) : task(std::move(task)) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(Task_t &&task, Job_t &&callback) : task(std::move(task)), callback(std::move(callback)) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(const Task_t &task) : task(task) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(const Task_t &task, const Job_t &callback) : task(task), callback(callback) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(const Task_t &task, const Job_t &callback, const Info_t &info) : task(task), callback(callback), input(info) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(Task_t &&task, const Job_t &callback, const Info_t &info) : task(std::move(task)), callback(callback), input(info) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(Task_t &&task, Job_t &&callback, const Info_t &info) : task(std::move(task)), callback(std::move(callback)), input(info) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(Task_t &&task, Job_t &&callback, Info_t &&info) : task(std::move(task)), callback(std::move(callback)), input(std::move(info)) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - BasicWorkCallback(const Task_t &task, const Job_t &callback, Info_t &&info) : task(task), callback(callback), input(info) - { - caller = GetAsyncApp()->GetCurrentThread(); - } - - Info_t input; - Task_t task; - Job_t callback; - - - BasicWorkCallback &SetTask(const Task_t &task) - { - this->task = task; - return *this; - } - - BasicWorkCallback &SetTask(const Job_t &callback) - { - this->callback = callback; - return *this; - } - - private: - - static constexpr bool IsCallbackPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; - static constexpr bool IsTaskPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; - - WorkerId_t caller; - - BasicWorkCtx secretContext_; - Result_t resultValue_; - - virtual void *GetPrivateData() override { return &secretContext_; } - - void DispatchFrame(ProcessInfo &info) override - { - try - { - if constexpr (IsTaskPtr) - { - resultValue_ = task->onFrame(input); - } - else - { - resultValue_ = task.onFrame(input); - } - } - catch (...) - { - Debug::PrintError(); - Shutdown(); - return; - } - - auto pin = std::static_pointer_cast>(this->shared_from_this()); - - std::function func = [pin]() - { - try - { - pin->secretContext_.opt = &pin->resultValue_; - if constexpr (IsCallbackPtr) - { - pin->callback->onSuccess(pin->input, pin->resultValue_); - } - else - { - if (pin->callback.onSuccess) - { - pin->callback.onSuccess(pin->input,pin->resultValue_); - } - } - } - catch (...) - { - Debug::PrintError(); - } - }; - - try - { - if (caller == GetAsyncApp()->GetCurrentThread()) - { - func(); - } - else - { - std::function err = [pin]() - { - pin->CallOnFailure(); - }; - - // TODO: this is somewhat evil. double alloc when we could reuse this - if (!NewWorkItem(caller, AuMakeShared(func, err))->Dispatch()) - { - pin->CallOnFailure(); - } - } - } - catch (...) - { - Debug::PrintError(); - Shutdown(); - } - } - - void Shutdown() override - { - try - { - CallOnFailure(); - } - catch (...) - { - Debug::PrintError(); - } - } - - void CallOnFailure() - { - if constexpr (IsCallbackPtr) - { - if constexpr (AuIsBaseOfTemplateonFailure)>::value) - { - if (!callback->onFailure) - { - return; - } - } - - callback->onFailure(input); - } - else - { - if constexpr (AuIsBaseOfTemplate::value) - { - if (!callback.onFailure) - { - return; - } - } - callback.onFailure(input); - } - } - }; - - /// @hideinitializer - template, typename Cleanup_t = std::function> + template, typename Cleanup_t = AuFunction> struct WorkItemCallable : IWorkItemHandler { Frame_t frame; @@ -581,12 +123,13 @@ namespace Aurora::Async cleanup(); } }; - + + #define ASYNC_ERROR(exp) { if constexpr (std::is_same_v) { SysPushErrorGen(exp); return {}; } else { throw std::string(exp); } } #define ASYNC_FINISH { if constexpr (std::is_same_v) { return true; } } template || std::is_void::value)> - static std::function TranslateAsyncFunctionToDispatcherWithThread(WorkerId_t id, std::function func) + static AuFunction TranslateAsyncFunctionToDispatcherWithThread(WorkerId_t id, AuFunction func) { if (!func) return {}; return [=](Args&&... in) -> T @@ -594,26 +137,28 @@ namespace Aurora::Async auto work = AuMakeShared([=]() -> void { func(in...); }); - if (!work) ASYNC_ERROR("can't dispatch async call; out of memory"); + if (!work) ASYNC_ERROR("can't dispatch async call. out of memory"); auto workItem = NewWorkItem(id, work); - if (!workItem) ASYNC_ERROR("can't dispatch async call; out of memory"); + if (!workItem) ASYNC_ERROR("can't dispatch async call. out of memory"); workItem->Dispatch(); ASYNC_FINISH; }; } + /// Async app only template || std::is_void::value)> - static std::function TranslateAsyncFunctionToDispatcher(std::function func) + static AuFunction TranslateAsyncFunctionToDispatcher(AuFunction func) { return TranslateAsyncFunctionToDispatcherWithThread(GetAsyncApp()->GetCurrentThread(), func); } + /// Async app only template || std::is_void::value)> - static std::function, Args...)> TranslateAsyncReturnableFunctionToDispatcherWithThread(WorkerId_t id, std::function func) + static AuFunction, Args...)> TranslateAsyncReturnableFunctionToDispatcherWithThread(WorkerId_t id, AuFunction func) { - return [=](std::function callback, Args... in) -> T + return [=](AuFunction callback, Args... in) -> T { - auto work = AuMakeShared>(); + auto work = AuMakeShared>(); if (!work) ASYNC_ERROR("can't dispatch async call; out of memory"); work.task.onProcess = [=](const AVoid &) -> B { @@ -631,69 +176,6 @@ namespace Aurora::Async }; } - template, typename Job_t = FJob> - static AuSPtr NewBasicWorkCallback(const WorkerId_t &worker, const Task_t &task, const Job_t &job, bool enableWait = false) - { - return NewWorkItem(worker, AuMakeShared>(std::move(task), job), enableWait); - } - - template, typename Job_t = FJob> - static AuSPtr DispatchBasicWorkCallback(const WorkerId_t &worker, const Task_t &task, const Job_t &job, bool enableWait = false) - { - return NewBasicWorkCallback(worker, task, job, enableWait)->Dispatch(); - } - - template, typename Job_t = FJob> - static AuSPtr NewBasicWorkCallback(const WorkerId_t &worker, Task_t &&task, const Job_t &job, bool enableWait = false) - { - return NewWorkItem(worker, AuMakeShared>(std::move(task), job), enableWait); - } - - template, typename Job_t = FJob> - static AuSPtr DispatchBasicWorkCallback(const WorkerId_t &worker, Task_t &&task, const Job_t &job, bool enableWait = false) - { - return NewBasicWorkCallback(worker, std::move(task), job, enableWait)->Dispatch(); - } - - template, typename Job_t = FJob> - static AuSPtr NewBasicWorkCallback(const WorkerId_t &worker, Task_t &&task, Job_t &&job, bool enableWait = false) - { - return NewWorkItem(worker, AuMakeShared>(std::move(task), std::move(job)), enableWait); - } - - template, typename Job_t = FJob> - static AuSPtr DispatchBasicWorkCallback(const WorkerId_t &worker, Task_t &&task, Job_t &&job, bool enableWait = false) - { - return NewBasicWorkCallback(worker, std::move(task), std::move(job), enableWait)->Dispatch(); - } - - template, typename Job_t = FJob> - static AuSPtr DispatchBasicWorkCallback(const WorkerId_t &worker, const Task_t &task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) - { - // TOOD: use faster object if job parguments are invalid - // It would be nice if we didn't have to drag the job callback pair around with us - return NewWorkItem(worker, AuMakeShared>(task, job, inputParameters), enableWait)->Dispatch(); - } - - template, typename Job_t = FJob, typename ClazzImpl> - AuSPtr DispatchFunctional(const WorkerId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) - { - return DispatchBasicWorkCallback(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); - } - - template, typename Job_t = FJob, typename ClazzImpl> - AuSPtr DispatchFunctor(const WorkerId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) - { - return DispatchBasicWorkCallback(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); - } - - template, typename Job_t = FJob, typename ClazzImpl> - AuSPtr DispatchVoid(const WorkerId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) - { - return DispatchBasicWorkCallback(worker, TaskFromVoidVoid(task), job, inputParameters, enableWait); - } - - #undef ASYNC_ERROR #undef ASYNC_FINISH @@ -701,3 +183,11 @@ namespace Aurora::Async #pragma endregion EASE_OF_READING } + + +#if !defined(_CPPSHARP) +#include "JobFrom.hpp" +#include "TaskFrom.hpp" +#include "WorkPairImpl.hpp" +#include "WorkBasic.hpp" +#endif \ No newline at end of file diff --git a/Include/Aurora/Async/AsyncTypes.hpp b/Include/Aurora/Async/AsyncTypes.hpp new file mode 100644 index 00000000..33c417c4 --- /dev/null +++ b/Include/Aurora/Async/AsyncTypes.hpp @@ -0,0 +1,75 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: AsyncTypes.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Loop +{ + class ILoopSource; +} + +namespace Aurora::Async +{ + struct IWorkItem; + struct IAsyncApp; + struct IThreadPool; + + struct AVoid + { + AuUInt8 x[1]; + }; + + /// IAsyncApp: + /// ThreadGroup_t: + /// 0 = system main thread + /// 1+ = user defined | user allocated thread + using ThreadGroup_t = AuUInt8; + + /// ThreadId_t: + /// -1 = invalid or any + /// index = tid/runner id + using ThreadId_t = AuUInt16; + + static const ThreadId_t kThreadIdAny = -1; + + struct WorkerId_t : AuPair + { + WorkerId_t() : AuPair(0, 0) + {} + + WorkerId_t(ThreadGroup_t group) : AuPair(group, kThreadIdAny) + {} + + WorkerId_t(ThreadGroup_t group, ThreadId_t id) : AuPair(group, id) + {} + + WorkerId_t(const WorkerId_t &cpy) : AuPair(cpy.first, cpy.second) + {} + }; + + struct WorkPriv + { + AuUInt32 magic; + }; + + struct WorkerPId_t : WorkerId_t + { + WorkerPId_t() + {} + + WorkerPId_t(const AuSPtr &pool, ThreadGroup_t group) : WorkerId_t(group, kThreadIdAny), pool(pool) + {} + + WorkerPId_t(const AuSPtr &pool, ThreadGroup_t group, ThreadId_t id) : WorkerId_t(group, id), pool(pool) + {} + + WorkerPId_t(const AuSPtr &pool, const WorkerId_t &cpy) : WorkerId_t(cpy.first, cpy.second), pool(pool) + {} + + AuSPtr pool; + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/IAsyncApp.hpp b/Include/Aurora/Async/IAsyncApp.hpp new file mode 100644 index 00000000..85922238 --- /dev/null +++ b/Include/Aurora/Async/IAsyncApp.hpp @@ -0,0 +1,25 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IAsyncApp.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + struct IAsyncApp : IThreadPool + { + // Main thread logic + + /// Initializes the async application before the hand-off + virtual void Start() = 0; + + /// Hand off + virtual void Main() = 0; + + /// Console config + virtual void SetConsoleCommandDispatcher(WorkerId_t id) = 0; + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/IThreadPool.hpp b/Include/Aurora/Async/IThreadPool.hpp new file mode 100644 index 00000000..0b160980 --- /dev/null +++ b/Include/Aurora/Async/IThreadPool.hpp @@ -0,0 +1,70 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IThreadPool.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Async +{ + struct IThreadPool + { + // Spawning + /// @deprecated + virtual bool Spawn(WorkerId_t workerId) = 0; + + // Event runner threads release the entire application upon encountering a zero work condition allowing for a clean exit of event driven apps without the headache of carefully chaining together exit callbacks + // Applications that aren't designed around an event driven model should set runner to false to keep the around during global work exhaustion + // tl'dr: runner = true, async app; runner = false, thread pool + virtual void SetRunningMode(bool eventRunning) = 0; + + // Converts the current thread into an async thread with runner = false + // Use with RunOnce, Poll + virtual bool Create(WorkerId_t workerId) = 0; + + //virtual bool CreateAndRun(WorkerId_t workerId) = 0; + + // @returns any threads with runner = true + virtual bool InRunnerMode() = 0; + + // Manual execution (system thread and ::Create() users) + virtual bool Poll() = 0; + virtual bool RunOnce() = 0; + virtual bool Run() = 0; + + // + virtual void Shutdown() = 0; + virtual bool Exiting() = 0; + + // + virtual AuSPtr NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking = false) = 0; + virtual AuSPtr NewFence() = 0; + + // + virtual Threading::Threads::ThreadShared_t ResolveHandle(WorkerId_t) = 0; + + // + virtual AuBST> GetThreads() = 0; + + // + virtual WorkerId_t GetCurrentThread() = 0; + + // Synchronization + // Note: syncing to yourself will nullify requireSignal to prevent deadlock + virtual bool Sync(WorkerId_t workerId, AuUInt32 timeoutMs = 0, bool requireSignal = false) = 0; + virtual void Signal(WorkerId_t workerId) = 0; + virtual void SyncAllSafe() = 0; + + // Features + virtual void AddFeature(WorkerId_t id, AuSPtr feature, bool async = false) = 0; + + // Debug + virtual void AssertInThreadGroup(ThreadGroup_t group) = 0; + virtual void AssertWorker(WorkerId_t id) = 0; + + // Async subsystem glue + virtual bool ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) = 0; + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/IWorkItem.hpp b/Include/Aurora/Async/IWorkItem.hpp new file mode 100644 index 00000000..98decb95 --- /dev/null +++ b/Include/Aurora/Async/IWorkItem.hpp @@ -0,0 +1,54 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IWorkItem.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + struct IWorkItem + { + virtual AuSPtr WaitFor(const AuSPtr &workItem) = 0; + virtual AuSPtr WaitFor(const AuList> &workItem) = 0; + + // ms = time relative to the current time + virtual AuSPtr SetSchedTime(AuUInt32 ms) = 0; + + // ns = Aurora::Time::CurrentClockMS() + relativeMs + virtual AuSPtr SetSchedTimeAbs(AuUInt32 ms) = 0; + + // ns = time relative to the current time + virtual AuSPtr SetSchedTimeNs(AuUInt64 ns) = 0; + + // ns = Aurora::Time::CurrentClockNS() + relativeNs + virtual AuSPtr SetSchedTimeNsAbs(AuUInt64 ns) = 0; + + // ms = time relative to the time at which the work item would otherwise dispatch + virtual AuSPtr AddDelayTime(AuUInt32 ms) = 0; + + // ns = time relative to the time at which the work item would otherwise dispatch + virtual AuSPtr AddDelayTimeNs(AuUInt64 ns) = 0; + + virtual AuSPtr Then(const AuSPtr &next) = 0; + + virtual AuSPtr Dispatch() = 0; + + // per work frame, available work is dequed and sorted. + // work is sorted by prio and then by initial submission index. + // + // prios under .25 will yield for new work at time of possible dispatch + // only if there are work entries of > .5 in the pending work queue. + virtual void SetPrio(float val = 0.5f) = 0; + + virtual bool BlockUntilComplete() = 0; + virtual bool HasFinished() = 0; + virtual bool HasFailed() = 0; + virtual void Cancel() = 0; + + virtual void *GetPrivateData() = 0; + virtual AuOptional ToWorkResultT() = 0; + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/IWorkItemHandler.hpp b/Include/Aurora/Async/IWorkItemHandler.hpp new file mode 100644 index 00000000..0fa410b6 --- /dev/null +++ b/Include/Aurora/Async/IWorkItemHandler.hpp @@ -0,0 +1,47 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IWorkItemHandler.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + struct IWorkItemHandler + { + enum class EProcessNext + { + eInvalid = -1, + eFinished = 0, + eRerun, + eSchedule, + eFailed + }; + + struct ProcessInfo + { + ProcessInfo(bool finished) : type(finished ? EProcessNext::eFinished : EProcessNext::eFailed) {} + ProcessInfo(EProcessNext type) : type(type) {} + ProcessInfo(const AuList> &blockedBy) : type(EProcessNext::eSchedule), waitFor(blockedBy) {} + // ... + + EProcessNext type; + AuList> waitFor; + AuUInt32 reschedMs; + AuUInt64 reschedNs; + // @hideinitializer + IThreadPool *pool; + }; + + virtual void DispatchFrame(ProcessInfo &info) = 0; + + /// A really terrible name for the overloadable method that serves as the critical failure callback + /// You have a 'shutdown'/free function you can overload, it's called the dtor + /// Don't moan about the shitty naming of this, im not refactoring it + virtual void Shutdown() = 0; + + virtual void *GetPrivateData() { return nullptr; } + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/JobFrom.hpp b/Include/Aurora/Async/JobFrom.hpp new file mode 100644 index 00000000..4a4f9682 --- /dev/null +++ b/Include/Aurora/Async/JobFrom.hpp @@ -0,0 +1,186 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: JobFrom.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + template + static inline FJob JobFromPairConsumer(const /*AuConsumer */ Callable_t&onSuccess) + { + FJob ret; + ret.onSuccess = [=](const Info_t &in, const Result_t &a) + { + onSuccess(in, a); + }; + return ret; + } + + template + static inline FJob JobFromResultConsumer(/*AuConsumer */ Callable_t&&onSuccess) + { + FJob ret; + ret.onSuccess = [=](const Info_t &in, const Result_t &a) + { + onSuccess(a); + }; + return ret; + } + + template + FJob, Out_t> JobFromTupleConsumer(/*AuConsumer */ Callable_t &&onSuccess) + { + FJob, Out_t> ret; + static_assert(std::is_same_v, std::tuple>); + ret.onSuccess = [=](const std::tuple &in, const Out_t &a) + { + std::apply(onSuccess, std::tuple_cat(in, std::make_tuple(a))); + }; + return ret; + } + + template + FJob, Out_t> JobFromTupleConsumerEx(/*AuConsumer */ Callable_t &&onSuccess, FailureCallable_t &&onFailure) + { + FJob, Out_t> ret; + static_assert(std::is_same_v, std::tuple>); + ret.onSuccess = [=](const std::tuple &in, const Out_t &a) + { + std::apply(onSuccess, std::tuple_cat(in, std::make_tuple(a))); + }; + + ret.onFailure = [=](const std::tuple &in) + { + std::apply(onFailure, in); + }; + return ret; + } + + template + static inline FJob JobFromPairConsumerEx(const AuConsumer &onSuccess, const AuConsumer &onFailure) + { + FJob ret; + ret.onSuccess = [=](const Info_t &in, const Result_t &a) + { + onSuccess(in, a); + }; + ret.onFailure = [=](const Info_t &a) + { + onFailure(a); + }; + return ret; + } + + template + static inline FJob JobFromConsumer(const AuConsumer &onSuccess, const AuVoidFunc &onFailure) + { + FJob ret; + ret.onSuccess = [=](const Info_t &in, const Result_t &a) + { + onSuccess(a); + }; + ret.onFailure = [=](const Info_t &a) + { + onFailure(); + }; + return ret; + } + + template + static inline FJob JobFromDerivedJob(const FJob &reference) + { + FJob ret; + ret.onSuccess = [=](const Info_t &in, const Result_t &a) + { + if (reference.onSuccess) + { + reference.onSuccess(in, a); + } + }; + ret.onFailure = [=](const Info_t &a) + { + if (reference.onFailure) + { + reference.onFailure(a); + } + }; + return ret; + } + + template + static inline FJob, ReturnValue_t> JobFromTupleClazz(const AuConsumer &onSuccess) + { + FJob, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple &in, const ReturnValue_t &out) + { + onSuccess(std::get<0>(in), out); + }; + return ret; + } + + template + static inline FJob, ReturnValue_t> JobFromTupleClazzEx(const AuConsumer &onSuccess, const AuConsumer &onFailure) + { + FJob, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple &in, const ReturnValue_t &out) + { + onSuccess(std::get<0>(in), out); + }; + + ret.onFailure = [=](const std::tuple &in) + { + onFailure(std::get<0>(in)); + }; + return ret; + } + +#if 0 + template + static inline FJob, Args...>, ReturnValue_t> JobFromTupleClazz(const AuConsumer &onSuccess) + { + FJob, Args...>, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple, Args...> &in, const ReturnValue_t &out) + { + onSuccess(out); + }; + return ret; + } + + template + static inline FJob, Args...>, ReturnValue_t> JobFromTupleClazz(const AuConsumer &onSuccess) + { + FJob, Args...>, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple, Args...> &in, const ReturnValue_t &out) + { + std::apply(onSuccess, std::tuple_cat(AuTuplePopFront(in), std::make_tuple(out))); + }; + return ret; + } +#endif + + template + static inline FJob, ReturnValue_t> JobFromTupleResultConsumer(const AuConsumer &onSuccess) + { + FJob, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple &in, const ReturnValue_t &out) + { + onSuccess(out); + }; + return ret; + } + + template + static inline FJob, ReturnValue_t> JobFromTuple(const AuConsumer &onSuccess) + { + FJob, ReturnValue_t> ret; + ret.onSuccess = [=](const std::tuple &in, const ReturnValue_t &out) + { + std::apply(onSuccess, std::tuple_cat(in, std::make_tuple(out))); + }; + return ret; + } +} \ No newline at end of file diff --git a/Include/Aurora/Async/Jobs.hpp b/Include/Aurora/Async/Jobs.hpp new file mode 100644 index 00000000..49704b0d --- /dev/null +++ b/Include/Aurora/Async/Jobs.hpp @@ -0,0 +1,33 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: Jobs.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + template + struct FJob + { + using InfoType_t = Info_t; + using ResultType_t = Result_t; + + AuFunction onSuccess; + AuFunction onFailure; + }; + + using FVoidJob = FJob; + + template + struct CJob + { + using InfoType_t = Info_t; + using ResultType_t = Result_t; + + void(* onSuccess)(const Info_t &, const Result_t &); + void(* onFailure)(const Info_t &); + }; +} \ No newline at end of file diff --git a/Include/Aurora/Async/TaskFrom.hpp b/Include/Aurora/Async/TaskFrom.hpp new file mode 100644 index 00000000..a05a311f --- /dev/null +++ b/Include/Aurora/Async/TaskFrom.hpp @@ -0,0 +1,87 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: TaskFrom.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + template + FTask TaskFromConsumerRefT(ClazzImpl &&func) + { + FTask ret; + ret.onFrame = [callable = func](const Info_t &in) -> Out_t + { + if constexpr (std::is_same_v) + { + callable(in); + return {}; + } + else + { + return callable(in); + } + }; + return ret; + } + + template + FTask, Out_t> TaskFromTupleCallable(Functor &&func) + { + FTask, Out_t> ret; + static_assert(std::is_same_v, std::tuple>); + ret.onFrame = [callable = func](const std::tuple &in) -> Out_t + { + return std::apply(callable, in); + }; + return ret; + } + + template + FTask, Out_t> TaskFromTupleCallableWithOwnerArg(AuFunction &&func, const Owner_t &ownerToPin) + { + FTask, Out_t> ret; + ret.onFrame = [callable = func](const std::tuple &in) -> Out_t + { + return std::apply(callable, std::tuple_cat(std::make_tuple(ownerToPin), in)); + }; + return ret; + } + + template + Task_t TaskFromTupleCallableWithBindOwner(Functor &&func) + { + Task_t ret; + ret.onFrame = [callable = func](const auto &in) -> ReturnValue_t + { + return std::apply(callable, AuTuplePopFront(in)); + }; + return ret; + } + + template + Task_t TaskFromTupleCallableWithBindOwner2(Functor func) + { + Task_t ret; + ret.onFrame = [callable = func](const auto &in) -> ReturnValue_t + { + return std::apply(callable, AuTuplePopFront(in)); + }; + return ret; + } + + template + FTask TaskFromVoidVoid(const AuVoidFunc &func) + { + FTask ret; + ret.onFrame = [callable = func](const Info_t &in) -> Out_t + { + callable(); + return {}; + }; + return ret; + } +} \ No newline at end of file diff --git a/Include/Aurora/Async/Tasks.hpp b/Include/Aurora/Async/Tasks.hpp new file mode 100644 index 00000000..61ce5b42 --- /dev/null +++ b/Include/Aurora/Async/Tasks.hpp @@ -0,0 +1,32 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: Tasks.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + template + struct FTask + { + using InfoType_t = Info_t; + using ResultType_t = Result_t; + + AuFunction onFrame; + }; + + + template + struct CTask + { + using InfoType_t = Info_t; + using ResultType_t = Result_t; + + Result_t(* onFrame)(const Info_t &) = 0; + }; + + using FVoidTask = FTask; +} \ No newline at end of file diff --git a/Include/Aurora/Async/WorkBasic.hpp b/Include/Aurora/Async/WorkBasic.hpp new file mode 100644 index 00000000..6157f834 --- /dev/null +++ b/Include/Aurora/Async/WorkBasic.hpp @@ -0,0 +1,118 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: WorkBasic.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Async +{ + + /// --- THREAD POOL -- + + template, typename Job_t = FJob> + static AuSPtr NewWork(const WorkerPId_t &worker, const Task_t &task, const Job_t &job, bool enableWait = false) + { + return worker.pool->NewWorkItem(worker, AuMakeShared>(std::move(task), job), enableWait); + } + + template, typename Job_t = FJob> + static AuSPtr DispatchWork(const WorkerPId_t &worker, const Task_t &task, const Job_t &job, bool enableWait = false) + { + return NewWorkPairImpl(worker, task, job, enableWait)->Dispatch(); + } + + template, typename Job_t = FJob> + static AuSPtr NewWork(const WorkerPId_t &worker, Task_t &&task, const Job_t &job, bool enableWait = false) + { + return worker.pool->NewWorkItem(worker, AuMakeShared>(std::move(task), job), enableWait); + } + + template, typename Job_t = FJob> + static AuSPtr DispatchWork(const WorkerPId_t &worker, Task_t &&task, const Job_t &job, bool enableWait = false) + { + return NewWorkPairImpl(worker, std::move(task), job, enableWait)->Dispatch(); + } + + template, typename Job_t = FJob> + static AuSPtr NewWork(const WorkerPId_t &worker, Task_t &&task, Job_t &&job, bool enableWait = false) + { + return worker.pool->NewWorkItem(worker, AuMakeShared>(std::move(task), std::move(job)), enableWait); + } + + template, typename Job_t = FJob> + static AuSPtr DispatchWork(const WorkerPId_t &worker, Task_t &&task, Job_t &&job, bool enableWait = false) + { + return NewWorkPairImpl(worker, std::move(task), std::move(job), enableWait)->Dispatch(); + } + + template, typename Job_t = FJob> + static AuSPtr DispatchWork(const WorkerPId_t &worker, const Task_t &task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) + { + return worker.pool->NewWorkItem(worker, AuMakeShared>(task, job, inputParameters), enableWait)->Dispatch(); + } + + template, typename Job_t = FJob, typename ClazzImpl> + AuSPtr DispatchFunctional(const WorkerPId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) + { + return DispatchWorkPairImpl(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); + } + + template, typename Job_t = FJob, typename ClazzImpl> + AuSPtr DispatchFunctor(const WorkerPId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) + { + return DispatchWorkPairImpl(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); + } + + template, typename Job_t = FJob, typename ClazzImpl> + AuSPtr DispatchVoid(const WorkerPId_t &worker, ClazzImpl task, const Job_t &job, const Info_t &inputParameters, bool enableWait = false) + { + return DispatchWorkPairImpl(worker, TaskFromVoidVoid(task), job, inputParameters, enableWait); + } + + /// --- ASYNC APP -- + + template, typename Job_t = FJob, class a = const Job_t &, class b = Task_t &> + static AuSPtr NewWork(const WorkerId_t &worker, a task, b job, bool enableWait = false) + { + return NewWorkItem(worker, AuMakeShared>(task, job), enableWait); + } + + template, typename Job_t = FJob, class a = const Job_t &, class b = Task_t &> + static AuSPtr DispatchWork(const WorkerId_t &worker, a task, b job, bool enableWait = false) + { + return NewWork(worker, task, job, enableWait)->Dispatch(); + } + + template, typename Job_t = FJob, class a, class b, class c> + static AuSPtr NewWork(const WorkerId_t &worker, a task, b job, c info, bool enableWait = false) + { + return NewWorkItem(worker, AuMakeShared>(task, job, info), enableWait); + } + + template, typename Job_t = FJob, class a, class b, class c> + static AuSPtr DispatchWork(const WorkerId_t &worker, a task, b job, c info, bool enableWait = false) + { + return NewWork(worker, task, job, info, enableWait)->Dispatch(); + } + + template, typename Job_t = FJob, typename ClazzImpl, class a = const Job_t &, class b = const Info_t &> + AuSPtr DispatchFunctional(const WorkerId_t &worker, ClazzImpl task, a job, b inputParameters, bool enableWait = false) + { + return NewWork(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); + } + + template, typename Job_t = FJob, typename ClazzImpl, class a = const Job_t &, class b = const Info_t &> + AuSPtr DispatchFunctor(const WorkerId_t &worker, ClazzImpl task, a job, b inputParameters, bool enableWait = false) + { + return NewWork(worker, TaskFromConsumerRefT(task), job, inputParameters, enableWait); + } + + template, typename Job_t = FJob, typename ClazzImpl, class a = const Job_t &, class b = const Info_t &> + AuSPtr DispatchVoid(const WorkerId_t &worker, ClazzImpl task, a job, b inputParameters, bool enableWait = false) + { + return NewWork(worker, TaskFromVoidVoid(task), job, inputParameters, enableWait); + } +} \ No newline at end of file diff --git a/Include/Aurora/Async/WorkPairImpl.hpp b/Include/Aurora/Async/WorkPairImpl.hpp new file mode 100644 index 00000000..cc7361bf --- /dev/null +++ b/Include/Aurora/Async/WorkPairImpl.hpp @@ -0,0 +1,372 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: WorkPairImpl.hpp + Date: 2021-11-1 + Author: Reece +***/ + #pragma once + +namespace Aurora::Threading::Primitives +{ + class SpinLoop; +} + +namespace Aurora::Async +{ + struct BasicWorkStdFunc; + + /// @hideinitializer + struct BasicWorkCtx : WorkPriv + { + BasicWorkCtx() + { + magic = AuConvertMagicTag32("BWOT"); + opt = nullptr; + } + void *opt; + }; + +#if 0 + /// @hideinitializer + template, typename Job_t = FJob> + struct WorkPairImpl : IWorkItemHandler, std::enable_shared_from_this + { + + Info_t input; + Task_t task; + Job_t callback; + + + + private: + + static constexpr bool IsCallbackPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; + static constexpr bool IsTaskPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; + + WorkerPId_t caller_; + + BasicWorkCtx secretContext_; + Result_t resultValue_; + Threading::Primitives::SpinLock lock_; + + virtual void *GetPrivateData() override { return &secretContext_; } + + void DispatchFrame(ProcessInfo &info) override + { + + try + { + if constexpr (IsTaskPtr) + { + resultValue_ = task->onFrame(input); + } + else + { + resultValue_ = task.onFrame(input); + } + } + catch (...) + { + Debug::PrintError(); + Shutdown(); + return; + } + + auto pin = std::static_pointer_cast>(this->shared_from_this()); + + AuFunction func = [pin]() + { + try + { + pin->secretContext_.opt = &pin->resultValue_; + if constexpr (IsCallbackPtr) + { + pin->callback->onSuccess(pin->input, pin->resultValue_); + } + else + { + if (pin->callback.onSuccess) + { + pin->callback.onSuccess(pin->input,pin->resultValue_); + } + } + } + catch (...) + { + Debug::PrintError(); + } + }; + + try + { + + if (caller_ == Async::GetCurrentWorkerPId()) + { + func(); + } + else + { + AuFunction err = [pin]() + { + pin->CallOnFailure(); + }; + + // TODO: this is somewhat evil. double alloc when we could reuse this + if (!caller_.pool->NewWorkItem(caller_, AuMakeShared(func, err))->Dispatch()) + { + pin->CallOnFailure(); + } + } + + caller_ = {}; + } + catch (...) + { + Debug::PrintError(); + Shutdown(); + } + } + + void Shutdown() override + { + AU_LOCK_GUARD(this->lock_); + ShutdownNoLock(); + } + + inline void ShutdownNoLock() + { + caller_ = {}; + try + { + CallOnFailure(); + } + catch (...) + { + Debug::PrintError(); + } + } + + inline void CallOnFailure() + { + if constexpr (IsCallbackPtr) + { + if constexpr (AuIsBaseOfTemplateonFailure)>::value) + { + if (!callback->onFailure) + { + return; + } + } + + callback->onFailure(input); + } + else + { + if constexpr (AuIsBaseOfTemplate::value) + { + if (!callback.onFailure) + { + return; + } + } + callback.onFailure(input); + } + } + }; +#else + template, typename Job_t = FJob> + struct WorkPairImpl : IWorkItemHandler, std::enable_shared_from_this + { + WorkPairImpl() : caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(Task_t &&task) : task(std::move(task)), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(Task_t &&task, Job_t &&callback) : task(std::move(task)), callback(std::move(callback)), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(const Task_t &task) : task(task), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(const Task_t &task, const Job_t &callback) : task(task), callback(callback), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(const Task_t &task, const Job_t &callback, const Info_t &info) : task(task), callback(callback), input(info), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(Task_t &&task, const Job_t &callback, const Info_t &info) : task(std::move(task)), callback(callback), input(info), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(Task_t &&task, Job_t &&callback, const Info_t &info) : task(std::move(task)), callback(std::move(callback)), input(info), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(Task_t &&task, Job_t &&callback, Info_t &&info) : task(std::move(task)), callback(std::move(callback)), input(std::move(info)), caller_(Async::GetCurrentWorkerPId()) + {} + + WorkPairImpl(const Task_t &task, const Job_t &callback, Info_t &&info) : task(task), callback(callback), input(info), caller_(Async::GetCurrentWorkerPId()) + {} + + Info_t input; + Task_t task; + Job_t callback; + + + WorkPairImpl &SetTask(const Task_t &task) + { + this->task = task; + return *this; + } + + WorkPairImpl &SetTask(const Job_t &callback) + { + this->callback = callback; + return *this; + } + + private: + + static constexpr bool IsCallbackPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; + static constexpr bool IsTaskPtr = std::is_pointer_v || AuIsBaseOfTemplate::value; + + //WorkerId_t caller; + WorkerPId_t caller_; + Threading::Primitives::SpinLock lock_; + + BasicWorkCtx secretContext_; + Result_t resultValue_; + + virtual void *GetPrivateData() override { return &secretContext_; } + + void DispatchFrame(ProcessInfo &info) override + { + AU_LOCK_GUARD(this->lock_); + + try + { + if constexpr (IsTaskPtr) + { + resultValue_ = task->onFrame(input); + } + else + { + resultValue_ = task.onFrame(input); + } + + task = {}; + } + catch (...) + { + Debug::PrintError(); + Shutdown(); + return; + } + + auto pin = AuSharedFromThis(); + + std::function func = [pin]() + { + try + { + pin->secretContext_.opt = &pin->resultValue_; + if constexpr (IsCallbackPtr) + { + pin->callback->onSuccess(pin->input, pin->resultValue_); + } + else + { + if (pin->callback.onSuccess) + { + pin->callback.onSuccess(pin->input,pin->resultValue_); + } + } + } + catch (...) + { + Debug::PrintError(); + } + + pin->callback = {}; + pin->caller_ = {}; + }; + + try + { + if (caller_ == Async::GetCurrentWorkerPId()) + { + func(); + } + else + { + AuFunction err = [pin]() + { + pin->CallOnFailure(); + }; + + // TODO: this is somewhat evil. double alloc when we could reuse this + if (!caller_.pool->NewWorkItem(caller_, AuMakeShared(func, err))->Dispatch()) + { + pin->CallOnFailure(); + } + } + + caller_ = {}; + } + catch (...) + { + Debug::PrintError(); + Shutdown(); + } + } + + void Shutdown() override + { + AU_LOCK_GUARD(this->lock_); + ShutdownNoLock(); + } + + inline void ShutdownNoLock() + { + caller_ = {}; + try + { + CallOnFailure(); + } + catch (...) + { + Debug::PrintError(); + } + callback = {}; + task = {}; + } + + inline void CallOnFailure() + { + if constexpr (IsCallbackPtr) + { + if constexpr (AuIsBaseOfTemplateonFailure)>::value) + { + if (!callback->onFailure) + { + return; + } + } + + callback->onFailure(input); + } + else + { + if constexpr (AuIsBaseOfTemplate::value) + { + if (!callback.onFailure) + { + return; + } + } + callback.onFailure(input); + } + } + }; +#endif +} \ No newline at end of file diff --git a/Include/Aurora/Console/Commands/Commands.hpp b/Include/Aurora/Console/Commands/Commands.hpp index ebf03c58..99694e79 100644 --- a/Include/Aurora/Console/Commands/Commands.hpp +++ b/Include/Aurora/Console/Commands/Commands.hpp @@ -12,10 +12,10 @@ namespace Aurora::Async { struct WorkerId_t; + struct WorkerPId_t; } #include "ICommandSubscriber.hpp" -#include "ITextLineSubscriber.hpp" namespace Aurora::Console::Commands { @@ -30,20 +30,15 @@ namespace Aurora::Console::Commands * Parses `string` and dispatches the parsed command on the current thread instantly */ AUKN_SYM bool DispatchCommandThisThread(const AuString &string); - + /** * Parses `string` on the current thread and then schedules the ICommandSubscriber callback on the specified thread */ AUKN_SYM bool DispatchCommandToAsyncRunner(const AuString &string, Async::WorkerId_t id); + /** - * Simulates a processed stdin line given `string` + * Parses `string` on the current thread and then schedules the ICommandSubscriber callback on the specified thread */ - AUKN_SYM bool DispatchRawLine(const AuString &string); - - /** - * Hijacks the UTF-8 input line processor coming from the consoles. DispatchCommand remains unaffected - * Call with an empty interface pointer to reenable command processing - */ - AUKN_SYM void SetCallbackAndDisableCmdProcessing(const AuSPtr &subscriber); + AUKN_SYM bool DispatchCommandToAsyncRunner(const AuString &string, Async::WorkerPId_t id); } \ No newline at end of file diff --git a/Include/Aurora/Console/Console.hpp b/Include/Aurora/Console/Console.hpp index 22994777..55d9162a 100644 --- a/Include/Aurora/Console/Console.hpp +++ b/Include/Aurora/Console/Console.hpp @@ -16,16 +16,17 @@ namespace Aurora::Console AUKN_SYM void WriteLine(const ConsoleMessage &msg); /// Consider using the following function for asynchronous utf-8 processed line based input - - /// Aurora::Console::Commands::SetCallbackAndDisableCmdProcessing(...) - /// [!!!] nonblocking - /// [!!!] you must disable the stdio console logger before using this api + /// Hooks::SetCallbackAndDisableCmdProcessing(...) AUKN_SYM AuUInt32 ReadStdIn(void *buffer, AuUInt32 length); - /// [!!!] nonblocking - /// [!!!] you should disable the stdio console logger before using this api - /// [!!!] expect system locale if the logger is not enabled - AUKN_SYM AuUInt32 WriteStdIn(const void *buffer, AuUInt32 length); + /// Consider using AuLog for general purpose use + AUKN_SYM AuUInt32 WriteStdOut(const void *buffer, AuUInt32 length); + /** + * Simulates a processed stdin line given a UTF8 string + */ + AUKN_SYM bool DispatchRawLine(const AuString &string); + AUKN_SYM void OpenLateStd(); AUKN_SYM void OpenLateGUI(); } diff --git a/Include/Aurora/Console/Hooks/Hooks.hpp b/Include/Aurora/Console/Hooks/Hooks.hpp index 43d60a55..7e01df84 100644 --- a/Include/Aurora/Console/Hooks/Hooks.hpp +++ b/Include/Aurora/Console/Hooks/Hooks.hpp @@ -8,6 +8,7 @@ #pragma once #include "IConsoleSubscriber.hpp" +#include "ITextLineSubscriber.hpp" namespace Aurora::Console::Hooks { @@ -19,4 +20,13 @@ namespace Aurora::Console::Hooks /// @deprecated wont ever remove /// Most c++ styleguides would have you chuck an IConsoleSubscriber in your app somewhere AUKN_SYM void AddFunctionalHook(LineHook_cb hook); + + + /** + * Hijacks the UTF-8 input line processor coming from the consoles. + * Call with an empty interface pointer to reenable command processing + * + * Requires AsyncApp or RuntimePump() calls + */ + AUKN_SYM void SetCallbackAndDisableCmdProcessing(const AuSPtr &subscriber); } \ No newline at end of file diff --git a/Include/Aurora/Console/Commands/ITextLineSubscriber.hpp b/Include/Aurora/Console/Hooks/ITextLineSubscriber.hpp similarity index 88% rename from Include/Aurora/Console/Commands/ITextLineSubscriber.hpp rename to Include/Aurora/Console/Hooks/ITextLineSubscriber.hpp index 96760f10..8df8ba18 100644 --- a/Include/Aurora/Console/Commands/ITextLineSubscriber.hpp +++ b/Include/Aurora/Console/Hooks/ITextLineSubscriber.hpp @@ -7,7 +7,7 @@ ***/ #pragma once -namespace Aurora::Console::Commands +namespace Aurora::Console::Hooks { AUKN_INTERFACE(ITextLineSubscriber, AUI_METHOD(void, OnProcessedLineUTF8, (const AuString &, line)) diff --git a/Include/Aurora/Console/Logging/IBasicSink.hpp b/Include/Aurora/Console/Logging/IBasicSink.hpp new file mode 100644 index 00000000..197d620e --- /dev/null +++ b/Include/Aurora/Console/Logging/IBasicSink.hpp @@ -0,0 +1,17 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IBasicSink.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Console::Logging +{ + AUKN_INTERFACE(IBasicSink, + AUI_METHOD(void, OnMessageBlocking, (const ConsoleMessage &, msg)), + AUI_METHOD(void, OnMessageNonblocking, (const ConsoleMessage &, msg)), + AUI_METHOD(void, OnFlush, ()) + ) +} \ No newline at end of file diff --git a/Include/Aurora/Console/Logging/IBasicSinkRB.hpp b/Include/Aurora/Console/Logging/IBasicSinkRB.hpp new file mode 100644 index 00000000..4dbe4e19 --- /dev/null +++ b/Include/Aurora/Console/Logging/IBasicSinkRB.hpp @@ -0,0 +1,17 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: IBasicSinkRB.hpp + Date: 2021-11-2 + Author: Reece +***/ +#pragma once + +namespace Aurora::Console::Logging +{ + struct IBasicSinkRB : IBasicSink + { + virtual void SaveToPath(const AuString &path, bool plainText = false) = 0; + virtual AuList Export() = 0; + }; +} \ No newline at end of file diff --git a/Include/Aurora/Console/Logging/ILogger.hpp b/Include/Aurora/Console/Logging/ILogger.hpp new file mode 100644 index 00000000..2357ff9e --- /dev/null +++ b/Include/Aurora/Console/Logging/ILogger.hpp @@ -0,0 +1,84 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: ILogger.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Console::Logging +{ + struct ILogger + { + virtual void WriteMessage(const ConsoleMessage &msg) = 0; + + #if defined(_AUHAS_FMT) + template + inline void WriteLinef(const AuString &tag, const AuString &msg, T&& ... args) + { + WriteMessage(ConsoleMessage(EAnsiColor::eReset, tag, fmt::format(msg, std::forward(args)...))); + } + + template + inline void WriteLinef(EAnsiColor color, const AuString &tag, const AuString &msg, T&& ... args) + { + WriteMessage(ConsoleMessage(color, tag, fmt::format(msg, std::forward(args)...))); + } + + template + inline void LogVerbose(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eYellow, "Verbose", line, std::forward(args)...); + } + + #if defined(STAGING) || defined(DEBUG) + template + inline void LogVerboseNoShip(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eYellow, "Verbose", line, std::forward(args)...); + } + #else + template + inline void LogVerboseNoShip(const AuString &line, T&& ... args) + { + } + #endif + + inline void DoNothing() + { + + } + + template + inline void LogInfo(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eGreen, "Info", line, std::forward(args)...); + } + + template + inline void LogDbg(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eYellow, "Debug", line, std::forward(args)...); + } + + template + inline void LogWarn(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eRed, "Warn", line, std::forward(args)...); + } + + template + inline void LogError(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eBoldRed, "Error", line, std::forward(args)...); + } + + template + inline void LogGame(const AuString &line, T&& ... args) + { + WriteLinef(EAnsiColor::eBlue, "Game", line, std::forward(args)...); + } + #endif + }; +} \ No newline at end of file diff --git a/Include/Aurora/Console/Logging/Logging.hpp b/Include/Aurora/Console/Logging/Logging.hpp index 4cf32218..3ac342cf 100644 --- a/Include/Aurora/Console/Logging/Logging.hpp +++ b/Include/Aurora/Console/Logging/Logging.hpp @@ -2,11 +2,16 @@ Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Logging.hpp - Date: 2021-6-9 + Date: 2021-9-21 Author: Reece ***/ #pragma once +#include "ILogger.hpp" +#include "IBasicSink.hpp" +#include "IBasicSinkRB.hpp" +#include "Sinks.hpp" + namespace Aurora::Console::Logging { #if defined(_AUHAS_FMT) @@ -35,9 +40,14 @@ namespace Aurora::Console::Logging WriteLinef(EAnsiColor::eYellow, "Verbose", line, std::forward(args)...); } #else - #define LogVerboseNoShip(...) + #define LogVerboseNoShip(...) DoNothing() #endif + static inline void DoNothing() + { + + } + template static inline void LogInfo(const AuString &line, T&& ... args) { diff --git a/Include/Aurora/Console/Logging/Sinks.hpp b/Include/Aurora/Console/Logging/Sinks.hpp new file mode 100644 index 00000000..1418f553 --- /dev/null +++ b/Include/Aurora/Console/Logging/Sinks.hpp @@ -0,0 +1,22 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: Sinks.hpp + Date: 2021-11-2 + Author: Reece +***/ +#pragma once + +namespace Aurora +{ + struct SocketConsole; +} + +namespace Aurora::Console::Logging +{ + AUKN_SHARED_API(NewGlobalPipeSink, IBasicSink); + AUKN_SHARED_API(NewFileSink, IBasicSink, const AuString &path, bool binary = false); + AUKN_SHARED_API(NewIPCSink, IBasicSink, const SocketConsole &console); + AUKN_SHARED_API(NewRingLogger, IBasicSinkRB, AuUInt32 approxMaxBytes); + AUKN_SHARED_API(NewLogger, ILogger, const AuList> &sinks); +} \ No newline at end of file diff --git a/Include/Aurora/Console/README.md b/Include/Aurora/Console/README.md new file mode 100644 index 00000000..315b44f5 --- /dev/null +++ b/Include/Aurora/Console/README.md @@ -0,0 +1,15 @@ +# Features + +* Simple global logger +* UTF8 support +* Colorful +* WxWidgets GUI for Windowed applications +* Async process line in subscription interface +* Async logger write message out subscription interface +* Optional command processing using the parse subsystem +** Commands may be dispatched on specific threads +* Supports various backends +** A log file keeping under branded user app directory OR the cwd +** Windows debugger `OutputDebugString()` +** Stdout respecting system locale and color support +** syslog diff --git a/Include/Aurora/Data/Data.hpp b/Include/Aurora/Data/Data.hpp index ee6b23be..4cbebb18 100644 --- a/Include/Aurora/Data/Data.hpp +++ b/Include/Aurora/Data/Data.hpp @@ -54,7 +54,7 @@ namespace Aurora::Data case EDataType::kTypeStructInt32: return 4; case EDataType::kTypeStructUInt64: return 8; case EDataType::kTypeStructInt64: return 8; - default: return 0; + default: return 0; } } #endif diff --git a/Include/Aurora/IO/Net/Net.hpp b/Include/Aurora/IO/Net/Net.hpp index 71fcc70c..6b2d51ec 100644 --- a/Include/Aurora/IO/Net/Net.hpp +++ b/Include/Aurora/IO/Net/Net.hpp @@ -2,210 +2,345 @@ Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Net.hpp - Date: 2021-6-9 + Date: 2021-9-21 Author: Reece ***/ #pragma once +namespace Aurora::Async +{ + struct IThreadPool; +} + namespace Aurora::IO::Net { - enum TransportProtocol - { - kProtocolUDP, - kProtocolTCP - }; + static const AuUInt16 kMagicPortAny = 65535; - enum IPProtocol - { - kIPProtocolV4, - kIPProtocolV6 - }; + struct INetworkStream; + struct IBasicSocket; + struct IClientSocket; + struct IServer; + + AUE_DEFINE(ETransportProtocol, ( + eProtocolInvalid, + eProtocolUDP, + eProtocolTCP + )); + + AUE_DEFINE(EIPProtocol, ( + eIPProtocolInvalid, + eIPProtocolV4, + eIPProtocolV6 + )); struct IPAddress { - IPProtocol ip; - bool hasIP; - AuString address; + EIPProtocol ip; union { AuUInt8 v4[4]; AuUInt16 v6[8]; }; + + AUKN_SYM IPAddress(const AuString &parse); + + AUKN_SYM AuString ToString(); + AUKN_SYM bool IsValid(); - AUKN_SYM AuString toString(); + inline bool operator ==(const IPAddress &cmp) const + { + if (cmp.ip != this->ip) return false; + + if (cmp.ip == EIPProtocol::eIPProtocolV4) + { + return memcmp(cmp.v4, this->v4, sizeof(this->v4)) == 0; + } + else + { + return memcmp(cmp.v6, this->v6, sizeof(this->v6)) == 0; + } + } }; - struct Endpoint - { - TransportProtocol protocol; - IPAddress address; - AuUInt16 port; + AUE_DEFINE(ESocketInfo, ( + eByDns, + eByEndpoint + )); + struct SocketHostName + { + SocketHostName(const AuString &name) : info(ESocketInfo::eByDns), hostname(name), address({}) + {} + + SocketHostName(const IPAddress &endpoint) : info(ESocketInfo::eByEndpoint), address(endpoint), hostname() + {} + + const ESocketInfo info; + const AuString hostname; + const IPAddress address; + + inline bool operator ==(const SocketHostName &cmp) const + { + if (cmp.info != this->info) return false; + if (cmp.info == ESocketInfo::eByEndpoint) + { + return cmp.address == this->address; + } + else + { + return cmp.hostname == this->hostname; + } + } + }; + + struct IPEndpoint + { + IPAddress ip; + AuUInt16 port; + }; + + struct ConnectionEndpoint + { + ETransportProtocol protocol; + bool tls {}; + bool compressed {}; // 0 - destination is a stateless datagram server // 1 - destination is a psuedo-stateful server - AuUInt UDPTimeout; - - AUKN_SYM AuString toString(); + AuUInt32 UDPTimeoutInMS; }; - class INetworkStream; - class IBasicSocket; - class IClientSocket; + struct ITLSHandshakeAuthenticate + { + virtual AuSPtr GetCertificate() = 0; + }; - /** - * DTLS/UDP/TCP/TLS -> TRUE = expects another datagram or read pump - * -> FALSE = end of socket life - */ - using StreamHasData_cb = std::function &socket)>; - - using InitializeSocket_cb = std::function &socket)>; - using ShutdownSocket_cb = std::function &socket)>; + AUE_DEFINE(EHandleErrorClass, ( + ePinError, + eUserDeny, + eBrokenPacket, + eInvalidCipher, + eBadCert + )); - class IClientHasData + struct TLSHandshakeError { - public: - // DTLS/UDP/TCP/TLS -> TRUE = expects another datagram or read pump - // FALSE = end of socket life - virtual bool OnSocketHasData(const AuSPtr &socket) = 0; + EHandleErrorClass error; + AuSPtr session; + AuString message; }; - class FunctionalClientHasData : public IClientHasData - { - private: - StreamHasData_cb socket_; - public: - FunctionalClientHasData(const StreamHasData_cb &socket) : socket_(socket) - {} - FunctionalClientHasData(StreamHasData_cb &&socket) : socket_(std::move(socket)) - {} - }; - - class IClientShutdown - { - public: - virtual void OnSocketShutdown(const AuSPtr &socket) = 0; - }; + AUKN_INTERFACE(IClientSubscriber, + // + AUI_METHOD(void, OnServerConnectSuccess, (const AuSPtr &, socket)), + AUI_METHOD(void, OnServerConnectFailed, (const AuSPtr &, socket)), + + // DTLS/UDP/TCP/TLS -> TRUE = expects another datagram or read pump + // FALSE = end of socket life + AUI_METHOD(bool, OnSockeData, (const AuSPtr &, socket)), - class FunctionalClientShutdown : public IClientShutdown - { - private: - ShutdownSocket_cb socket_; - public: - FunctionalClientShutdown(const ShutdownSocket_cb &socket) : socket_(socket) - {} - FunctionalClientShutdown(ShutdownSocket_cb &&socket) : socket_(std::move(socket)) - {} - }; - - class IClientDoS - { - public: - virtual void OnSocketDoS(const AuSPtr &socket) = 0; - }; + // + AUI_METHOD(void, OnSocketError, (const AuSPtr &, socket)), - class FunctionalClientDoS: public IClientDoS - { - private: - ShutdownSocket_cb socket_; - public: - FunctionalClientDoS(const ShutdownSocket_cb &socket) : socket_(socket) - {} - FunctionalClientDoS(ShutdownSocket_cb &&socket) : socket_(std::move(socket)) - {} - }; - - class IClientTLSBreakdown - { - public: - virtual void OnSocketTLSBreakdown(const AuSPtr &socket) = 0; - }; - - class FunctionalClientTLSBreakdown : public IClientTLSBreakdown - { - private: - ShutdownSocket_cb socket_; - public: - FunctionalClientTLSBreakdown(const ShutdownSocket_cb &socket) : socket_(socket) - {} - FunctionalClientTLSBreakdown(ShutdownSocket_cb &&socket) : socket_(std::move(socket)) - {} - }; - - class IClientEvents - { - public: - virtual void AddSubscriptionHasData(const AuSPtr &iface) = 0; - virtual void RemoveSubscriptionHasData(const AuSPtr &iface) = 0; + // + AUI_METHOD(void, OnSocketShutdown, (const AuSPtr &, socket)) + ); - virtual void AddSubscriptionShutdown(const AuSPtr &iface) = 0; - virtual void RemoveSubscriptionShutdown(const AuSPtr &iface) = 0; + AUKN_INTERFACE(IClientSubscriberTls, + AUI_METHOD(bool, OnVerifySocketCertificate, (const AuSPtr &, socket, const AuSPtr, session)), + AUI_METHOD(bool, OnTLSHandleError, (const AuSPtr &, socket, const TLSHandshakeError &, error)) + ); - virtual void AddSubscriptionDoS(const AuSPtr &iface) = 0; - virtual void RemoveSubscriptionDoS(const AuSPtr &iface) = 0; - - virtual void AddSubscriptionTlsBreakdown(const AuSPtr &iface) = 0; - virtual void RemoveSubscriptionTlsBreakdown(const AuSPtr &iface) = 0; - }; + AUKN_INTERFACE(IServerSubscriber, + AUI_METHOD(bool, OnClientAccept, (const AuSPtr &, server, const AuSPtr &, socket)), + AUI_METHOD(bool, OnClientDoS, (const AuSPtr &, server, const AuSPtr &, socket)), + AUI_METHOD(void, OnClientError, (const AuSPtr &, server, const AuSPtr &, socket)), + AUI_METHOD(void, OnClientShutdown, (const AuSPtr &, server, const AuSPtr &, socket)), + AUI_METHOD(bool, OnReadFrame, (const AuSPtr &, server, const AuList> &, sockets)), + AUI_METHOD(void, OnShutdown, (const AuSPtr &, server)) + ); + AUKN_INTERFACE(IServerSubscriberTls, + AUI_METHOD(bool, OnClientTLSReport, (const AuSPtr &, server, const AuSPtr &, socket, const TLSHandshakeError &, error)) + ); + + // TODO: We should introduce another std:: customer overloadable type reproducing hardcoded ascii and an int, basically std::error_code + // Maybe AuErrorCode = [std::, my_fav_stl::]error_code + // AuError = something more flexable using Error_t = std::error_code; - class IBasicSocket + struct IBasicSocketPrivateContext { - public: - virtual bool IsActive() = 0; - virtual Error_t GetLastError() = 0; - virtual void Shutdown() = 0; - virtual bool GetLocalEndpoint(Endpoint &out) = 0; + // force vtbl + virtual ~IBasicSocketPrivateContext() + {} + }; + + struct SocketStatStream + { + AuUInt64 total; + AuUInt64 averageBytesPerSecond; // interpolated, behind 1 frame + AuUInt64 extrapolatedBytesPerSecond; // this uses an extrapolated time point to predict the network bandwidth of one whole second of data }; - class IClientSocket : public IBasicSocket + struct SocketStat { - public: - - virtual IClientEvents *GetClientEvents() = 0; + AuUInt32 timeStartMs; + AuUInt32 uptimeMs; + AuUInt64 bandwidthCost; - virtual bool Initialize() = 0; + SocketStatStream rx; + SocketStatStream rt; + }; - virtual bool GetRemoteEndpoint(Endpoint &out) = 0; + enum class EUnderlyingModel + { + eAsync, + eBlocking + }; + + struct IBasicSocket + { + virtual bool IsActive() = 0; + virtual Error_t GetLastError() = 0; + virtual void Shutdown() = 0; + virtual AuSPtr SetContext(const AuSPtr &newContext) = 0; + virtual AuSPtr GetContext() = 0; + virtual bool GetLocalEndpoint(ConnectionEndpoint &out) = 0; + virtual SocketStat GetStats() = 0; + virtual EUnderlyingModel GetThreadModel() = 0; + }; + + AUKN_INTERFACE(ISocketSubmissionComplete, + AUI_METHOD(void, OnWriteFinished, (AuUInt, fence)) + ); + + struct StreamConfig + { + bool enableBufferedInput {true}; + bool enableBufferedOutput {false}; + + // 30000 * (4096 * 10) / 1024 / 1024 / 1024 + // = 1.1GB per ~1/2 of 2^16 + // Plenty of overhead for cpu blts to go brr without becoming + // cpu bound. If you're expecting 30k connections, you can + // lose 1GB to our extended seak interface by default + AuUInt32 bufferedReadSize {4096 * 10}; // see: enableBufferedInput + AuUInt32 bufferedWriteSize {4096 * 10}; // see: enableBufferedOutput + }; + + + struct IClientSocket : public IBasicSocket + { + virtual bool GetRemoteEndpoint(ConnectionEndpoint &out) = 0; + virtual bool GetLocalEndpoint(ConnectionEndpoint &out) = 0; virtual bool PumpRead() = 0; virtual bool PumpWrite() = 0; + virtual bool Pump() = 0; + + virtual void Run(int idx, AuUInt32 timeout) = 0; + - virtual bool ReadAsync(AuUInt8 *buffer, AuUInt32 &length, bool all = false) = 0; - virtual bool PeakAsync(AuUInt8 *buffer, AuUInt32 &length) = 0; - virtual bool ReadSync(AuUInt8 *buffer, AuUInt32 &length, bool all = true) = 0; + // If memory.ptr is a nullptr, this method immediately returns with the expected write length in memory.out + // + // If all is true and the internal buffer is not saturated enough yet, no data is read and + // zero readable bytes are returned. + // + // If all is false, copies memory.length into memory.ptr, up to memory.length + // + // psuedocode: + // If all is false, + // memory.outVariable = readableBytes + // if memory.ptr, + // begin copy from the internal upto max(memory.length, outVariable) into memory.ptr, + // end + // else + // return readExactly(memory) + // + // NOTE: BufferInputStreamAdhoc usage applys to async reads as well + virtual bool ReadAsync(const Memory::MemoryViewStreamWrite &memory, bool all = false) = 0; + + // Atomic + // ReadAsync(memory, false) + // SeekAsync(-memory.outVariable) + virtual bool PeakAsync(const Memory::MemoryViewStreamWrite &memory) = 0; + + // Attempts to seek backwards or forwards in the UDP or TCP packet + // If you are under the callstack of a HasXXXHasData callback, you are guaranteed (bufferedReadSize - streamPosition - streamRemaining) bytes backwards + // + virtual bool SeekAsync(int signedDistanceFromCur = 0); - virtual bool WriteAsync(const AuUInt8 *buffer, AuUInt32 length) = 0; - virtual bool WriteAsync(const AuUInt8 *buffer, AuUInt32 length, std::function) = 0; - virtual bool WriteSync(const AuUInt8 *buffer, AuUInt32 length) = 0; + // When BufferInputStreamAdhoc is called with false / in the default condition, read sync will be constrained by the async buffer + // When BufferInputStreamAdhoc is called with true, you will block until you can get the requested data (all of it if, all = true; any of it, all = false) + virtual bool ReadSync(const Memory::MemoryViewStreamWrite &memory, bool all = true) = 0; + + // Writes max(cumulative memory.length per frame, (enableBufferedInput ? bufferedWriteSize : os page allowance)) to the sockets asynchronous stream + // Returns false when no data whatsoever was written, generic error condition + // Returns true when some data was collected, regardless of any errors that may have arose (defer to IServerSubscriber::OnClientError, IClientSubscriber::OnSocketError for error handling) + virtual bool WriteAsync(const Memory::MemoryViewStreamRead &memory) = 0; + + // Alternative WriteAsync method that calls a submission notification callback on flush on the dispatching thread + virtual bool WriteAsync(const Memory::MemoryViewStreamRead &memory, AuUInt fence, const AuSPtr &input) = 0; + + // When BufferInputStreamAdhoc is called with false / in the default condition, read sync will be constrained by the async buffer + // When BufferInputStreamAdhoc is called with true, you will block until you can get the requested data (all of it if, all = true; any of it, all = false) + virtual bool WriteSync(const Memory::MemoryViewStreamRead &memory) = 0; /** * Sets the internal application buffer size * Noting that Linux's default network buffer looks like this: * Minimum, Initial, Maximum: 10240 87380 12582912 | 10KB, 86KB, 12MB */ - virtual AuUInt GetInternalRingBuffer() = 0; - virtual bool SetInternalRingBuffer(AuUInt bytes) = 0; + virtual AuUInt GetInternalInputRingBuffer() = 0; + virtual bool SetInternalInputRingBuffer(AuUInt bytes) = 0; + /** * Defines the maximum amount of bytes each recieve frame can ingest */ - virtual void SetRecvLength(AuOptional length) = 0; + virtual void SetRecvLength(AuUInt32 length) = 0; virtual AuUInt32 GetRecvLength() = 0; /** * Enable ad-hoc input buffer ingestion into our internal buffer * Only use when PumpRead will not suffice by design (ie: sync apps) + * + * When set to true, there is nothing preventing you from consuming a + * DoS-worthy amount of bytes from the stream per frame + * + * This does not disable enableBufferedInput, it merely allows you + * to read beyond enableBufferedInput/bufferedReadSize by accessing + * the os stream page or hitting the network device inline + * + * There may be a performance bottleneck and/or gain depending on your + * circumstances + * + * During high bandwidth transfers, you should set this to true + * + * Default: false (0) */ - virtual void BufferAdhoc(AuOptional value) = 0; + virtual void BufferInputStreamAdhoc(bool value) = 0; + + virtual void ConfigureHasDataCallback(bool enabled) = 0; + + virtual void ReconfigureStreams(const StreamConfig &config) = 0; }; - class ILocalClientSocket : public IClientSocket + struct ILocalClientSocket : public IClientSocket { - public: - using ConnectCallback_cb = std::function; - virtual bool Connect() = 0; - virtual void ConnectAsync(ConnectCallback_cb cb) = 0; + // Connects to the endpoint defined in the ClientConfig + // Completion will be notified by the following callbacks; + // IClientSubscriber::OnServerConnectSuccess, + // IClientSubscriber::OnServerConnectFailed + // returns true on success + virtual bool Connect() = 0; + + // Completion will be notified by the following callbacks; + // IClientSubscriber::OnServerConnectSuccess, + // IClientSubscriber::OnServerConnectFailed + // + // ...on any worker thread under a generic pump or read only pump cycle + virtual void ConnectAsync() = 0; }; struct TlsConnect @@ -214,77 +349,121 @@ namespace Aurora::IO::Net AuSPtr socket; }; - using TlsConnect_cb = std::function; - + struct SocketConfig + { + StreamConfig stream; + }; + struct ServerInfo { - Endpoint listen; + ConnectionEndpoint listen; + AuUInt32 maxSessions; + SocketConfig clientDefaults; + AuSPtr serverSubscriber; }; struct TLSServerInfo : ServerInfo { Aurora::Crypto::RSAPair cert; + AuSPtr tlsServerSubscriber; + }; + + struct ClientConfig : SocketConfig + { + SocketHostName socket; + //AuString service; + AuUInt16 port; + bool enableHasDataCallback {true}; + AuSPtr clientSubscriber; }; - struct ClientInfo + struct TLSClientConfig : ClientConfig { - Endpoint socket; - AuString service; + AuSPtr clientSubscriber; }; - struct TLSClientInfo : ClientInfo + struct IServer : public IBasicSocket { - TlsConnect_cb pinning; - }; - - class IServer : public IBasicSocket - { - public: - virtual bool AddAcceptCallback(const InitializeSocket_cb &accept) = 0; // on accept* - virtual bool AddExitCallback(const ShutdownSocket_cb &accept) = 0; // server shutdown* - virtual void GetClients(AuList> &clients) = 0; virtual bool Listen() = 0; + virtual void ReconfigureDefaultStream(const StreamConfig &config) = 0; }; - - class INetworkingPool + + struct ISocketFactory { - public: - /** - Supported thread models: - while (true) - { - // read from sockets pre-frame - PumpRead(); - - // process other async network logic here - - // process writes and a few early reads before we sleep - PumpWrite(); - - // yield - Yield(); - } - - while (true) - { - // process other async network logic here - - // run once - Pump() - } - - ...and none with BufferAdhoc enabled - */ - virtual void PumpRead() = 0; - virtual void PumpWrite() = 0; - - virtual void Pump() = 0; - virtual bool NewServer(const ServerInfo &listen, AuSPtr &out) = 0; virtual bool NewTlsServer(const TLSServerInfo &keys, AuSPtr &out) = 0; - virtual bool NewClient(const ClientInfo &info, AuSPtr &out) = 0; - virtual bool NewTlsClient(const TLSClientInfo &info, AuSPtr &out) = 0; + virtual bool NewClient(const ClientConfig &info, AuSPtr &out) = 0; + virtual bool NewTlsClient(const TLSClientConfig &info, AuSPtr &out) = 0; }; + + struct ServiceEndpoint + { + ETransportProtocol protocol; + AuString hostname; + AuString service; + }; + + struct INetworkInterface + { + virtual IPEndpoint ResolveSocketSync(const SocketHostName &hostname, AuUInt16 port); + virtual IPEndpoint ResolveServiceSync(const ServiceEndpoint &service); + + virtual bool SendDatagramAsync(const ConnectionEndpoint &endpoint, const Memory::MemoryViewRead &memory); + + virtual AuSPtr GetSocketFactory() = 0; + }; + + struct INetworkingPool + { + // A: + virtual AuUInt32 Pump(int idx) = 0; + + // B: + virtual AuUInt32 PumpRead(int idx) = 0; + virtual AuUInt32 PumpWrite(int idx) = 0; + + // C: + virtual AuUInt32 PollWorker(int idx) = 0; + virtual AuUInt32 RunWorker(int idx, AuUInt32 timeout) = 0; + + virtual AuUInt8 GetWorkers() = 0; + + virtual AuSPtr GetNetworkInterface() = 0; + + virtual void Shutdown() = 0; + }; + + AUE_DEFINE(ENetworkPoolModel, ( + + /// given a group id, uses prespawned async application event queues + eAsyncApp, + + /// + eAsyncThreadPool, + + /// it just werks + eInternalThreadPool, + + /// Developer intends to call the INetworkingPoll poll functions with the respective thread id + eUserCreateThreadRunnable + )) + + struct NetworkPool + { + AuUInt8 workers {1}; + + ENetworkPoolModel mode; + + AuUInt8 asyncWorkGroup; + AuUInt8 asyncWorkerIdOffset {}; + + AuSPtr threadPool; + + /// Ignore me. Used on platforms that suck (win32) when the model is defined as eAsyncApp or eAsyncThreadPool + AuUInt32 frequencyNotAsync {50}; + }; + + AUKN_SHARED_API(CreateNetworkPool, INetworkingPool, const NetworkPool & meta); } \ No newline at end of file diff --git a/Include/Aurora/Locale/ECodePage.hpp b/Include/Aurora/Locale/ECodePage.hpp index 81390339..276a92ec 100644 --- a/Include/Aurora/Locale/ECodePage.hpp +++ b/Include/Aurora/Locale/ECodePage.hpp @@ -11,6 +11,7 @@ namespace Aurora::Locale { enum class ECodePage { + eUnsupported, eUTF32, eUTF32BE, eUTF16, @@ -23,7 +24,6 @@ namespace Aurora::Locale eSJIS, eLatin1, eSysUnk, - eUnsupported, eMax = eUnsupported }; } \ No newline at end of file diff --git a/Include/Aurora/Locale/Encoding/Encoding.hpp b/Include/Aurora/Locale/Encoding/Encoding.hpp index 2c3bc525..210cd3f0 100644 --- a/Include/Aurora/Locale/Encoding/Encoding.hpp +++ b/Include/Aurora/Locale/Encoding/Encoding.hpp @@ -9,18 +9,32 @@ namespace Aurora::Locale::Encoding { - AUKN_SYM AuStreamReadWrittenPair_t EncodeUTF8(const Memory::MemoryViewRead &utf8, const Memory::MemoryViewWrite &binary, ECodePage page = ECodePage::eUnsupported); - - static inline AuStreamReadWrittenPair_t EncodeUTF8(const AuString &utf8, const Memory::MemoryViewWrite &binary, ECodePage page = ECodePage::eUnsupported) + struct BOM { - return EncodeUTF8(Memory::MemoryViewRead(utf8), binary, page); - } + ECodePage page; + AuUInt8 length; + }; - AUKN_SYM std::optional> DecodeBOM(const Memory::MemoryViewRead & binary); + // Attempt to guess the code page signature of the string view provided by the binary view + AUKN_SYM BOM DecodeBOM(const Memory::MemoryViewRead &binary); - /// Translates a buffer, possibly a slice of a stream, to UTF-8 - /// Returns a pair; bytes consumed, bytes written + // General purpose arbitrary page to UTF8 (AuStrings are UTF-8 - not 16 or 32; bite me) + AUKN_SYM AuStreamReadWrittenPair_t EncodeUTF8(const Memory::MemoryViewRead &utf8, const Memory::MemoryViewWrite &binary, ECodePage page = ECodePage::eUnsupported); AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead &binary, const Memory::MemoryViewWrite &utf8, ECodePage page = ECodePage::eUnsupported); - AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead &binary, AuString &out, ECodePage page = ECodePage::eUnsupported); + + // Optimized UTF translation functions + AUKN_SYM AuStreamReadWrittenPair_t ReadUTF32IntoUTF8ByteString(const Memory::MemoryViewRead &utf32, const Memory::MemoryViewWrite &utf8); + AUKN_SYM AuStreamReadWrittenPair_t ReadUTF8IntoUTF32ByteString(const Memory::MemoryViewRead &utf8, const Memory::MemoryViewWrite &utf32); + + // Endianswap functions, could be subject to SIMD optimizations + AUKN_SYM void SwapUTF32(const Memory::MemoryViewWrite &utf32); + AUKN_SYM void SwapUTF16(const Memory::MemoryViewWrite &utf16); + + // Counst the amount of codepoints in a buffer, breaking when the stream is incomplete, giving you the accurate amount of bytes or relevant codepoints in a stream view + AUKN_SYM AuUInt32 CountUTF32Length(const Memory::MemoryViewRead &utf32, bool bytes = false); // codepoint = U32 encoded; always 4 bytes per codepoint + AUKN_SYM AuUInt32 CountUTF16Length(const Memory::MemoryViewRead &utf16, bool bytes = false); // codepoint = U32 encoded; at most: 4 bytes per codepoint, usual: 2 bytes + AUKN_SYM AuUInt32 CountUTF8Length(const Memory::MemoryViewRead &utf8, bool bytes = false); // codepoint = U32 encoded; at most: 6 bytes per codepoint + AUKN_SYM AuUInt32 CountSJISLength(const Memory::MemoryViewRead &sjis, bool bytes = false); // codepoint = one character + AUKN_SYM AuUInt32 CountGBK16Length(const Memory::MemoryViewRead &gbk, bool bytes = false); // codepoint = at most; one GBK byte pair } \ No newline at end of file diff --git a/Include/Aurora/Memory/ByteBuffer.hpp b/Include/Aurora/Memory/ByteBuffer.hpp index 786a2d96..42a4c6b9 100644 --- a/Include/Aurora/Memory/ByteBuffer.hpp +++ b/Include/Aurora/Memory/ByteBuffer.hpp @@ -193,7 +193,6 @@ namespace Aurora::Memory } } - auto oldptr = readPtr; auto skipped = Read(&out, sizeof(T)); if (skipped != sizeof(T)) { diff --git a/Include/Aurora/Process/Process.hpp b/Include/Aurora/Process/Process.hpp index 4152c0ab..28157d22 100644 --- a/Include/Aurora/Process/Process.hpp +++ b/Include/Aurora/Process/Process.hpp @@ -14,15 +14,15 @@ namespace Aurora::Process { AUKN_SYM AU_NORETURN void Exit(AuUInt32 exitcode); - enum class EModulePath - { + AUE_DEFINE(EModulePath, + ( eModulePathCWD, eModulePathSystemDir, /// /lib/, windir/system32 eModulePathUserDir, /// /usr/lib/ eProcessDirectory, /// eOSSpecified, /// LD_LIBRARY_PATH + /etc/ld.so.conf, AddDllDirectory eSpecified - }; + )); static AuList kUserOverloadableSearchPath = {EModulePath::eSpecified, EModulePath::eModulePathCWD, EModulePath::eProcessDirectory, EModulePath::eModulePathUserDir, EModulePath::eModulePathSystemDir}; static AuList kAdminOverloadableSearchPath = {EModulePath::eSpecified, EModulePath::eModulePathSystemDir, EModulePath::eProcessDirectory, EModulePath::eModulePathCWD, EModulePath::eModulePathUserDir}; @@ -30,12 +30,13 @@ namespace Aurora::Process struct ModuleLoadRequest { AuString mod; - AuOptional version; - AuOptional extension; + AuString version; + AuString extension; AuList const *specifiedSearchPaths {}; AuList const *searchPath {}; - bool verifyOne {}; // always true if the executable is signed - bool verifyAll {}; + bool verify {}; // always effectively true if the executable is signed and enableMitigations is true + bool unixCheckPlusX {}; + bool enableMitigations {true}; }; AUKN_SYM bool LoadModule(const ModuleLoadRequest &request); diff --git a/Include/Aurora/Processes/IProcess.hpp b/Include/Aurora/Processes/IProcess.hpp index e9f3eabd..1158c6a3 100644 --- a/Include/Aurora/Processes/IProcess.hpp +++ b/Include/Aurora/Processes/IProcess.hpp @@ -30,6 +30,7 @@ namespace Aurora::Processes /// @param exitOnSpawn atomically destroys the current process
/// some platforms only support this /// + // TODO(Reece): what in the hell this is ugly virtual bool Start(enum ESpawnType, bool fwdOut, bool fwdErr, bool fwdIn) = 0; virtual bool Terminate() = 0; diff --git a/Include/Aurora/RNG/IRandomDevice.hpp b/Include/Aurora/RNG/IRandomDevice.hpp index a02a32dd..3d9a6e65 100644 --- a/Include/Aurora/RNG/IRandomDevice.hpp +++ b/Include/Aurora/RNG/IRandomDevice.hpp @@ -1,4 +1,4 @@ -/*** + /*** Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: IRandomDevice.hpp diff --git a/Include/Aurora/Runtime.hpp b/Include/Aurora/Runtime.hpp index f9a70314..007a7e47 100644 --- a/Include/Aurora/Runtime.hpp +++ b/Include/Aurora/Runtime.hpp @@ -19,6 +19,7 @@ #include #include +#include #include "../AuroraMacros.hpp" @@ -81,6 +82,7 @@ namespace AuData = Aurora::Data; namespace AuDebug = Aurora::Debug; namespace AuThreading = Aurora::Threading; namespace AuThreadPrimitives = Aurora::Threading::Primitives; +namespace AuThreads = Aurora::Threading::Threads; namespace AuHwInfo = Aurora::HWInfo; namespace AuIO = Aurora::IO; namespace AuIOFS = Aurora::IO::FS; @@ -95,6 +97,9 @@ namespace AuTypes = Aurora::Types; namespace AuLog = Aurora::Console::Logging; namespace AuMemory = Aurora::Memory; +using AuWorkerId_t = AuAsync::WorkerId_t; +using AuWorkerPId_t = AuAsync::WorkerPId_t; + static inline void AuDebugBreak() { AuDebug::DebugBreak(); @@ -108,6 +113,11 @@ namespace Aurora bool autoCompressOldLogs {false}; AuUInt32 maxSizeMB {128 * 1024 * 1024}; // these numbers feel insane, but at least we have a max threshold int maxLogs {1024}; // by default, we neither leak disk space or waste opportunities of being able to dig through old data + #if defined(SHIP) + bool writeLogsToUserDir {true}; // use user directory + #else + bool writeLogsToUserDir {false}; // use cwd + #endif }; struct TelemetryConfigDoc @@ -137,13 +147,21 @@ namespace Aurora TelemetryConfigDoc defaultConfig; }; + struct SocketConsole + { + bool enableLogging {false}; + bool binaryProto {false}; + AuString path; + AuIONet::ConnectionEndpoint net; + }; + struct ConsoleConfig { /// Enables Aurora::Console::xxxStd functions; defer to enableStdXX for default logger behaviour - bool enableStdPassthrough {}; + bool enableStdPassthrough {false}; - /// Disables standard, debug, and GUI consoles - bool disableAllConsoles {}; + /// Enables standard, debug, and GUI consoles + bool enableConsole {true}; /// Attempt to force a terminal emulator host under graphical subsystems bool forceConsoleWindow {}; @@ -151,7 +169,7 @@ namespace Aurora /// Attempt to force a GUI console under command line targets bool forceToolKitWindow {}; - /// In conjunction with enableStdPassthrough, enables Aurora::Console::ReadStd to read binary + /// In conjunction with enableStdPassthrough, Aurora::Console::ReadStd reads a binary stream /// In conjunction with !enableStdPassthrough, enables stdout logging bool enableStdIn {true}; @@ -162,14 +180,31 @@ namespace Aurora /// Use WxWidgets when possible bool enableWxWidgets {true}; + #if 1 /// FIO config LocalLogInfo fio; + #endif AuString titleBrand = "Aurora SDK Sample"; AuString supportPublic {"https://git.reece.sx/AuroraSupport/AuroraRuntime/issues"}; AuString supportInternal {"https://jira.reece.sx"}; }; + + struct LoggerConfig + { + /// + bool enableStdIn {true}; + + /// + bool enableStdOut {true}; + + /// FIO config + LocalLogInfo fileConfiguration; + + /// Socket config + SocketConsole socketConfiguration; + }; struct CryptoConfig { @@ -192,13 +227,16 @@ namespace Aurora struct AsyncConfig { + bool enableSchedularThread {true}; // turn this off to make your application lighter weight + AuUInt32 threadPoolDefaultStackSize {}; AuUInt32 schedularFrequency {2}; // * 0.5 or 1 MS depending on the platform AuUInt32 sysPumpFrequency {25}; // x amount of schedularFrequencys }; struct FIOConfig { - AuOptional defaultBrand = "Aurora"; + /// You can bypass branding by assigning an empty string to 'defaultBrand' + AuString defaultBrand = "Aurora"; }; struct RuntimeStartInfo diff --git a/Include/Aurora/Threading/Threads/EThreadPrio.hpp b/Include/Aurora/Threading/Threads/EThreadPrio.hpp index 676fe960..dcd43f59 100644 --- a/Include/Aurora/Threading/Threads/EThreadPrio.hpp +++ b/Include/Aurora/Threading/Threads/EThreadPrio.hpp @@ -9,13 +9,13 @@ namespace Aurora::Threading::Threads { - enum class EThreadPrio - { + AUE_DEFINE(EThreadPrio, + ( eInvalid, ePrioAboveHigh, ePrioHigh, ePrioNormal, ePrioSub, ePrioRT - }; + )); } \ No newline at end of file diff --git a/Include/Aurora/Threading/Threads/TLSVariable.hpp b/Include/Aurora/Threading/Threads/TLSVariable.hpp index 8756899d..df2e792b 100644 --- a/Include/Aurora/Threading/Threads/TLSVariable.hpp +++ b/Include/Aurora/Threading/Threads/TLSVariable.hpp @@ -34,6 +34,11 @@ namespace Aurora::Threading::Threads { return Get(); } + + T operator *() + { + return Get(); + } TLSVariable& operator =(const T & val) { diff --git a/Include/Aurora/Threading/Threads/ThreadInfo.hpp b/Include/Aurora/Threading/Threads/ThreadInfo.hpp index c1ecf1e3..f4c332aa 100644 --- a/Include/Aurora/Threading/Threads/ThreadInfo.hpp +++ b/Include/Aurora/Threading/Threads/ThreadInfo.hpp @@ -14,10 +14,10 @@ namespace Aurora::Threading::Threads ThreadInfo() {} - ThreadInfo(const AuSPtr &callbacks) : callbacks(callbacks) + ThreadInfo(const AuSPtr &callbacks) : callbacks(callbacks), stackSize(0) {} - ThreadInfo(const AuSPtr &callbacks, const AuString &name) : callbacks(callbacks), name(name) + ThreadInfo(const AuSPtr &callbacks, const AuString &name) : callbacks(callbacks), name(name), stackSize(0) {} ThreadInfo(const AuSPtr &callbacks, const AuString &name, AuUInt32 stackSize) : callbacks(callbacks), name(name), stackSize(stackSize) @@ -27,7 +27,7 @@ namespace Aurora::Threading::Threads {} AuSPtr callbacks; - AuOptional stackSize; + AuUInt32 stackSize; AuOptional name; }; } \ No newline at end of file diff --git a/Include/AuroraTypedefs.hpp b/Include/AuroraTypedefs.hpp index 99beddbd..e3b0a5ae 100644 --- a/Include/AuroraTypedefs.hpp +++ b/Include/AuroraTypedefs.hpp @@ -99,7 +99,11 @@ using AuWPtr = AURORA_RUNTIME_AU_WEAK_PTR; #define AURORA_RUNTIME_AU_UNIQUE_PTR std::unique_ptr #endif -template +#if !defined(AURORA_RUNTIME_AU_DEFAULT_DELETER) +#define AURORA_RUNTIME_AU_DEFAULT_DELETER = std::default_delete +#endif + +template using AuUPtr = AURORA_RUNTIME_AU_UNIQUE_PTR; #if !defined(AURORA_RUNTIME_AU_PAIR) diff --git a/Include/AuroraUtils.hpp b/Include/AuroraUtils.hpp index 1f898ba7..90a9997e 100644 --- a/Include/AuroraUtils.hpp +++ b/Include/AuroraUtils.hpp @@ -550,6 +550,18 @@ struct is_base_of_template_impl_au template < template class base,typename derived> using AuIsBaseOfTemplate = typename is_base_of_template_impl_au::type; +template +auto AuTuplePopFrontImpl(const Tuple& tuple, std::index_sequence) +{ + return std::make_tuple(std::get<1 + Is>(tuple)...); +} + +template +auto AuTuplePopFront(const Tuple& tuple) +{ + return AuTuplePopFrontImpl(tuple, std::make_index_sequence::value - 1>()); +} + template static inline bool AuTestBit(T value, AuUInt8 idx) { @@ -568,6 +580,9 @@ static inline void AuClearBit(T &value, AuUInt8 idx) value &= ~(T(1) << T(idx)); } +// TODO: AuPopCnt +// TODO: AuBitScanForward + #if defined(AURORA_ARCH_X64) || defined(AURORA_ARCH_X86) || defined(AURORA_ARCH_ARM) #define AURORA_PERMIT_ARBITRARY_REF #endif diff --git a/Source/Async/Async.cpp b/Source/Async/Async.cpp index cb7786c0..4faa4004 100644 --- a/Source/Async/Async.cpp +++ b/Source/Async/Async.cpp @@ -8,16 +8,19 @@ #include #include "Async.hpp" #include "Schedular.hpp" +#include "AsyncApp.hpp" namespace Aurora::Async { void InitAsync() { InitSched(); + InitApp(); } void ShutdownAsync() { DeinitSched(); + ReleaseApp(); } } \ No newline at end of file diff --git a/Source/Async/Async.hpp b/Source/Async/Async.hpp index 01a3e8ad..8f6a6d0f 100644 --- a/Source/Async/Async.hpp +++ b/Source/Async/Async.hpp @@ -7,41 +7,12 @@ ***/ #pragma once +#include "GroupState.hpp" +#include "ThreadState.hpp" +#include "AsyncRunnable.hpp" + namespace Aurora::Async { - class IAsyncRunnable - { - public: - virtual void RunAsync() = 0; - - virtual void CancelAsync() {} - }; - - class AsyncFuncRunnable : public IAsyncRunnable - { - public: - - std::function callback; - - AsyncFuncRunnable(std::function &&callback) : callback(std::move(callback)) - {} - - AsyncFuncRunnable(const std::function &callback) : callback(callback) - {} - - void RunAsync() override - { - try - { - callback(); - } - catch (...) - { - Debug::PrintError(); - } - } - }; - void InitAsync(); void ShutdownAsync(); } \ No newline at end of file diff --git a/Source/Async/AsyncApp.cpp b/Source/Async/AsyncApp.cpp index 131f09f6..80e79670 100644 --- a/Source/Async/AsyncApp.cpp +++ b/Source/Async/AsyncApp.cpp @@ -15,878 +15,30 @@ namespace Aurora::Async { - static AsyncApp gAsyncApp; - static std::atomic_int gRunningTasks {}; + static AsyncApp *gAsyncApp; + static AuSPtr gAsyncAppMem; - void IncRunningTasks() + void InitApp() { - gRunningTasks++; + gAsyncAppMem = AuMakeShared(); + gAsyncApp = gAsyncAppMem.get(); } - void DecRunningTasks() + void ReleaseApp() { - if ((--gRunningTasks) == 0) - { - gAsyncApp.Shutdown(); - } - - } - - //STATIC_TLS(WorkerId_t, tlsWorkerId); - static Threading::Threads::TLSVariable tlsWorkerId; - - using WorkEntry_t = AuPair>; - - struct AsyncAppWaitSourceRequest - { - AuConsumer, bool> callback; - AuSPtr loopSource; - AuUInt32 requestedOffset; - AuUInt64 startTime; - AuUInt64 endTime; - }; - - struct ThreadState - { - WorkerId_t id; - - bool eventDriven {}; - - AuUInt8 multipopCount = 1; - AuUInt32 lastFrameTime {}; - - Threading::Threads::ThreadShared_t threadObject; - - //std::stack jmpStack; - AuWPtr parent; - - AuThreadPrimitives::SemaphoreUnique_t syncSema; - AuList> features; - - bool rejecting {}; - bool exiting {}; - bool inLoopSourceMode {}; - bool shuttingdown {}; - - AuThreadPrimitives::EventUnique_t running; - //bool running; - - bool inline IsSysThread() - { - return id.first == 0; - } - - AuList loopSources; - AuList pendingWorkItems; - }; - - struct GroupState - { - ThreadGroup_t group; - - AuThreadPrimitives::ConditionMutexUnique_t cvWorkMutex; - AuThreadPrimitives::ConditionVariableUnique_t cvVariable; - AuSPtr eventLs; - - AuList workQueue; - - AuBST> workers; - - bool Init(); - - bool inline IsSysThread() - { - return group == 0; - } - }; - - bool GroupState::Init() - { - this->cvWorkMutex = AuThreadPrimitives::ConditionMutexUnique(); - if (!this->cvWorkMutex) - { - return false; - } - - this->cvVariable = AuThreadPrimitives::ConditionVariableUnique(AuUnsafeRaiiToShared(this->cvWorkMutex)); - if (!this->cvVariable) - { - return false; - } - - this->eventLs = Loop::NewLSEvent(false, false, true); - if (!this->eventLs) - { - return false; - } - return true; - } - - AsyncApp::AsyncApp() - { - this->rwlock_ = AuThreadPrimitives::RWLockUnique(); - SysAssert(static_cast(this->rwlock_), "Couldn't initialize AsyncApp. Unable to allocate an RWLock"); - } - - // TODO: barrier multiple - bool AsyncApp::Barrier(WorkerId_t worker, AuUInt32 ms, bool requireSignal, bool drop) - { - auto &semaphore = GetThreadState()->syncSema; - auto unsafeSemaphore = semaphore.get(); - - auto work = AuMakeShared(([=]() - { - auto state = GetThreadState(); - - if (drop) - { - state->rejecting = true; - } - - if (requireSignal) - { - state->running->Reset(); - } - - unsafeSemaphore->Unlock(1); - - if (requireSignal) - { - state->running->Lock(); - } - })); - - #if 0 - NewWorkItem({worker.first, worker.second}, work)->Dispatch(); - #else - Run(worker, work); - #endif - - return WaitFor(worker, AuUnsafeRaiiToShared(semaphore), ms); - } - - void AsyncApp::Run(WorkerId_t target, AuSPtr runnable) - { - auto state = GetGroup(target.first); - SysAssert(static_cast(state), "couldn't dispatch a task to an offline group"); - - IncRunningTasks(); - - { - AU_LOCK_GUARD(state->cvWorkMutex); - - #if defined(STAGING) || defined(DEBUG) - AU_LOCK_GUARD(rwlock_->AsReadable()); - - if (target.second != Async::kThreadIdAny) - { - auto itr = state->workers.find(target.second); - if ((itr == state->workers.end()) || (itr->second->rejecting)) - { - SysPushErrorGen("worker: {}:{} is offline", target.first, target.second); - DecRunningTasks(); - throw "Requested job worker is offline"; - } - } - else - { - auto workers = state->workers; - bool found = false; - - for (const auto &worker : state->workers) - { - if (!worker.second->rejecting) - { - found = true; - break; - } - } - - if (!found) - { - DecRunningTasks(); - throw "No workers available"; - } - } - #endif - - state->workQueue.push_back(AuMakePair(target.second, runnable)); - state->eventLs->Set(); - } - - if (target.second == Async::kThreadIdAny) - { - state->cvVariable->Signal(); - } - else - { - // sad :( - state->cvVariable->Broadcast(); - } - } - - int AsyncApp::CfxPollPush() - { - // TOOD (Reece): implement a context switching library - // Refer to the old implementation of this oin pastebin - return 0; - } - - void AsyncApp::CtxPollReturn(const AuSPtr &state, int status, bool hitTask) - { - // TOOD (Reece): implement a context switching library - // Refer to the old implementation of this oin pastebin - } - - bool AsyncApp::CtxYield() - { - bool ranAtLeastOne = false; - - while (this->Poll(false)) - { - ranAtLeastOne = true; - } - - return ranAtLeastOne; - } - - bool AsyncApp::ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) - { - auto thread = this->GetThreadHandle(workerId); - if (!thread) - { - return false; - } - - auto group = thread->parent.lock(); - - { - AU_LOCK_GUARD(group->cvWorkMutex); - - AsyncAppWaitSourceRequest req {}; - req.startTime = Time::CurrentClockMS(); - if (timeout) - { - req.requestedOffset = timeout; - req.endTime = req.startTime + timeout; - } - req.loopSource = loopSource; - req.callback = callback; - - if (!AuTryInsert(thread->loopSources, req)) - { - return false; - } - - thread->inLoopSourceMode = thread->loopSources.size(); - } - - return true; - } - - bool AsyncApp::Poll(bool block) - { - auto state = GetThreadState(); - bool success {}; - - do - { - if (state->inLoopSourceMode) - { - success = PollLoopSource(block); - } - else - { - success = PollInternal(block); - success |= state->inLoopSourceMode; - } - } while (success); - - return success; - } - - bool AsyncApp::PollLoopSource(bool block) - { - auto state = GetThreadState(); - auto group = state->parent.lock(); - - //state->pendingWorkItems.clear(); - - auto magic = CfxPollPush(); - bool retValue {}; - - // TODO (reece): This function isn't very efficient - { - AU_LOCK_GUARD(group->cvWorkMutex); - - AuList curLoopReq = state->loopSources; - AuList> curLoopSources; - - auto lenLoopReqs = curLoopReq.size(); - - curLoopSources.resize(lenLoopReqs + 1); - - for (auto i = 0; i < lenLoopReqs; i++) - { - curLoopSources[i] = curLoopReq[i].loopSource; - } - - curLoopSources[lenLoopReqs] = group->eventLs; - - AuList> nextLoopSources; - if (block) - { - nextLoopSources = Loop::WaitMultipleObjects(curLoopSources, 0); - } - else - { - nextLoopSources.reserve(curLoopSources.size()); - for (const auto &source : curLoopSources) - { - if (source->IsSignaled()) - { - nextLoopSources.push_back(source); - } - } - } - - auto time = Time::CurrentClockMS(); - - state->loopSources.clear(); - state->loopSources.reserve(curLoopReq.size()); - - if (AuExists(nextLoopSources, group->eventLs)) - { - PollInternal(false); - } - - for (const auto &request : curLoopReq) - { - bool remove {}; - bool removeType {}; - - if (AuExists(nextLoopSources, request.loopSource)) - { - remove = true; - removeType = true; - } - else - { - if (request.requestedOffset) - { - if (request.endTime < time) - { - remove = true; - removeType = false; - } - } - } - - if (!remove) - { - state->loopSources.push_back(request); - } - else - { - request.callback(request.loopSource, removeType); - retValue |= removeType; - } - } - - state->inLoopSourceMode = state->loopSources.size(); - } - - return retValue; - } - - bool AsyncApp::PollInternal(bool blocking) - { - auto state = GetThreadState(); - auto group = state->parent.lock(); - - //state->pendingWorkItems.clear(); - - auto magic = CfxPollPush(); - - { - AU_LOCK_GUARD(group->cvWorkMutex); - - do - { - // Deque tasks the current thread runner could dipatch - // Noting that `multipopCount` determines how aggressive threads are in dequeuing work - // It's probable `multipopCount` will equal 1 for your use case - // - // Only increment when you know tasks within a group queue will not depend on one another - // *and* tasks require a small amount of execution time - // - // This could be potentially useful for an event dispatcher whereby you're dispatching - // hundreds of items per second, across a thread or two, knowing dequeuing one instead of all - // is a waste of CPU cycles. - // - // Remember, incrementing `multipopCount` is potentially dangerous the second you have local - // thread group waits - for (auto itr = group->workQueue.begin(); - ((itr != group->workQueue.end()) && - (state->pendingWorkItems.size() < state->multipopCount)); - ) - { - if (itr->first == Async::kThreadIdAny) - { - state->pendingWorkItems.push_back(*itr); - itr = group->workQueue.erase(itr); - continue; - } - - if ((itr->first != Async::kThreadIdAny) && (itr->first == state->id.second)) - { - state->pendingWorkItems.push_back(*itr); - itr = group->workQueue.erase(itr); - continue; - } - - itr++; - } - - // Consider blocking for more work - if (!blocking) - { - break; - } - - // Block if no work items are present - if (state->pendingWorkItems.empty()) - { - group->cvVariable->WaitForSignal(); - } - - // Post-wakeup thread terminating check - if (state->threadObject->Exiting() || state->shuttingdown) - { - break; - } - - } while (state->pendingWorkItems.empty()); - - if (group->workQueue.empty()) - { - group->eventLs->Reset(); - } - } - - if (state->pendingWorkItems.empty()) - { - CtxPollReturn(state, magic, false); - return false; - } - - int runningTasks {}; - - for (auto itr = state->pendingWorkItems.begin(); itr != state->pendingWorkItems.end(); ) - { - if (state->threadObject->Exiting() || state->shuttingdown) - { - break; - } - - // Set the last frame time for a watchdog later down the line - state->lastFrameTime = Time::CurrentClockMS(); - - // Dispatch - itr->second->RunAsync(); - - // Remove from our local job queue - itr = state->pendingWorkItems.erase(itr); - - // Atomically decrement global task counter - runningTasks = gRunningTasks.fetch_sub(1) - 1; - } - - // Return popped work back to the groups work pool when our -pump loops were preempted - if (state->pendingWorkItems.size()) - { - AU_LOCK_GUARD(group->cvWorkMutex); - group->workQueue.insert(group->workQueue.end(), state->pendingWorkItems.begin(), state->pendingWorkItems.end()); - group->eventLs->Set(); - state->pendingWorkItems.clear(); - } - - CtxPollReturn(state, magic, true); - - - if (state->eventDriven) - { - if (runningTasks == 0) - { - ShutdownZero(); - } - } - - return true; - } - - bool AsyncApp::WaitFor(WorkerId_t unlocker, const AuSPtr &primitive, AuUInt32 timeoutMs) - { - auto curThread = GetThreadState(); - - bool workerIdMatches = (unlocker.second == curThread->id.second) || ((unlocker.second == Async::kThreadIdAny) && (GetThreadWorkersCount(unlocker.first) == 1)); - - if ((unlocker.first == curThread->id.first) && // work group matches - (workerIdMatches)) // well, crap - { - - bool queryAsync = false; - while (!(queryAsync ? primitive->TryLock() : Threading::WaitFor(primitive.get(), 2))) - { - queryAsync = CtxYield(); - } - - return true; - } - else - { - return Threading::WaitFor(primitive.get(), timeoutMs); - } + gAsyncAppMem.reset(); } void AsyncApp::Start() { - SysAssert(Spawn({0, 0})); - StartSched(); + ThreadPool::SetRunningMode(true); + SysAssert(ThreadPool::Create({0, 0})); + StartSched(); // this is now an init once function } void AsyncApp::Main() { - Entrypoint({0, 0}); - } - - void AsyncApp::ShutdownZero() - { - Shutdown(); - } - - void AsyncApp::Shutdown() - { - // Nested shutdowns can happen a write lock - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - if (shuttingdown_) - { - return; - } - } - - // Set shutdown flag - { - AU_LOCK_GUARD(rwlock_->AsWritable()); - if (std::exchange(shuttingdown_, true)) - { - return; - } - } - - // Noting - // 1) that StopSched may lockup under a writable lock - // -> we will terminate a thread that may be dispatching a sys pump event - // 2) that barrier doesn't need to be under a write lock - // - // Perform the following shutdown of the schedular and other available threads under a read lock - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - StopSched(); - - for (auto &[groupId, group] : this->threads_) - { - for (auto &[id, worker] : group->workers) - { - Barrier(worker->id, 0, false, true); - } - } - } - - // Finally set the shutdown flag on all of our thread contexts - // then release them from the runners/workers list - // then release all group contexts - AuList threads; - { - AU_LOCK_GUARD(rwlock_->AsWritable()); - - for (auto &[groupId, group] : this->threads_) - { - for (auto &[id, worker] : group->workers) - { - worker->shuttingdown = true; - - if (groupId != 0) - { - worker->threadObject->SendExitSignal(); - threads.push_back(worker->threadObject); - } - - auto &event = worker->running; - if (event) - { - event->Set(); - } - } - - if (group->cvVariable) - { - AU_LOCK_GUARD(group->cvWorkMutex); - group->cvVariable->Broadcast(); - } - } - } - - // Sync to shutdown threads to prevent a race condition whereby the async subsystem shuts down before the threads - for (const auto &thread : threads) - { - thread->Exit(); - } - } - - bool AsyncApp::Exiting() - { - return shuttingdown_ || GetThreadState()->exiting; - } - - - void AsyncApp::SetWorkerIdIsThreadRunner(WorkerId_t workerId, bool callerOwns) - { - GetThreadHandle(workerId)->eventDriven = callerOwns; - } - - bool AsyncApp::Spawn(WorkerId_t workerId) - { - AU_LOCK_GUARD(rwlock_->AsWritable()); - - AuSPtr group; - - // Try fetch or allocate group - { - AuSPtr* groupPtr; - if (!AuTryFind(this->threads_, workerId.first, groupPtr)) - { - group = AuMakeShared(); - - if (!group->Init()) - { - SysPushErrorMem("Not enough memory to intiialize a new group state"); - return false; - } - - if (!AuTryInsert(this->threads_, AuMakePair(workerId.first, group))) - { - return false; - } - } - else - { - group = *groupPtr; - } - } - - // Assert worker does not already exist - { - AuSPtr* ret; - - if (AuTryFind(group->workers, workerId.second, ret)) - { - SysPushErrorGen("Thread ID already exists"); - return false; - } - } - - auto threadState = AuMakeShared(); - threadState->parent = group; - threadState->running = AuThreadPrimitives::EventUnique(true, false, true); - threadState->syncSema = AuThreadPrimitives::SemaphoreUnique(0); - threadState->id = workerId; - threadState->eventDriven = true; - - if (!threadState->IsSysThread()) - { - threadState->threadObject = AuThreading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(&AsyncApp::Entrypoint, this, threadState->id)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}) - )); - threadState->threadObject->Run(); - } - else - { - threadState->threadObject = AuSPtr(Threading::Threads::GetThread(), [](Threading::Threads::IAuroraThread *){}); - } - - group->workers.insert(AuMakePair(workerId.second, threadState)); - return true; - } - - - AuSPtr AsyncApp::GetThreadHandle(WorkerId_t id) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - auto group = GetGroup(id.first); - if (!group) - { - return {}; - } - - AuSPtr* ret; - if (!AuTryFind(group->workers, id.second, ret)) - { - return {}; - } - - return *ret; - } - - Threading::Threads::ThreadShared_t AsyncApp::ResolveHandle(WorkerId_t id) - { - return GetThreadHandle(id)->threadObject; - } - - AuBST> AsyncApp::GetThreads() - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - AuBST> ret; - - for (const auto &group : this->threads_) - { - AuList workers; - - for (const auto &thread : group.second->workers) - { - workers.push_back(thread.second->id.second); - } - - ret[group.first] = workers; - } - - return ret; - } - - WorkerId_t AsyncApp::GetCurrentThread() - { - return tlsWorkerId; - } - - bool AsyncApp::Sync(WorkerId_t groupId, AuUInt32 timeoutMs, bool requireSignal) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - auto group = GetGroup(groupId.first); - auto currentWorkerId = GetCurrentThread().second; - - if (groupId.second == Async::kThreadIdAny) - { - for (auto &jobWorker : group->workers) - { - if (!Barrier(jobWorker.second->id, timeoutMs, requireSignal && jobWorker.second->id.second != currentWorkerId, false)) // BAD!, should subtract time elapsed, clamp to, i dunno, 5ms min? - { - return false; - } - } - } - else - { - return Barrier(groupId, timeoutMs, requireSignal && groupId.second != currentWorkerId, false); - } - - return true; - } - - void AsyncApp::Signal(WorkerId_t groupId) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - auto group = GetGroup(groupId.first); - if (groupId.second == Async::kThreadIdAny) - { - for (auto &jobWorker : group->workers) - { - jobWorker.second->running->Set(); - } - } - else - { - GetThreadHandle(groupId)->running->Set(); - } - } - - void AsyncApp::SyncAllSafe() - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - for (const auto &re : this->threads_) - { - for (auto &jobWorker : re.second->workers) - { - SysAssert(Barrier(jobWorker.second->id, 0, false, false)); - } - } - } - - AuSPtr AsyncApp::GetGroup(ThreadGroup_t type) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - AuSPtr* ret; - if (!AuTryFind(this->threads_, type, ret)) - { - return {}; - } - - return *ret; - } - - size_t AsyncApp::GetThreadWorkersCount(ThreadGroup_t group) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - return GetGroup(group)->workers.size(); - } - - AuSPtr AsyncApp::GetThreadState() - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - auto id = GetCurrentThread(); - auto state = GetGroup(id.first); - return state->workers[id.second]; - } - - void AsyncApp::Entrypoint(WorkerId_t id) - { - tlsWorkerId = id; - - auto auThread = Threading::Threads::GetThread(); - auto job = GetThreadState(); - - while ((!auThread->Exiting()) && (!job->shuttingdown)) - { - // Do work (blocking) - Poll(true); - } - - LogDbg("Thread leaving: {} {} {}", auThread->GetName(), auThread->Exiting(), job->shuttingdown); - - if (id != WorkerId_t {0, 0}) - { - AU_LOCK_GUARD(rwlock_->AsReadable()); - - if (!shuttingdown_ && !job->rejecting) - { - // Pump and barrier + reject all after atomically - Barrier(id, 0, false, true); - } - } - - ThisExiting(); - - if (id == WorkerId_t {0, 0}) - { - Shutdown(); - } + ThreadPool::Entrypoint({0, 0}); } void AsyncApp::SetConsoleCommandDispatcher(WorkerId_t id) @@ -895,79 +47,129 @@ namespace Aurora::Async Console::Commands::UpdateDispatcher(commandDispatcher_); } - void AsyncApp::ThisExiting() + void AsyncApp::CleanUpWorker(WorkerId_t id) { - auto id = GetCurrentThread(); - auto state = GetGroup(id.first); - + // This shouldn't be a problem; however, we're going to handle the one edge case where + // some angry sysadmin is spamming commands + if ((commandDispatcher_.has_value()) + && (commandDispatcher_.value() == id)) { - AU_LOCK_GUARD(rwlock_->AsWritable()); - - auto itr = state->workers.find(id.second); - auto &jobWorker = itr->second; - - // This shouldn't be a problem; however, we're going to handle the one edge case where - // some angry sysadmin is spamming commands - if ((commandDispatcher_.has_value()) - && (commandDispatcher_.value() == id)) - { - Console::Commands::UpdateDispatcher({}); - } - - // Abort scheduled tasks - TerminateSceduledTasks(id); - - // Clean up thread features - // -> transferable TLS handles - // -> thread specific vms - // -> anything your brain wishes to imagination - for (const auto &thread : jobWorker->features) - { - try - { - thread->Cleanup(); - } - catch (...) - { - LogWarn("Couldn't clean up thread feature!"); - Debug::PrintError(); - } - } - - jobWorker->features.clear(); - - state->workers.erase(itr); - } - } - - void AsyncApp::AddFeature(WorkerId_t id, AuSPtr feature, bool async) - { - auto work = AuMakeShared(([=]() - { - GetThreadState()->features.push_back(feature); - feature->Init(); - })); - - auto workItem = NewWorkItem(id, work, !async)->Dispatch(); - - if (!async) - { - workItem->BlockUntilComplete(); + Console::Commands::UpdateDispatcher({}); } } - void AsyncApp::AssertInThreadGroup(ThreadGroup_t group) + void AsyncApp::CleanWorkerPoolReservedZeroFree() { - SysAssert(static_cast(tlsWorkerId).first == group); - } - - void AsyncApp::AssertWorker(WorkerId_t id) - { - SysAssert(static_cast(tlsWorkerId) == id); + ThreadPool::Shutdown(); } AUKN_SYM IAsyncApp *GetAsyncApp() { - return &gAsyncApp; + return gAsyncApp; + } + + bool AsyncApp::Spawn(WorkerId_t workerId) + { + return ThreadPool::Spawn(workerId); + } + + void AsyncApp::SetRunningMode(bool eventRunning) + { + ThreadPool::SetRunningMode(eventRunning); + } + + bool AsyncApp::Create(WorkerId_t workerId) + { + return ThreadPool::Create(workerId); + } + + bool AsyncApp::InRunnerMode() + { + return ThreadPool::InRunnerMode(); + } + + bool AsyncApp::Poll() + { + return ThreadPool::Poll(); + } + + bool AsyncApp::RunOnce() + { + return ThreadPool::RunOnce(); + } + + bool AsyncApp::Run() + { + return ThreadPool::Run(); + } + + void AsyncApp::Shutdown() + { + ThreadPool::Shutdown(); + } + + bool AsyncApp::Exiting() + { + return ThreadPool::Exiting(); + } + + AuSPtr AsyncApp::NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) + { + return ThreadPool::NewWorkItem(worker, task, supportsBlocking); + } + + AuSPtr AsyncApp::NewFence() + { + return ThreadPool::NewFence(); + } + + Threading::Threads::ThreadShared_t AsyncApp::ResolveHandle(WorkerId_t id) + { + return ThreadPool::ResolveHandle(id); + } + + AuBST> AsyncApp::GetThreads() + { + return ThreadPool::GetThreads(); + } + + WorkerId_t AsyncApp::GetCurrentThread() + { + return ThreadPool::GetCurrentThread(); + } + + bool AsyncApp::Sync(WorkerId_t workerId, AuUInt32 timeoutMs, bool requireSignal) + { + return ThreadPool::Sync(workerId, timeoutMs, requireSignal); + } + + void AsyncApp::Signal(WorkerId_t workerId) + { + ThreadPool::Signal(workerId); + } + + void AsyncApp::SyncAllSafe() + { + ThreadPool::SyncAllSafe(); + } + + void AsyncApp::AddFeature(WorkerId_t id, AuSPtr feature, bool async) + { + ThreadPool::AddFeature(id, feature, async); + } + + void AsyncApp::AssertInThreadGroup(ThreadGroup_t group) + { + ThreadPool::AssertInThreadGroup(group); + } + + void AsyncApp::AssertWorker(WorkerId_t id) + { + ThreadPool::AssertWorker(id); + } + + bool AsyncApp::ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) + { + return ThreadPool::ScheduleLoopSource(loopSource, workerId, timeout, callback); } } \ No newline at end of file diff --git a/Source/Async/AsyncApp.hpp b/Source/Async/AsyncApp.hpp index 5b8b613c..e2c2c90a 100644 --- a/Source/Async/AsyncApp.hpp +++ b/Source/Async/AsyncApp.hpp @@ -7,87 +7,46 @@ ***/ #pragma once +#include "ThreadPool.hpp" + namespace Aurora::Async { - struct GroupState; - struct ThreadState; - //class WorkItem; + void InitApp(); + void ReleaseApp(); - - void DecRunningTasks(); - void IncRunningTasks(); - - class AsyncApp : public IAsyncApp + struct AsyncApp : public IAsyncApp, ThreadPool { - public: - AsyncApp(); + bool Spawn(WorkerId_t workerId) override; + void SetRunningMode(bool eventRunning) override; + bool Create(WorkerId_t workerId) override; + bool InRunnerMode() override; + bool Poll() override; + bool RunOnce() override; + bool Run() override; + void Shutdown() override; + bool Exiting() override; + AuSPtr NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) override; + AuSPtr NewFence() override; + Threading::Threads::ThreadShared_t ResolveHandle(WorkerId_t) override; + AuBST> GetThreads() override; + WorkerId_t GetCurrentThread() override; + bool Sync(WorkerId_t workerId, AuUInt32 timeoutMs = 0, bool requireSignal = false) override; + void Signal(WorkerId_t workerId) override; + void SyncAllSafe() override; + void AddFeature(WorkerId_t id, AuSPtr feature, bool async) override; + void AssertInThreadGroup(ThreadGroup_t group) override; + void AssertWorker(WorkerId_t id) override; + bool ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) override; + // Main thread logic void Start() override; void Main() override; - void Shutdown() override; - bool Exiting() override; void SetConsoleCommandDispatcher(WorkerId_t id) override; - // Spawning - bool Spawn(WorkerId_t workerId) override; - void SetWorkerIdIsThreadRunner(WorkerId_t, bool runner) override; + void CleanUpWorker(WorkerId_t wid) override; + void CleanWorkerPoolReservedZeroFree() override; - Threading::Threads::ThreadShared_t ResolveHandle(WorkerId_t) override; - - AuBST> GetThreads() override; - WorkerId_t GetCurrentThread() override; - - // Synchronization - bool Sync(WorkerId_t group, AuUInt32 timeoutMs, bool requireSignal) override; - void Signal(WorkerId_t group) override; - - bool WaitFor(WorkerId_t unlocker, const AuSPtr &primitive, AuUInt32 ms); // when unlocker = this, pump event loop - //bool WaitFor(DispatchTarget_t unlocker, Threading::IWaitable *primitive, AuUInt32 ms) override; // when unlocker = this, pump event loop - - void SyncAllSafe() override; - - // Features - void AddFeature(WorkerId_t id, AuSPtr feature, bool async = false) override; - - // Debug - void AssertInThreadGroup(ThreadGroup_t group) override; - void AssertWorker(WorkerId_t id) override; - - void Run(WorkerId_t target, AuSPtr runnable); - - bool Poll(bool block) override; - bool PollInternal(bool block); - bool PollLoopSource(bool block); - - size_t GetThreadWorkersCount(ThreadGroup_t group); - - bool CtxYield(); - int CfxPollPush(); - void CtxPollReturn(const AuSPtr &state, int status, bool hitTask); - bool ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) override; - - private: - AuSPtr GetThreadHandle(WorkerId_t id); - - void ThisExiting(); - void ShutdownZero(); - - // TODO: BarrierMultiple - bool Barrier(WorkerId_t, AuUInt32 ms, bool requireSignal, bool drop); - - AuThreadPrimitives::RWLockUnique_t rwlock_; - - AuSPtr GetGroup(ThreadGroup_t type); - - AuSPtr GetThreadState(); - - void Entrypoint(WorkerId_t id); - - using ThreadDb_t = AuBST>; - - ThreadDb_t threads_; - bool shuttingdown_ {}; AuOptional commandDispatcher_; }; } \ No newline at end of file diff --git a/Source/Async/AsyncRunnable.hpp b/Source/Async/AsyncRunnable.hpp new file mode 100644 index 00000000..8125899b --- /dev/null +++ b/Source/Async/AsyncRunnable.hpp @@ -0,0 +1,75 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: AsyncRunnable.hpp + Date: 2021-11-2 + Author: Reece +***/ +#pragma once + +namespace Aurora::Async +{ + class IAsyncRunnable + { + public: + virtual float GetPrio() { return 0.5f; }; + virtual void RunAsync() = 0; + + virtual void CancelAsync() {} + }; + + class AsyncFuncRunnable : public IAsyncRunnable + { + public: + + std::function callback; + std::function fail; + AuThreadPrimitives::SpinLock lock; + + AsyncFuncRunnable(std::function &&callback) : callback(std::move(callback)) + {} + + AsyncFuncRunnable(std::function &&callback, std::function &&fail) : callback(std::move(callback)), fail(std::move(fail)) + {} + + AsyncFuncRunnable(const std::function &callback) : callback(callback) + {} + + AsyncFuncRunnable(const std::function &callback, const std::function &fail) : callback(callback), fail(fail) + {} + + void RunAsync() override + { + AU_LOCK_GUARD(lock); + SysAssertDbgExp(callback, "Missing callback std::function"); + try + { + callback(); + } + catch (...) + { + Debug::PrintError(); + } + fail = {}; + callback = {}; + } + + void CancelAsync() override + { + AU_LOCK_GUARD(lock); + if (fail) + { + try + { + fail(); + } + catch (...) + { + Debug::PrintError(); + } + } + fail = {}; + callback = {}; + } + }; +} \ No newline at end of file diff --git a/Source/Async/GroupState.cpp b/Source/Async/GroupState.cpp new file mode 100644 index 00000000..14422d3f --- /dev/null +++ b/Source/Async/GroupState.cpp @@ -0,0 +1,35 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: GroupState.cpp + Date: 2021-11-1 + Author: Reece +***/ +#include +#include "Async.hpp" +#include "GroupState.hpp" + +namespace Aurora::Async +{ + bool GroupState::Init() + { + this->cvWorkMutex = AuThreadPrimitives::ConditionMutexUnique(); + if (!this->cvWorkMutex) + { + return false; + } + + this->cvVariable = AuThreadPrimitives::ConditionVariableUnique(AuUnsafeRaiiToShared(this->cvWorkMutex)); + if (!this->cvVariable) + { + return false; + } + + this->eventLs = Loop::NewLSEvent(false, false, true); + if (!this->eventLs) + { + return false; + } + return true; + } +} \ No newline at end of file diff --git a/Source/Async/GroupState.hpp b/Source/Async/GroupState.hpp new file mode 100644 index 00000000..e9fec374 --- /dev/null +++ b/Source/Async/GroupState.hpp @@ -0,0 +1,34 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: GroupState.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once +#include "ThreadState.hpp" + +namespace Aurora::Async +{ + struct GroupState + { + ThreadGroup_t group; + + AuThreadPrimitives::ConditionMutexUnique_t cvWorkMutex; + AuThreadPrimitives::ConditionVariableUnique_t cvVariable; + AuSPtr eventLs; + + AuList workQueue; + bool sorted {}; + AuUInt32 dirty {}; + + AuBST> workers; + + bool Init(); + + bool inline IsSysThread() + { + return group == 0; + } + }; +} \ No newline at end of file diff --git a/Source/Async/Schedular.cpp b/Source/Async/Schedular.cpp index aa10eea7..0cff386e 100644 --- a/Source/Async/Schedular.cpp +++ b/Source/Async/Schedular.cpp @@ -8,7 +8,8 @@ #include #include "Async.hpp" #include "Schedular.hpp" -#include "AsyncApp.hpp" +//#include "AsyncApp.hpp" +#include "ThreadPool.hpp" namespace Aurora::Async { @@ -17,9 +18,10 @@ namespace Aurora::Async AuUInt64 ns; WorkerId_t target; AuSPtr runnable; + IThreadPoolInternal *pool; }; - static Threading::Threads::ThreadUnique_t gThread; + static AuThreads::ThreadUnique_t gThread; static AuThreadPrimitives::MutexUnique_t gSchedLock; static AuList gEntries; @@ -56,7 +58,7 @@ namespace Aurora::Async AuUInt32 counter {}; AuList pending; - auto thread = Threading::Threads::GetThread(); + auto thread = AuThreads::GetThread(); while (!thread->Exiting()) { @@ -70,25 +72,29 @@ namespace Aurora::Async { try { - static_cast(GetAsyncApp())->Run(entry.target, entry.runnable); - DecRunningTasks(); + entry.pool->Run(entry.target, entry.runnable); + entry.pool->DecrementTasksRunning(); } catch (...) { - LogWarn("Dropped scheduled task! Expect a leaky counter!"); - LogWarn("Would you rather `Why u no exit?!` or `WHY DID U JUST CRASH REEEE` in production?"); + if (entry.pool->ToThreadPool()->InRunnerMode()) + { + LogWarn("Dropped scheduled task! Expect a leaky counter!"); + LogWarn("Would you rather `Why u no exit?!` or `WHY DID U JUST CRASH REEEE` in production?"); + } Debug::PrintError(); } } counter++; + if ((!gRuntimeConfig.async.sysPumpFrequency) || ((gRuntimeConfig.async.sysPumpFrequency) && (counter % gRuntimeConfig.async.sysPumpFrequency) == 0)) { try { if (!std::exchange(gLockedPump, true)) { - NewWorkItem({0, 0}, AuMakeShared(PumpSysThread))->Dispatch(); + NewWorkItem(AuWorkerId_t{0, 0}, AuMakeShared(PumpSysThread))->Dispatch(); } } catch (...) @@ -98,7 +104,6 @@ namespace Aurora::Async } } } - } void InitSched() @@ -114,32 +119,41 @@ namespace Aurora::Async void StartSched() { - gThread = Threading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(SchedThread)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}) + AU_LOCK_GUARD(gSchedLock); + if (gThread) return; + + gThread = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(SchedThread)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}) )); + gThread->Run(); } + /// @deprecated void StopSched() { - gThread.reset(); + // intentionally no-op + #if 0 + gThread.reset(); + #endif } - void Schedule(AuUInt64 ns, WorkerId_t target, AuSPtr runnable) + void Schedule(AuUInt64 ns, IThreadPoolInternal *pool, WorkerId_t target, AuSPtr runnable) { AU_LOCK_GUARD(gSchedLock); - IncRunningTasks(); - gEntries.push_back({ns, target, runnable}); + pool->IncrementTasksRunning(); + gEntries.push_back({ns, target, runnable, pool}); } - void TerminateSceduledTasks(WorkerId_t target) + void TerminateSceduledTasks(IThreadPoolInternal *pool, WorkerId_t target) { AU_LOCK_GUARD(gSchedLock); for (auto itr = gEntries.begin(); itr != gEntries.end(); ) { - if (itr->target <= target) + if ((itr->pool == pool) && + ((itr->target == target) || (target.second == Async::kThreadIdAny && target.first == itr->target.first))) { itr->runnable->CancelAsync(); itr = gEntries.erase(itr); diff --git a/Source/Async/Schedular.hpp b/Source/Async/Schedular.hpp index a68241c6..59927157 100644 --- a/Source/Async/Schedular.hpp +++ b/Source/Async/Schedular.hpp @@ -9,11 +9,13 @@ namespace Aurora::Async { + struct IThreadPoolInternal; + void InitSched(); void DeinitSched(); void StartSched(); void StopSched(); - void Schedule(AuUInt64 ns, WorkerId_t target, AuSPtr runnable); - void TerminateSceduledTasks(WorkerId_t target); + void Schedule(AuUInt64 ns, IThreadPoolInternal *pool, WorkerId_t target, AuSPtr runnable); + void TerminateSceduledTasks(IThreadPoolInternal *pool, WorkerId_t target); } \ No newline at end of file diff --git a/Source/Async/ThreadPool.cpp b/Source/Async/ThreadPool.cpp new file mode 100644 index 00000000..fd9a4d31 --- /dev/null +++ b/Source/Async/ThreadPool.cpp @@ -0,0 +1,1061 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: ThreadPool.cpp + Date: 2021-10-30 + Author: Reece +***/ +#include +#include "Async.hpp" +#include "ThreadPool.hpp" +#include "WorkItem.hpp" +#include "Schedular.hpp" + +namespace Aurora::Async +{ + //STATIC_TLS(WorkerId_t, tlsWorkerId); + static thread_local AuWPtr gCurrentPool; + static const auto kMagicResortThreshold = 15; + + AUKN_SYM WorkerPId_t GetCurrentWorkerPId() + { + auto lkPool = gCurrentPool.lock(); + if (!lkPool) + { + LogWarn("s.."); + return {}; + } + auto cpy = *lkPool->tlsWorkerId; + auto lkPool2 = cpy.pool.lock(); + if (!lkPool2) + { + LogWarn(".."); + return {}; + } + if (lkPool2 != lkPool){ + LogWarn(".r."); + return {}; + } + return WorkerPId_t(lkPool, cpy); + } + + // + + ThreadPool::ThreadPool() + { + this->rwlock_ = AuThreadPrimitives::RWLockUnique(); + SysAssert(static_cast(this->rwlock_), "Couldn't initialize ThreadPool. Unable to allocate an RWLock"); + } + + // internal pool interface + + bool ThreadPool::WaitFor(WorkerId_t unlocker, const AuSPtr &primitive, AuUInt32 timeoutMs) + { + auto curThread = GetThreadState(); + + bool workerIdMatches = (unlocker.second == curThread->id.second) || ((unlocker.second == Async::kThreadIdAny) && (GetThreadWorkersCount(unlocker.first) == 1)); + + if ((unlocker.first == curThread->id.first) && // work group matches + (workerIdMatches)) // well, crap + { + + bool queryAsync = false; + while (!(queryAsync ? primitive->TryLock() : Threading::WaitFor(primitive.get(), 2))) + { + queryAsync = CtxYield(); + } + + return true; + } + else + { + return Threading::WaitFor(primitive.get(), timeoutMs); + } + } + + void ThreadPool::Run(WorkerId_t target, AuSPtr runnable) + { + auto state = GetGroup(target.first); + SysAssert(static_cast(state), "couldn't dispatch a task to an offline group"); + + IncrementTasksRunning(); + + { + AU_LOCK_GUARD(state->cvWorkMutex); + + #if defined(STAGING) || defined(DEBUG) + AU_LOCK_GUARD(rwlock_->AsReadable()); + + if (target.second != Async::kThreadIdAny) + { + auto itr = state->workers.find(target.second); + if ((itr == state->workers.end()) || (itr->second->rejecting)) + { + SysPushErrorGen("worker: {}:{} is offline", target.first, target.second); + DecrementTasksRunning(); + #if 0 + throw "Requested job worker is offline"; + #else + runnable->CancelAsync(); + return; + #endif + } + } + else + { + auto workers = state->workers; + bool found = false; + + for (const auto &worker : state->workers) + { + if (!worker.second->rejecting) + { + found = true; + break; + } + } + + if (!found) + { + DecrementTasksRunning(); + #if 0 + throw "No workers available"; + #else + runnable->CancelAsync(); + return; + #endif + } + } + #endif + + if (!AuTryInsert(state->workQueue, AuMakePair(target.second, runnable))) + { + runnable->CancelAsync(); + return; + } + + state->dirty++; + if (state->dirty > kMagicResortThreshold) + { + state->dirty = 0; + state->sorted = false; + } + state->eventLs->Set(); + } + if (target.second == Async::kThreadIdAny) + { + state->cvVariable->Signal(); + } + else + { + // sad :( + // TODO: when we have wait any, add support (^ the trigger) for it here + state->cvVariable->Broadcast(); + } + } + + IThreadPool *ThreadPool::ToThreadPool() + { + return this; + } + + void ThreadPool::IncrementTasksRunning() + { + this->tasksRunning_++; + } + + void ThreadPool::DecrementTasksRunning() + { + if ((--this->tasksRunning_) == 0) + { + if (InRunnerMode()) + { + Shutdown(); + } + } + } + + // ithreadpool + + size_t ThreadPool::GetThreadWorkersCount(ThreadGroup_t group) + { + AU_LOCK_GUARD(rwlock_->AsReadable()); + return GetGroup(group)->workers.size(); + } + + void ThreadPool::SetRunningMode(bool eventRunning) + { + this->runnersRunning_ = eventRunning; + } + + bool ThreadPool::Spawn(WorkerId_t workerId) + { + return Spawn(workerId, false); + } + + bool ThreadPool::Create(WorkerId_t workerId) + { + return Spawn(workerId, true); + } + + bool ThreadPool::InRunnerMode() + { + return this->runnersRunning_; + } + + bool ThreadPool::Poll() + { + return InternalRunOne(false); + } + + bool ThreadPool::RunOnce() + { + return InternalRunOne(true); + } + + bool ThreadPool::Run() + { + bool ranOnce {}; + + auto auThread = AuThreads::GetThread(); + auto job = GetThreadState(); + + while ((!auThread->Exiting()) && (!job->shuttingdown)) + { + // Do work (blocking) + InternalRunOne(true); + ranOnce = true; + } + + return ranOnce; + } + + bool ThreadPool::InternalRunOne(bool block) + { + auto state = GetThreadState(); + bool success {}; + + do + { + if (state->inLoopSourceMode) + { + success = PollLoopSource(block); + } + else + { + success = PollInternal(block); + success |= state->inLoopSourceMode; + } + } while (success); + + return success; + } + + bool ThreadPool::PollInternal(bool block) + { + auto state = GetThreadState(); + auto group = state->parent.lock(); + + //state->pendingWorkItems.clear(); + + auto magic = CtxPollPush(); + + { + AU_LOCK_GUARD(group->cvWorkMutex); + + if (group->workQueue.size() > 2) + { + if (!group->sorted && false) + { + auto cpy = group->workQueue; + + std::sort(group->workQueue.begin(), group->workQueue.end(), [&](const WorkEntry_t &a, const WorkEntry_t &b) + { + if (a.second->GetPrio() != b.second->GetPrio()) + return a.second->GetPrio() > b.second->GetPrio(); + + AuUInt32 ia {}, ib {}; + for (; ia < cpy.size(); ia++) + if (cpy[ia].second == a.second) + break; + + for (; ib < cpy.size(); ib++) + if (cpy[ib].second == b.second) + break; + + return ia < ib; + }); + + group->sorted = true; + group->dirty = 0; + } + } + + do + { + // Deque tasks the current thread runner could dipatch + // Noting that `multipopCount` determines how aggressive threads are in dequeuing work + // It's probable `multipopCount` will equal 1 for your use case + // + // Only increment when you know tasks within a group queue will not depend on one another + // *and* tasks require a small amount of execution time + // + // This could be potentially useful for an event dispatcher whereby you're dispatching + // hundreds of items per second, across a thread or two, knowing dequeuing one instead of all + // is a waste of CPU cycles. + // + // Remember, incrementing `multipopCount` is potentially dangerous the second you have local + // thread group waits + for (auto itr = group->workQueue.begin(); + ((itr != group->workQueue.end()) && + (state->pendingWorkItems.size() < state->multipopCount)); + ) + { + if (itr->first == Async::kThreadIdAny) + { + state->pendingWorkItems.push_back(*itr); + itr = group->workQueue.erase(itr); + continue; + } + + if ((itr->first != Async::kThreadIdAny) && (itr->first == state->id.second)) + { + state->pendingWorkItems.push_back(*itr); + itr = group->workQueue.erase(itr); + continue; + } + + itr++; + } + + // Consider blocking for more work + if (!block) + { + break; + } + + // Block if no work items are present + if (state->pendingWorkItems.empty()) + { + group->cvVariable->WaitForSignal(); + } + + // Post-wakeup thread terminating check + if (state->threadObject->Exiting() || state->shuttingdown) + { + break; + } + + } while (state->pendingWorkItems.empty()); + + if (group->workQueue.empty()) + { + group->eventLs->Reset(); + } + } + + if (state->pendingWorkItems.empty()) + { + CtxPollReturn(state, magic, false); + return false; + } + + int runningTasks {}; + + auto oldTlsHandle = std::exchange(gCurrentPool, AuSharedFromThis()); + + bool lowPrioCont {}; + bool lowPrioContCached {}; + + for (auto itr = state->pendingWorkItems.begin(); itr != state->pendingWorkItems.end(); ) + { + if (state->threadObject->Exiting() || state->shuttingdown) + { + break; + } + + // Set the last frame time for a watchdog later down the line + state->lastFrameTime = Time::CurrentClockMS(); + + if (itr->second->GetPrio() < 0.25) + { + if (lowPrioCont) continue; + + if (!lowPrioContCached) + { + AU_LOCK_GUARD(group->cvWorkMutex); + { + for (const auto &[pendingWorkA, pendingWorkB] : group->workQueue) + { + if (pendingWorkB->GetPrio() > .5) + { + lowPrioCont = true; + break; + } + } + } + + lowPrioContCached = true; + if (lowPrioCont) continue; + } + } + + // Dispatch + itr->second->RunAsync(); + + // Remove from our local job queue + itr = state->pendingWorkItems.erase(itr); + + // Atomically decrement global task counter + runningTasks = this->tasksRunning_.fetch_sub(1) - 1; + } + + gCurrentPool = oldTlsHandle; + + // Return popped work back to the groups work pool when our -pump loops were preempted + if (state->pendingWorkItems.size()) + { + AU_LOCK_GUARD(group->cvWorkMutex); + group->workQueue.insert(group->workQueue.end(), state->pendingWorkItems.begin(), state->pendingWorkItems.end()); + group->eventLs->Set(); + state->pendingWorkItems.clear(); + } + + CtxPollReturn(state, magic, true); + + if (InRunnerMode()) + { + if (runningTasks == 0) + { + Shutdown(); + } + } + + return true; + } + + bool ThreadPool::PollLoopSource(bool block) + { + auto state = GetThreadState(); + auto group = state->parent.lock(); + + //state->pendingWorkItems.clear(); + + auto magic = CtxPollPush(); + bool retValue {}; + + // TODO (reece): This function isn't very efficient + { + AU_LOCK_GUARD(group->cvWorkMutex); + + AuList curLoopReq = state->loopSources; + AuList> curLoopSources; + + auto lenLoopReqs = curLoopReq.size(); + + curLoopSources.resize(lenLoopReqs + 1); + + for (auto i = 0; i < lenLoopReqs; i++) + { + curLoopSources[i] = curLoopReq[i].loopSource; + } + + curLoopSources[lenLoopReqs] = group->eventLs; + + AuList> nextLoopSources; + if (block) + { + // TODO (reece): work on async epoll like abstraction + nextLoopSources = Loop::WaitMultipleObjects(curLoopSources, 0); + } + else + { + nextLoopSources.reserve(curLoopSources.size()); + for (const auto &source : curLoopSources) + { + if (source->IsSignaled()) + { + nextLoopSources.push_back(source); + } + } + } + + auto time = Time::CurrentClockMS(); + + state->loopSources.clear(); + state->loopSources.reserve(curLoopReq.size()); + + if (AuExists(nextLoopSources, group->eventLs)) + { + PollInternal(false); + } + + for (const auto &request : curLoopReq) + { + bool remove {}; + bool removeType {}; + + if (AuExists(nextLoopSources, request.loopSource)) + { + remove = true; + removeType = true; + } + else + { + if (request.requestedOffset) + { + if (request.endTime < time) + { + remove = true; + removeType = false; + } + } + } + + if (!remove) + { + state->loopSources.push_back(request); + } + else + { + request.callback(request.loopSource, removeType); + retValue |= removeType; + } + } + + state->inLoopSourceMode = state->loopSources.size(); + } + + return retValue; + } + + + void ThreadPool::Shutdown() + { + // Nested shutdowns can happen; prevent a write lock + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + if (this->shuttingdown_) + { + return; + } + } + + // Set shutdown flag + { + AU_LOCK_GUARD(this->rwlock_->AsWritable()); + if (std::exchange(this->shuttingdown_, true)) + { + return; + } + } + + // Noting + // 1) that StopSched may lockup under a writable lock + // -> we will terminate a thread that may be dispatching a sys pump event + // 2) that barrier doesn't need to be under a write lock + // + // Perform the following shutdown of the schedular and other available threads under a read lock + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + + StopSched(); + + for (auto &[groupId, group] : this->threads_) + { + for (auto &[id, worker] : group->workers) + { + Barrier(worker->id, 0, false, true); + } + } + } + + // Finally set the shutdown flag on all of our thread contexts + // then release them from the runners/workers list + // then release all group contexts + AuList threads; + { + AU_LOCK_GUARD(this->rwlock_->AsWritable()); + + for (auto &[groupId, group] : this->threads_) + { + for (auto &[id, worker] : group->workers) + { + worker->shuttingdown = true; + + if (groupId != 0) + { + worker->threadObject->SendExitSignal(); + threads.push_back(worker->threadObject); + } + + auto &event = worker->running; + if (event) + { + event->Set(); + } + } + + if (group->cvVariable) + { + AU_LOCK_GUARD(group->cvWorkMutex); + group->cvVariable->Broadcast(); + } + } + } + + // Sync to shutdown threads to prevent a race condition whereby the async subsystem shuts down before the threads + for (const auto &thread : threads) + { + thread->Exit(); + } + } + + bool ThreadPool::Exiting() + { + return shuttingdown_ || GetThreadState()->exiting; + } + + AuSPtr ThreadPool::NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) + { + if (!task) + { + return {}; + } + return AuMakeShared(this, worker, task, supportsBlocking); + } + + AuSPtr ThreadPool::NewFence() + { + return AuMakeShared(this, WorkerId_t{}, AuSPtr{}, true); + } + + AuThreads::ThreadShared_t ThreadPool::ResolveHandle(WorkerId_t id) + { + return GetThreadHandle(id)->threadObject; + } + + AuBST> ThreadPool::GetThreads() + { + AU_LOCK_GUARD(rwlock_->AsReadable()); + + AuBST> ret; + + for (const auto &group : this->threads_) + { + AuList workers; + + for (const auto &thread : group.second->workers) + { + workers.push_back(thread.second->id.second); + } + + ret[group.first] = workers; + } + + return ret; + } + + WorkerId_t ThreadPool::GetCurrentThread() + { + return tlsWorkerId; + } + + bool ThreadPool::Sync(WorkerId_t workerId, AuUInt32 timeoutMs, bool requireSignal) + { + AU_LOCK_GUARD(rwlock_->AsReadable()); + + auto group = GetGroup(workerId.first); + auto currentWorkerId = GetCurrentThread().second; + + if (workerId.second == Async::kThreadIdAny) + { + for (auto &jobWorker : group->workers) + { + if (!Barrier(jobWorker.second->id, timeoutMs, requireSignal && jobWorker.second->id.second != currentWorkerId, false)) // BAD!, should subtract time elapsed, clamp to, i dunno, 5ms min? + { + return false; + } + } + } + else + { + return Barrier(workerId, timeoutMs, requireSignal && workerId.second != currentWorkerId, false); + } + + return true; + } + + void ThreadPool::Signal(WorkerId_t workerId) + { + AU_LOCK_GUARD(rwlock_->AsReadable()); + + auto group = GetGroup(workerId.first); + if (workerId.second == Async::kThreadIdAny) + { + for (auto &jobWorker : group->workers) + { + jobWorker.second->running->Set(); + } + } + else + { + GetThreadHandle(workerId)->running->Set(); + } + } + + void ThreadPool::SyncAllSafe() + { + AU_LOCK_GUARD(rwlock_->AsReadable()); + + for (const auto &re : this->threads_) + { + for (auto &jobWorker : re.second->workers) + { + SysAssert(Barrier(jobWorker.second->id, 0, false, false)); + } + } + } + + void ThreadPool::AddFeature(WorkerId_t id, AuSPtr feature, bool async) + { + auto work = AuMakeShared(([=]() + { + GetThreadState()->features.push_back(feature); + feature->Init(); + })); + + auto workItem = this->NewWorkItem(id, work, !async)->Dispatch(); + + if (!async) + { + workItem->BlockUntilComplete(); + } + } + + void ThreadPool::AssertInThreadGroup(ThreadGroup_t group) + { + SysAssert(static_cast(tlsWorkerId).first == group); + } + + void ThreadPool::AssertWorker(WorkerId_t id) + { + SysAssert(static_cast(tlsWorkerId) == id); + } + + bool ThreadPool::ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) + { + auto thread = this->GetThreadHandle(workerId); + if (!thread) + { + return false; + } + + auto group = thread->parent.lock(); + + { + AU_LOCK_GUARD(group->cvWorkMutex); + + AsyncAppWaitSourceRequest req {}; + req.startTime = Time::CurrentClockMS(); + if (timeout) + { + req.requestedOffset = timeout; + req.endTime = req.startTime + timeout; + } + req.loopSource = loopSource; + req.callback = callback; + + if (!AuTryInsert(thread->loopSources, req)) + { + return false; + } + + thread->inLoopSourceMode = thread->loopSources.size(); + } + + return true; + } + + // Unimplemented fiber hooks, 'twas used for science + + int ThreadPool::CtxPollPush() + { + // TOOD (Reece): implement a context switching library + // Refer to the old implementation of this on pastebin + return 0; + } + + void ThreadPool::CtxPollReturn(const AuSPtr &state, int status, bool hitTask) + { + } + + bool ThreadPool::CtxYield() + { + bool ranAtLeastOne = false; + + while (this->InternalRunOne(false)) + { + ranAtLeastOne = true; + } + + return ranAtLeastOne; + } + + + // internal api + + bool ThreadPool::Spawn(WorkerId_t workerId, bool create) + { + AU_LOCK_GUARD(rwlock_->AsWritable()); + + if (GetCurrentWorkerPId().pool && create) + { + SysPushErrorGeneric("TODO (reece): add support for multiple runners per thread"); + return {}; + } + + AuSPtr group; + + // Try fetch or allocate group + { + AuSPtr* groupPtr; + if (!AuTryFind(this->threads_, workerId.first, groupPtr)) + { + group = AuMakeShared(); + + if (!group->Init()) + { + SysPushErrorMem("Not enough memory to intiialize a new group state"); + return false; + } + + if (!AuTryInsert(this->threads_, AuMakePair(workerId.first, group))) + { + return false; + } + } + else + { + group = *groupPtr; + } + } + + // Assert worker does not already exist + { + AuSPtr* ret; + + if (AuTryFind(group->workers, workerId.second, ret)) + { + SysPushErrorGen("Thread ID already exists"); + return false; + } + } + + auto threadState = AuMakeShared(); + threadState->parent = group; + threadState->running = AuThreadPrimitives::EventUnique(true, false, true); + threadState->syncSema = AuThreadPrimitives::SemaphoreUnique(0); + threadState->id = workerId; + //threadState->eventDriven = runner; + + if (!create) + { + threadState->threadObject = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(&ThreadPool::Entrypoint, this, threadState->id)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}), + gRuntimeConfig.async.threadPoolDefaultStackSize + )); + if (!threadState->threadObject) + { + return {}; + } + threadState->threadObject->Run(); + } + else + { + threadState->threadObject = AuSPtr(AuThreads::GetThread(), [](AuThreads::IAuroraThread *){}); + + // TODO: this is just a hack + // we should implement this properly + threadState->threadObject->AddLastHopeTlsHook(AuMakeShared([]() -> void + { + + }, []() -> void + { + auto pid = GetCurrentWorkerPId(); + if (pid.pool) + { + std::static_pointer_cast(pid.pool)->ThisExiting(); + } + })); + + // + gCurrentPool = AuWeakFromThis(); + tlsWorkerId = WorkerPId_t(AuSharedFromThis(), workerId); + } + + group->workers.insert(AuMakePair(workerId.second, threadState)); + return true; + } + + // private api + + bool ThreadPool::Barrier(WorkerId_t workerId, AuUInt32 ms, bool requireSignal, bool drop) + { + // TODO: barrier multiple + auto &semaphore = GetThreadState()->syncSema; + auto unsafeSemaphore = semaphore.get(); + bool failed {}; + + auto work = AuMakeShared( + [=]() + { + auto state = GetThreadState(); + + if (drop) + { + state->rejecting = true; + } + + if (requireSignal) + { + state->running->Reset(); + } + + unsafeSemaphore->Unlock(1); + + if (requireSignal) + { + state->running->Lock(); + } + }, + [&]() + { + unsafeSemaphore->Unlock(1); + failed = true; + } + ); + + if (!work) + { + return false; + } + + Run(workerId, work); + + return WaitFor(workerId, AuUnsafeRaiiToShared(semaphore), ms) && !failed; + } + + void ThreadPool::Entrypoint(WorkerId_t id) + { + gCurrentPool = AuWeakFromThis(); + tlsWorkerId = WorkerPId_t(AuSharedFromThis(), id); + + auto job = GetThreadState(); + + Run(); + + if (id != WorkerId_t {0, 0}) + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + + if (!this->shuttingdown_ && !job->rejecting) + { + // Pump and barrier + reject all after atomically + Barrier(id, 0, false, true); + } + } + + ThisExiting(); + + if (id == WorkerId_t {0, 0}) + { + CleanWorkerPoolReservedZeroFree(); + } + } + + void ThreadPool::ThisExiting() + { + auto id = GetCurrentThread(); + auto state = GetGroup(id.first); + + { + AU_LOCK_GUARD(this->rwlock_->AsWritable()); + + auto itr = state->workers.find(id.second); + auto &jobWorker = itr->second; + + CleanUpWorker(id); + + // Abort scheduled tasks + TerminateSceduledTasks(this, id); + + // Clean up thread features + // -> transferable TLS handles + // -> thread specific vms + // -> anything your brain wishes to imagination + for (const auto &thread : jobWorker->features) + { + try + { + thread->Cleanup(); + } + catch (...) + { + LogWarn("Couldn't clean up thread feature!"); + Debug::PrintError(); + } + } + + jobWorker->features.clear(); + + state->workers.erase(itr); + } + } + + AuSPtr ThreadPool::GetGroup(ThreadGroup_t type) + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + AuSPtr* ret; + if (!AuTryFind(this->threads_, type, ret)) + { + return {}; + } + return *ret; + } + + AuSPtr ThreadPool::GetThreadState() + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + auto id = GetCurrentThread(); + auto state = GetGroup(id.first); + return state->workers[id.second]; + } + + AuSPtr ThreadPool::GetThreadHandle(WorkerId_t id) + { + AU_LOCK_GUARD(this->rwlock_->AsReadable()); + + auto group = GetGroup(id.first); + if (!group) + { + return {}; + } + + AuSPtr *ret; + if (!AuTryFind(group->workers, id.second, ret)) + { + return {}; + } + + return *ret; + } + + AUKN_SYM AuSPtr NewThreadPool() + { + // apps that don't require async shouldn't be burdened with the overhead of this litl spiner + StartSched(); + return AuMakeShared(); + } +} \ No newline at end of file diff --git a/Source/Async/ThreadPool.hpp b/Source/Async/ThreadPool.hpp new file mode 100644 index 00000000..800c714a --- /dev/null +++ b/Source/Async/ThreadPool.hpp @@ -0,0 +1,127 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: ThreadPool.hpp + Date: 2021-10-30 + Author: Reece +***/ +#pragma once + +namespace Aurora::Async +{ + struct GroupState; + struct ThreadState; + //class WorkItem; + + struct IThreadPoolInternal + { + virtual bool WaitFor(WorkerId_t unlocker, const AuSPtr &primitive, AuUInt32 ms) = 0; + virtual void Run(WorkerId_t target, AuSPtr runnable) = 0; + virtual IThreadPool *ToThreadPool() = 0; + virtual void IncrementTasksRunning() = 0; + virtual void DecrementTasksRunning() = 0; + }; + + + struct ThreadPool : public IThreadPool, public IThreadPoolInternal, std::enable_shared_from_this + { + ThreadPool(); + + // IThreadPoolInternal + bool WaitFor(WorkerId_t unlocker, const AuSPtr &primitive, AuUInt32 ms) override; + void Run(WorkerId_t target, AuSPtr runnable) override; + IThreadPool *ToThreadPool() override; + void IncrementTasksRunning() override; + void DecrementTasksRunning() override; + + // IThreadPool + virtual bool Spawn(WorkerId_t workerId) override; + + virtual void SetRunningMode(bool eventRunning) override; + + virtual bool Create(WorkerId_t workerId) override; + + virtual bool InRunnerMode() override; + + virtual bool Poll() override; + virtual bool RunOnce() override; + virtual bool Run() override; + + virtual void Shutdown() override; + virtual bool Exiting() override; + + virtual AuSPtr NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) override; + virtual AuSPtr NewFence() override; + + virtual Threading::Threads::ThreadShared_t ResolveHandle(WorkerId_t) override; + + virtual AuBST> GetThreads() override; + + virtual WorkerId_t GetCurrentThread() override; + + virtual bool Sync(WorkerId_t workerId, AuUInt32 timeoutMs, bool requireSignal) override; + virtual void Signal(WorkerId_t workerId) override; + virtual void SyncAllSafe() override; + + virtual void AddFeature(WorkerId_t id, AuSPtr feature, bool async) override; + + virtual void AssertInThreadGroup(ThreadGroup_t group) override; + virtual void AssertWorker(WorkerId_t id) override; + + virtual bool ScheduleLoopSource(const AuSPtr &loopSource, WorkerId_t workerId, AuUInt32 timeout, const AuConsumer, bool> &callback) override; + + // Internal API + + bool Spawn(WorkerId_t workerId, bool create); + + bool InternalRunOne(bool block); + bool PollInternal(bool block); + bool PollLoopSource(bool block); + + size_t GetThreadWorkersCount(ThreadGroup_t group); + + virtual void CleanUpWorker(WorkerId_t wid) {}; + virtual void CleanWorkerPoolReservedZeroFree() {}; // calls shutdown under async apps + + // Secret old fiber api + bool CtxYield(); + int CtxPollPush(); + void CtxPollReturn(const AuSPtr &state, int status, bool hitTask); + + // TLS handle + struct WorkerWPId_t : WorkerId_t + { + WorkerWPId_t() + {} + + WorkerWPId_t(const WorkerPId_t &ref) : WorkerId_t(ref.first, ref.second), pool(ref.pool) + {} + + AuWPtr pool; + }; + + AuThreads::TLSVariable tlsWorkerId; + + private: + // TODO: BarrierMultiple + bool Barrier(WorkerId_t, AuUInt32 ms, bool requireSignal, bool drop); + + protected: + void Entrypoint(WorkerId_t id); + + private: + void ThisExiting(); + + AuSPtr GetGroup(ThreadGroup_t type); + AuSPtr GetThreadState(); + AuSPtr GetThreadHandle(WorkerId_t id); + + using ThreadDb_t = AuBST>; + + ThreadDb_t threads_; + bool shuttingdown_ {}; + AuThreadPrimitives::RWLockUnique_t rwlock_; + std::atomic_int tasksRunning_; + bool runnersRunning_; + }; +} \ No newline at end of file diff --git a/Source/Async/ThreadState.hpp b/Source/Async/ThreadState.hpp new file mode 100644 index 00000000..85f0cfe0 --- /dev/null +++ b/Source/Async/ThreadState.hpp @@ -0,0 +1,56 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: ThreadState.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +#include "AsyncRunnable.hpp" + +namespace Aurora::Async +{ + using WorkEntry_t = AuPair>; + + // TODO: this is a hack because i havent implemented an epoll abstraction yet + struct AsyncAppWaitSourceRequest + { + AuConsumer, bool> callback; + AuSPtr loopSource; + AuUInt32 requestedOffset; + AuUInt64 startTime; + AuUInt64 endTime; + }; + + struct GroupState; + + struct ThreadState + { + WorkerId_t id; + + //bool eventDriven {}; + + AuUInt8 multipopCount = 1; + AuUInt32 lastFrameTime {}; + + AuThreads::ThreadShared_t threadObject; + + //std::stack jmpStack; + AuWPtr parent; + + AuThreadPrimitives::SemaphoreUnique_t syncSema; + AuList> features; + + bool rejecting {}; + bool exiting {}; + bool inLoopSourceMode {}; + bool shuttingdown {}; + + AuThreadPrimitives::EventUnique_t running; + //bool running; + + AuList loopSources; + AuList pendingWorkItems; + }; +} \ No newline at end of file diff --git a/Source/Async/WorkItem.cpp b/Source/Async/WorkItem.cpp index f9110266..e5f5ca32 100644 --- a/Source/Async/WorkItem.cpp +++ b/Source/Async/WorkItem.cpp @@ -13,12 +13,12 @@ namespace Aurora::Async { - WorkItem::WorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) : worker_(worker), task_(task) + WorkItem::WorkItem(IThreadPoolInternal *owner, const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) : worker_(worker), task_(task), owner_(owner) { if (supportsBlocking) { - finishedEvent_ = AuThreadPrimitives::EventUnique(false, true, true); - SysAssert(finishedEvent_); + this->finishedEvent_ = AuThreadPrimitives::EventUnique(false, true, true); + SysAssert(this->finishedEvent_); } } @@ -34,7 +34,7 @@ namespace Aurora::Async { auto dependency = std::reinterpret_pointer_cast(workItem); - AU_LOCK_GUARD(lock); + AU_LOCK_GUARD(this->lock); AU_LOCK_GUARD(dependency->lock); if (dependency->HasFailed()) @@ -43,7 +43,7 @@ namespace Aurora::Async } dependency->waiters_.push_back(shared_from_this()); - waitOn_.push_back(workItem); + this->waitOn_.push_back(workItem); } if (status) @@ -53,14 +53,13 @@ namespace Aurora::Async return AU_SHARED_FROM_THIS; } - AuSPtr WorkItem::WaitFor(const AuList> &workItems) { bool status {}; { - AU_LOCK_GUARD(lock); + AU_LOCK_GUARD(this->lock); for (auto &workItem : workItems) { @@ -73,7 +72,7 @@ namespace Aurora::Async } dependency->waiters_.push_back(shared_from_this()); - waitOn_.push_back(workItem); + this->waitOn_.push_back(workItem); } } @@ -95,25 +94,37 @@ namespace Aurora::Async AuSPtr WorkItem::SetSchedTimeNs(AuUInt64 ns) { - dispatchTimeNs_ = Time::CurrentClockNS() + ns; + this->dispatchTimeNs_ = Time::CurrentClockNS() + ns; + return AU_SHARED_FROM_THIS; + } + + AuSPtr WorkItem::SetSchedTimeAbs(AuUInt32 ms) + { + this->dispatchTimeNs_ = AuUInt64(ms) * AuUInt64(1000000); + return AU_SHARED_FROM_THIS; + } + + AuSPtr WorkItem::SetSchedTimeNsAbs(AuUInt64 ns) + { + this->dispatchTimeNs_ = ns; return AU_SHARED_FROM_THIS; } AuSPtr WorkItem::SetSchedTime(AuUInt32 ms) { - dispatchTimeNs_ = Time::CurrentClockNS() + (AuUInt64(ms) * AuUInt64(1000000)); + this->dispatchTimeNs_ = Time::CurrentClockNS() + (AuUInt64(ms) * AuUInt64(1000000)); return AU_SHARED_FROM_THIS; } AuSPtr WorkItem::AddDelayTime(AuUInt32 ms) { - delayTimeNs_ += AuUInt64(ms) * AuUInt64(1000000); + this->delayTimeNs_ += AuUInt64(ms) * AuUInt64(1000000); return AU_SHARED_FROM_THIS; } AuSPtr WorkItem::AddDelayTimeNs(AuUInt64 ns) { - delayTimeNs_ += ns; + this->delayTimeNs_ += ns; return AU_SHARED_FROM_THIS; } @@ -129,7 +140,7 @@ namespace Aurora::Async if (check) { - if (dispatchPending_) + if (this->dispatchPending_) { return; } @@ -152,9 +163,9 @@ namespace Aurora::Async itr = waitOn_.erase(itr); } - dispatchPending_ = true; + this->dispatchPending_ = true; - if (Time::CurrentClockNS() < dispatchTimeNs_) + if (Time::CurrentClockNS() < this->dispatchTimeNs_) { Schedule(); return; @@ -162,7 +173,7 @@ namespace Aurora::Async if (auto delay = std::exchange(delayTimeNs_, {})) { - dispatchTimeNs_ = delay + Time::CurrentClockNS(); + this->dispatchTimeNs_ = delay + Time::CurrentClockNS(); Schedule(); return; } @@ -170,21 +181,32 @@ namespace Aurora::Async SendOff(); } + float WorkItem::GetPrio() + { + return prio_; + } + + void WorkItem::SetPrio(float val) + { + prio_ = val; + } + void WorkItem::CancelAsync() { - AU_LOCK_GUARD(lock); + AU_LOCK_GUARD(this->lock); Fail(); } void WorkItem::RunAsync() { - AU_LOCK_GUARD(lock); + AU_LOCK_GUARD(this->lock); IWorkItemHandler::ProcessInfo info(true); + info.pool = this->owner_->ToThreadPool(); - if (task_) + if (this->task_) { - task_->DispatchFrame(info); + this->task_->DispatchFrame(info); } switch (info.type) @@ -225,13 +247,13 @@ namespace Aurora::Async } } - finished = true; - if (finishedEvent_) + this->finished = true; + if (this->finishedEvent_) { - finishedEvent_->Set(); + this->finishedEvent_->Set(); } - for (auto &waiter : waiters_) + for (auto &waiter : this->waiters_) { std::reinterpret_pointer_cast(waiter)->DispatchEx(true); } @@ -246,59 +268,69 @@ namespace Aurora::Async task_->Shutdown(); } - for (auto &waiter : waiters_) + for (auto &waiter : this->waiters_) { std::reinterpret_pointer_cast(waiter)->Fail(); } - waiters_.clear(); - waitOn_.clear(); + this->waiters_.clear(); + this->waitOn_.clear(); - if (finishedEvent_) + if (this->finishedEvent_) { - finishedEvent_->Set(); + this->finishedEvent_->Set(); } } bool WorkItem::BlockUntilComplete() { - if (!finishedEvent_) return false; - return static_cast(GetAsyncApp())->WaitFor(this->worker_, AuUnsafeRaiiToShared(finishedEvent_), 0); + if (!this->finishedEvent_) return false; + return this->owner_->WaitFor(this->worker_, AuUnsafeRaiiToShared(this->finishedEvent_), 0); } bool WorkItem::HasFinished() { - return finished; + return this->finished; } void WorkItem::Cancel() { - AU_LOCK_GUARD(lock); + AU_LOCK_GUARD(this->lock); Fail(); } bool WorkItem::HasFailed() { - return failed; + return this->failed; } void WorkItem::Schedule() { - Async::Schedule(dispatchTimeNs_, worker_, this->shared_from_this()); + Async::Schedule(this->dispatchTimeNs_, this->owner_, this->worker_, this->shared_from_this()); } void WorkItem::SendOff() { - if (!task_) + if (!this->task_) { // If we aren't actually calling a task interface, we may as well just dispatch objects waiting on us from here RunAsync(); } else { - static_cast(GetAsyncApp())->Run(worker_, this->shared_from_this()); + this->owner_->Run(this->worker_, this->shared_from_this()); } } + + static auto GetWorkerInternal(const AuSPtr &pool) + { + return std::static_pointer_cast(pool).get(); + } + + static auto GetWorkerInternal() + { + return static_cast(GetAsyncApp()); + } AUKN_SYM AuSPtr NewWorkItem(const WorkerId_t &worker, const AuSPtr &task, bool supportsBlocking) { @@ -306,12 +338,28 @@ namespace Aurora::Async { return {}; } - return AuMakeShared(worker, task, supportsBlocking); + + return AuMakeShared(GetWorkerInternal(), worker, task, supportsBlocking); + } + + AUKN_SYM AuSPtr NewWorkItem(const WorkerPId_t &worker, const AuSPtr &task, bool supportsBlocking) + { + if (!task) + { + return {}; + } + + if (!worker.pool) + { + return {}; + } + + return AuMakeShared(GetWorkerInternal(), worker, task, supportsBlocking); } AUKN_SYM AuSPtr NewFence() { - return AuMakeShared(WorkerId_t{}, AuSPtr{}, true); + return AuMakeShared(GetWorkerInternal(), WorkerId_t{}, AuSPtr{}, true); } void *WorkItem::GetPrivateData() diff --git a/Source/Async/WorkItem.hpp b/Source/Async/WorkItem.hpp index 58215bc8..cc7315b9 100644 --- a/Source/Async/WorkItem.hpp +++ b/Source/Async/WorkItem.hpp @@ -7,12 +7,14 @@ ***/ #pragma once +#include "ThreadPool.hpp" + namespace Aurora::Async { class WorkItem : public IWorkItem, public IAsyncRunnable, public std::enable_shared_from_this { public: - WorkItem(const WorkerId_t &worker_, const AuSPtr &task_, bool supportsBlocking); + WorkItem(IThreadPoolInternal *owner, const WorkerId_t &worker_, const AuSPtr &task_, bool supportsBlocking); ~WorkItem(); AuSPtr WaitFor(const AuSPtr &workItem) override; @@ -21,6 +23,8 @@ namespace Aurora::Async AuSPtr SetSchedTimeNs(AuUInt64 ns) override; AuSPtr AddDelayTime(AuUInt32 ms) override; AuSPtr AddDelayTimeNs(AuUInt64 ns) override; + AuSPtr SetSchedTimeAbs(AuUInt32 ms) override; + AuSPtr SetSchedTimeNsAbs(AuUInt64 ns) override; AuSPtr Then(const AuSPtr &next) override; AuSPtr Dispatch() override; @@ -38,10 +42,14 @@ namespace Aurora::Async void *GetPrivateData() override; AuOptional ToWorkResultT() override; + float GetPrio() override; + void SetPrio(float val) override; + private: void DispatchEx(bool check); AuSPtr task_; WorkerId_t worker_; + float prio_ = 0.5f; AuList> waitOn_; AuList> waiters_; AuThreadPrimitives::SpinLock lock; @@ -52,10 +60,10 @@ namespace Aurora::Async bool dispatchPending_ {}; AuUInt64 dispatchTimeNs_ {}; AuUInt64 delayTimeNs_ {}; + IThreadPoolInternal *owner_ {}; void Fail(); void Schedule(); void SendOff(); }; - } \ No newline at end of file diff --git a/Source/Console/Commands/Commands.cpp b/Source/Console/Commands/Commands.cpp index 92478a37..92a90f9d 100644 --- a/Source/Console/Commands/Commands.cpp +++ b/Source/Console/Commands/Commands.cpp @@ -13,12 +13,12 @@ namespace Aurora::Console::Commands struct Command; struct CommandDispatch; - static AuHashMap gCommands; - static AuList gLineCallbacks; - static AuList gPendingCommands; - static auto gMutex = AuThreadPrimitives::MutexUnique(); - static auto gPendingCommandsMutex = AuThreadPrimitives::MutexUnique(); - static AuOptional gCommandDispatcher; + static AuHashMap gCommands; + static AuList gLineCallbacks; + static AuList gPendingCommands; + static auto gMutex = AuThreadPrimitives::MutexUnique(); + static auto gPendingCommandsMutex = AuThreadPrimitives::MutexUnique(); + static Async::WorkerPId_t gCommandDispatcher; struct Command { @@ -45,7 +45,7 @@ namespace Aurora::Console::Commands eAsync }; - static bool Dispatch(const AuString &string, EDispatchType type, Async::WorkerId_t workerId) + static bool Dispatch(const AuString &string, EDispatchType type, Async::WorkerPId_t workerId) { Parse::ParseResult res; AuSPtr callback; @@ -105,7 +105,7 @@ namespace Aurora::Console::Commands } else { - Async::DispatchBasicWorkCallback(workerId, + Async::DispatchWork(workerId, Async::TaskFromConsumerRefT([](const CommandDispatch &dispatch) -> void { dispatch.callback->OnCommand(dispatch.arguments); @@ -134,28 +134,15 @@ namespace Aurora::Console::Commands { return Dispatch(string, EDispatchType::eNow, {}); } - + AUKN_SYM bool DispatchCommandToAsyncRunner(const AuString &string, Async::WorkerId_t id) { - return Dispatch(string, EDispatchType::eAsync, id); + return Dispatch(string, EDispatchType::eAsync, AuAsync::WorkerPId_t(AuUnsafeRaiiToShared(AuAsync::GetAsyncApp()), id)); } - static AuSPtr gExternalLineProcessor; - - AUKN_SYM bool DispatchRawLine(const AuString &string) + AUKN_SYM bool DispatchCommandToAsyncRunner(const AuString &string, Async::WorkerPId_t id) { - if (gExternalLineProcessor) - { - gExternalLineProcessor->OnProcessedLineUTF8(string); - return true; - } - - return DispatchCommand(string); - } - - AUKN_SYM void SetCallbackAndDisableCmdProcessing(const AuSPtr &subscriber) - { - gExternalLineProcessor = subscriber; + return Dispatch(string, EDispatchType::eAsync, id); } void UpdateDispatcher(AuOptional target) @@ -173,7 +160,7 @@ namespace Aurora::Console::Commands } } - gCommandDispatcher = target; + gCommandDispatcher = Async::WorkerPId_t(AuUnsafeRaiiToShared(AuAsync::GetAsyncApp()), target.value()); } static void DispatchCommandsFromThis(const AuList &commands) @@ -192,13 +179,16 @@ namespace Aurora::Console::Commands auto commands = std::exchange(gPendingCommands, {}); gPendingCommandsMutex->Unlock(); - if (gCommandDispatcher.value_or(Async::WorkerId_t{}) == Async::WorkerId_t{}) + if ((gCommandDispatcher.pool == nullptr) || + ((gCommandDispatcher.pool.get() == Async::GetAsyncApp()) && + (gCommandDispatcher == Async::WorkerId_t{}))) { DispatchCommandsFromThis(commands); } else { - Async::NewWorkItem(gCommandDispatcher.value(), + + NewWorkItem(gCommandDispatcher, AuMakeShared([&commands]() { DispatchCommandsFromThis(commands); diff --git a/Source/Console/Console.cpp b/Source/Console/Console.cpp index e41824df..eccc4e32 100644 --- a/Source/Console/Console.cpp +++ b/Source/Console/Console.cpp @@ -26,11 +26,22 @@ namespace Aurora::Console return ConsoleStd::ReadStdIn(buffer, length); } - AUKN_SYM AuUInt32 WriteStdIn(const void *buffer, AuUInt32 length) + AUKN_SYM AuUInt32 WriteStdOut(const void *buffer, AuUInt32 length) { return ConsoleStd::WriteStdOut(buffer, length); } + AUKN_SYM bool DispatchRawLine(const AuString &string) + { + if (Hooks::gExternalLineProcessor) + { + Hooks::gExternalLineProcessor->OnProcessedLineUTF8(string); + return true; + } + + return Commands::DispatchCommand(string); + } + void Init() { Hooks::Init(); diff --git a/Source/Console/ConsoleFIO/ConsoleFIO.cpp b/Source/Console/ConsoleFIO/ConsoleFIO.cpp index bc454f87..ba2d4013 100644 --- a/Source/Console/ConsoleFIO/ConsoleFIO.cpp +++ b/Source/Console/ConsoleFIO/ConsoleFIO.cpp @@ -10,9 +10,9 @@ namespace Aurora::Console::ConsoleFIO { - static AuList gLogBuffer; + static AuList gLogBuffer; static AuThreadPrimitives::RWLockUnique_t gLogMutex; - static IO::FS::OpenWriteUnique_t gFileHandle; + static AuIOFS::OpenWriteUnique_t gFileHandle; static const auto &gLogConfig = gRuntimeConfig.console.fio; @@ -21,7 +21,7 @@ namespace Aurora::Console::ConsoleFIO AuString path; AuString procName; - if (!IO::FS::GetProfileDomain(path)) + if ((gLogConfig.writeLogsToUserDir) || (!IO::FS::GetProfileDomain(path))) { path = "."; } diff --git a/Source/Console/ConsoleStd/ConsoleStd.cpp b/Source/Console/ConsoleStd/ConsoleStd.cpp index 4ff74f5c..f1b74549 100644 --- a/Source/Console/ConsoleStd/ConsoleStd.cpp +++ b/Source/Console/ConsoleStd/ConsoleStd.cpp @@ -32,16 +32,16 @@ namespace Aurora::Console::ConsoleStd #if defined(ENABLE_STD_CONSOLE) #if defined(AURORA_IS_MODERNNT_DERIVED) - using StreamHandle_t = HANDLE; - #define IS_STREAM_HANDLE_VALID(h) (h != INVALID_HANDLE_VALUE) #define DEFAULT_HANDLE_VAL INVALID_HANDLE_VALUE - + using StreamHandle_t = HANDLE; + static StreamHandle_t gWin32Thread = INVALID_HANDLE_VALUE; #elif defined(IO_POSIX_STREAMS) - #define IS_STREAM_HANDLE_VALID(h) (h != 0) - #define DEFAULT_HANDLE_VAL 0 + #define DEFAULT_HANDLE_VAL 0xFFFFFFFF using StreamHandle_t = int; -#endif +#endif + #define IS_STREAM_HANDLE_VALID(h) (h != DEFAULT_HANDLE_VAL) + static bool AsyncReadAnyOrReadStreamBlock(); static const AuMach kLineBufferMax = 2048; @@ -229,12 +229,12 @@ namespace Aurora::Console::ConsoleStd if (GetConsoleWindow() == NULL) { - if (!gRuntimeConfig.console.forceConsoleWindow) + if (!gRuntimeConfig.console.enableConsole) { return; } - - if (gRuntimeConfig.console.disableAllConsoles) + + if (!gRuntimeConfig.console.forceConsoleWindow) { return; } @@ -353,7 +353,7 @@ namespace Aurora::Console::ConsoleStd if (line.size()) { - Console::Commands::DispatchRawLine(line); + Console::DispatchRawLine(line); } } diff --git a/Source/Console/ConsoleWxWidgets/ConsoleWxWidgets.cpp b/Source/Console/ConsoleWxWidgets/ConsoleWxWidgets.cpp index a8d11b95..81ffcab4 100644 --- a/Source/Console/ConsoleWxWidgets/ConsoleWxWidgets.cpp +++ b/Source/Console/ConsoleWxWidgets/ConsoleWxWidgets.cpp @@ -149,7 +149,7 @@ void ConsoleFrame::OnCmd(wxCommandEvent &event) textbox->Clear(); - Aurora::Console::Commands::DispatchRawLine(line); + Aurora::Console::DispatchRawLine(line); } WxSplitterLine *ConsoleFrame::NewSplitter(wxSize splitter, wxColor color) @@ -266,6 +266,8 @@ ConsoleFrame::ConsoleFrame(const wxString &title, const wxPoint &pos, const wxSi // \_/\_/ \__,_|_| \___|_| |_|_| |_| |_|\___||___/ \__,_|_| |_|\___|\__,_|\__,_| (_) (_) (_) (_) // // + // I dont even care if this function leaks on failure. sucks to be you, i dont care. + // valgrind can fail when your system is running low on resources at start up static const auto kEnableDark = (useWin32DarkHack) || @@ -637,7 +639,7 @@ namespace Aurora::Console::ConsoleWxWidgets static bool UseWxConsole() { - if (gRuntimeConfig.console.disableAllConsoles) + if (!gRuntimeConfig.console.enableConsole) { return false; } @@ -722,7 +724,7 @@ namespace Aurora::Console::ConsoleWxWidgets } - static Aurora::Threading::Threads::ThreadUnique_t gWxWidgetsThread; + static AuThreads::ThreadUnique_t gWxWidgetsThread; static void WxWidgetsThreadMain() { @@ -770,18 +772,17 @@ namespace Aurora::Console::ConsoleWxWidgets if (std::exchange(gConsoleStarted, true)) return; gMutex = AuThreadPrimitives::MutexUnique(); + if (!gMutex) return; Aurora::Console::Hooks::AddSubscription(AuUnsafeRaiiToShared(&gConsoleMessageSubscriber)); - gWxWidgetsThread = AuThreading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(WxWidgetsThreadMain)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}), + gWxWidgetsThread = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(WxWidgetsThreadMain)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}), "WxWidgets" )); - if (!gWxWidgetsThread) - { - return; - } + if (!gWxWidgetsThread) return; + gWxWidgetsThread->Run(); } diff --git a/Source/Console/Flusher.cpp b/Source/Console/Flusher.cpp index f9ac3835..4f6961c3 100644 --- a/Source/Console/Flusher.cpp +++ b/Source/Console/Flusher.cpp @@ -12,9 +12,9 @@ namespace Aurora::Console { - static Threading::Threads::ThreadUnique_t gWriterThread; + static AuThreads::ThreadUnique_t gWriterThread; - class ShutdownFlushHook : public Threading::Threads::IThreadFeature + class ShutdownFlushHook : public AuThreads::IThreadFeature { public: void Init() override; @@ -38,7 +38,7 @@ namespace Aurora::Console static void LogThreadEP() { - auto thread = Threading::Threads::GetThread(); + auto thread = AuThreads::GetThread(); SlowStartupTasks(); @@ -58,9 +58,9 @@ namespace Aurora::Console { // Startup a runner thread that will take care of all the stress inducing IO every so often on a remote thread - gWriterThread = Threading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(LogThreadEP)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}), + gWriterThread = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(LogThreadEP)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}), "CasualConsoleAsyncWritter" )); if (!gWriterThread) @@ -73,7 +73,7 @@ namespace Aurora::Console void InitFlusher() { // Add a 'ShutdownFlushHook' object to the main threads TLS hook - Threading::Threads::GetThread()->AddLastHopeTlsHook(AuMakeShared()); + AuThreads::GetThread()->AddLastHopeTlsHook(AuMakeShared()); InitFlushThread(); } diff --git a/Source/Console/Hooks/Hooks.cpp b/Source/Console/Hooks/Hooks.cpp index ebb7a696..92c52683 100644 --- a/Source/Console/Hooks/Hooks.cpp +++ b/Source/Console/Hooks/Hooks.cpp @@ -32,11 +32,16 @@ namespace Aurora::Console::Hooks AuTryInsert(gLineFunctionalCallbacks, hook); } + AUKN_SYM void SetCallbackAndDisableCmdProcessing(const AuSPtr &subscriber) + { + gExternalLineProcessor = subscriber; + } + void WriteLine(const ConsoleMessage &msg) { AU_LOCK_GUARD(gMutex); - if (msg.line.find('\n') == std::string::npos) [[likely]] + if (msg.line.find('\n') == AuString::npos) [[likely]] { for (const auto &callback : gLineFunctionalCallbacks) { diff --git a/Source/Console/Hooks/Hooks.hpp b/Source/Console/Hooks/Hooks.hpp index 9cfd7354..452066b9 100644 --- a/Source/Console/Hooks/Hooks.hpp +++ b/Source/Console/Hooks/Hooks.hpp @@ -9,6 +9,8 @@ namespace Aurora::Console::Hooks { + inline AuSPtr gExternalLineProcessor; + void Deinit(); void Init(); void WriteLine(const ConsoleMessage &msg); diff --git a/Source/Debug/ExceptionWatcher.Win32.cpp b/Source/Debug/ExceptionWatcher.Win32.cpp index 1ea71edf..b45c803f 100644 --- a/Source/Debug/ExceptionWatcher.Win32.cpp +++ b/Source/Debug/ExceptionWatcher.Win32.cpp @@ -91,9 +91,15 @@ namespace Aurora::Debug } } - if (SymFromAddr(process, (ULONG64)stack.AddrPC.Offset, &displacement, pSymbol)) + try + { + //if (SymFromAddr(process, (ULONG64)stack.AddrPC.Offset, &displacement, pSymbol)) + { + //frameCurrent.label = pSymbol->Name; + } + } + catch (...) { - frameCurrent.label = pSymbol->Name; } #if defined(DEBUG) || defined(STAGING) @@ -193,6 +199,7 @@ namespace Aurora::Debug SymInitialize(GetCurrentProcess(), NULL, TRUE); #endif + // This is really gross :( AddVectoredExceptionHandler(1, [](_EXCEPTION_POINTERS *ExceptionInfo) -> LONG { @@ -240,14 +247,13 @@ namespace Aurora::Debug const auto catchableTypeArray = reinterpret_cast(reinterpret_cast(handle) + static_cast(throwInfo->pCatchableTypeArray)); AuString suffix; - bool derivedFromException = {}; for (int i = 0; i < catchableTypeArray->nCatchableTypes; i++) { const auto type = reinterpret_cast (reinterpret_cast(handle) + static_cast(catchableTypeArray->arrayOfCatchableTypes[i])); const auto descriptor = reinterpret_cast (reinterpret_cast(handle) + static_cast(type->pType)); - entry.wincxx.str += (i == 0 ? "" : AuString(", ")) + descriptor->name(); + entry.wincxx.str += (i == 0 ? "" : AuString(", ")) + descriptor->name(); // __std_type_info_name if (strnicmp(descriptor->raw_name(), ".?AVException@", AuArraySize(".?AVException@") - 1) == 0) { @@ -258,7 +264,7 @@ namespace Aurora::Debug suffix = wptr; } } - else if (strncmp(descriptor->raw_name(), ".PEAD", AuArraySize(".PEAD") - 1) == 0) + else if (strncmp(descriptor->raw_name(), ".PEAD", AuArraySize(".PEAD")) == 0) { auto possibleStringPointer = reinterpret_cast(ExceptionInfo->ExceptionRecord->ExceptionInformation[1]); if (IsReadable(possibleStringPointer)) @@ -272,7 +278,7 @@ namespace Aurora::Debug } else if (strncmp(descriptor->raw_name(), kStringRawName.data(), kStringRawName.size()) == 0) { - auto possibleStdStringPointer = reinterpret_cast(ExceptionInfo->ExceptionRecord->ExceptionInformation[1]); + auto possibleStdStringPointer = reinterpret_cast(ExceptionInfo->ExceptionRecord->ExceptionInformation[1]); if (IsReadable(possibleStdStringPointer)) { auto string = *possibleStdStringPointer; diff --git a/Source/Entrypoint.cpp b/Source/Entrypoint.cpp index defb6d46..cace9dfa 100644 --- a/Source/Entrypoint.cpp +++ b/Source/Entrypoint.cpp @@ -55,10 +55,82 @@ static void Deinit() Aurora::Processes::Deinit(); } +namespace Aurora::Async +{ + + template + static AuSPtr DispathSmartWork(const WorkerId_t &worker, AuSPtr owner, FunctorTask_t task, FuckYou_t job, Args ... in) + { + return DispatchWork, Args...>, ReturnValue_t>(worker, + TaskFromTupleCallableWithBindOwner2, Args...>, ReturnValue_t>, ReturnValue_t, FunctorTask_t>(task), + Async::JobFromTupleClazz, Args...>(job), + std::make_tuple, Args...>(AU_FWD(owner), std::forward(in)...), + false); + } + + template + static AuSPtr DispathSmartWorkEx(const WorkerId_t &worker, AuSPtr owner, FunctorTask_t task, FuckYou_t success, FuckYou2_t failure, Args ... in) + { + return DispatchWork, Args...>, ReturnValue_t>(worker, + TaskFromTupleCallableWithBindOwner2, Args...>, ReturnValue_t>, ReturnValue_t, FunctorTask_t>(task), + Async::JobFromTupleClazzEx, Args...>(success, failure), + std::make_tuple, Args...>(AU_FWD(owner), std::forward(in)...), + false); + } +} + namespace Aurora { static bool gRuntimeHasStarted {}; + struct A : std::enable_shared_from_this + { + AuUInt64 DoWork(AuUInt64 in, AuUInt64 in2) + { + return in * 2 * in2; + } + + void DoWorkComplete(const AuSPtr &worker, const AuUInt64 &out) + { + AuLog::LogVerbose("Math calculation finished on a remote thread, result: {}", out); + } + + static void DoWorkFail(const AuSPtr &worker) + { + SysPanic("How did I fail?"); + } + + + void DoWorkComplete2(const AuUInt64 &a, const AuUInt64 &b, const AuUInt64 &c) + { + AuLog::LogVerbose("Math calculation finished on a remote thread, result: {} * 2 * {} = {}", a, b, c); + } + + void DoWorkFail2(AuUInt64 a, AuUInt64 b) + { + SysPanic("How did I fail?"); + } + + void RunAsyncMath(AuUInt64 in, AuUInt64 in2) + { + Async::DispathSmartWorkEx(AuWorkerId_t {}, + AuSharedFromThis(), + std::bind(&A::DoWork, AuSharedFromThis(), std::placeholders::_1, std::placeholders::_2), + std::bind(&A::DoWorkComplete, AuSharedFromThis(), std::placeholders::_1, std::placeholders::_2), + std::bind(DoWorkFail, std::placeholders::_1), + in, + in2); + + Async::DispatchWork,AuUInt64>(AuWorkerId_t {}, + AuAsync::TaskFromTupleCallable(std::bind(&A::DoWork, AuSharedFromThis(), std::placeholders::_1, std::placeholders::_2)), + AuAsync::JobFromTupleConsumerEx( + std::bind(&A::DoWorkComplete2, AuSharedFromThis(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), + std::bind(&A::DoWorkFail2, AuSharedFromThis(), std::placeholders::_1, std::placeholders::_2)), + std::make_tuple(in, in2)); + + } + }; + AUKN_SYM void RuntimeStart(const RuntimeStartInfo &info) { gRuntimeConfig = info; diff --git a/Source/HWInfo/CpuInfo.cpp b/Source/HWInfo/CpuInfo.cpp index cb155734..9d96bf89 100644 --- a/Source/HWInfo/CpuInfo.cpp +++ b/Source/HWInfo/CpuInfo.cpp @@ -41,27 +41,27 @@ namespace Aurora::HWInfo #if defined(AURORA_ARCH_X64) || defined(AURORA_ARCH_X86) #if defined(AURORA_COMPILER_MSVC) - static inline CPUIdContext cpuid(AuUInt32 a) + static CPUIdContext cpuid(AuUInt32 a) { CPUIdContext context; __cpuid(context.regs, a); return context; } #elif defined(AURORA_COMPILER_CLANG) || defined(AURORA_COMPILER_GCC) - static inline CPUIdContext cpuid(AuUInt32 a) + static CPUIdContext cpuid(AuUInt32 a) { CPUIdContext context; __get_cpuid(a, &context.eax, &context.ebx, &context.ecx, &context.edx); return context; } #else - static inline CPUIdContext cpuid(AuUInt32 a) + static CPUIdContext cpuid(AuUInt32 a) { return {}; } #endif #else - static inline CPUIdContext cpuid(AuUInt32 a) + static CPUIdContext cpuid(AuUInt32 a) { return {}; } diff --git a/Source/IO/FS/Async.NT.cpp b/Source/IO/FS/Async.NT.cpp index 76a82d6b..dbb90d83 100644 --- a/Source/IO/FS/Async.NT.cpp +++ b/Source/IO/FS/Async.NT.cpp @@ -216,7 +216,7 @@ namespace Aurora::IO::FS return; } - auto hold = std::exchange(this->pin_, {}); // this has side effects + auto hold = std::exchange(this->pin_, {}); if (hold->sub_) { diff --git a/Source/IO/FS/Resources.cpp b/Source/IO/FS/Resources.cpp index 3927f94e..a78346bd 100644 --- a/Source/IO/FS/Resources.cpp +++ b/Source/IO/FS/Resources.cpp @@ -146,6 +146,24 @@ namespace Aurora::IO::FS } } + static void SetXdg(AuString &out, const char *envvar, const char *home, const char *defaultHomeExt, const char *fallback) + { + auto value = getenv(envvar); + + if (value) + { + out = value; + } + else if (home) + { + out = AuString(home) + defaultHomeExt; + } + else + { + out = fallback; + } + } + static void SetNamespaceDirectories() { const char *homedir; @@ -156,22 +174,12 @@ namespace Aurora::IO::FS homedir = getpwuid(getuid())->pw_dir; } - gHomeDirectory = homedir ? homedir : ""; + // XDG Base Directory Specification + // $XDG_CONFIG_HOME defines the base directory relative to which user-specific configuration files should be stored. If $XDG_CONFIG_HOME is either not set or empty, a default equal to $HOME/.config should be used. + // $XDG_DATA_HOME defines the base directory relative to which user-specific data files should be stored + SetXdg(gApplicationData, "XDG_CONFIG_HOME", homedir, "/.config", kUnixAppData); + SetXdg(gHomeDirectory, "XDG_DATA_HOME", homedir, "/.local/share", "."); - if (gHomeDirectory.empty()) - { - gHomeDirectory = "."; - } - - if (FS::DirExists(kUnixAppData)) - { - gApplicationData = kUnixAppData; - } - else - { - gApplicationData = gHomeDirectory; - } - SetUnixPaths(gUserLibPath, gUserLibPath2, "/usr/lib"); SetUnixPaths(gSystemLibPath, gSystemLibPath2, "/lib"); } @@ -209,23 +217,23 @@ namespace Aurora::IO::FS static void ChangeDir() { #if !defined(AU_NO_AU_HOME_BRANDING) - if (gRuntimeConfig.fio.defaultBrand.has_value()) + if (gRuntimeConfig.fio.defaultBrand.size()) { - gApplicationData += "/" + gRuntimeConfig.fio.defaultBrand.value() + "/System"; - #if defined(AURORA_IS_POSIX_DERIVED) - gHomeDirectory += "/." + gRuntimeConfig.fio.defaultBrand.value() + "/Profile"; - #else - gHomeDirectory += "/" + gRuntimeConfig.fio.defaultBrand.value() + "/Profile"; - #endif + gApplicationData += "/" + gRuntimeConfig.fio.defaultBrand; + gHomeDirectory += "/" + gRuntimeConfig.fio.defaultBrand; } - else #endif - { - gHomeDirectory += "/.config"; //most unix programs hide their private user data under here - } NormalizePath(gApplicationData); NormalizePath(gHomeDirectory); + + if (gApplicationData == gHomeDirectory) + { + gApplicationData += kPathSplitter; + gHomeDirectory += kPathSplitter; + gApplicationData += "System"; + gHomeDirectory += "Profile"; + } // Noting we append a path splitter to prevent hair pulling over missing path delimiters // Eg: GetHome() + "myAwesomeApp/Config" = %HOME%/Aurora/ProfilemyAwsomeApp/Config diff --git a/Source/IO/Net/Net.cpp b/Source/IO/Net/Net.cpp index b227c85b..38d69067 100644 --- a/Source/IO/Net/Net.cpp +++ b/Source/IO/Net/Net.cpp @@ -2,6 +2,8 @@ Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Net.cpp - Date: 2021-6-17 + Date: 2021-7-2 Author: Reece ***/ +#include +#include "Net.hpp" \ No newline at end of file diff --git a/Source/IO/Net/Net.hpp b/Source/IO/Net/Net.hpp index 87204b21..1fe2a168 100644 --- a/Source/IO/Net/Net.hpp +++ b/Source/IO/Net/Net.hpp @@ -2,6 +2,9 @@ Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Net.hpp - Date: 2021-6-17 + Date: 2021-7-2 Author: Reece ***/ +#pragma once + +#include "SocketStatChannel.hpp" \ No newline at end of file diff --git a/Source/IO/Net/SocketStatAverageBps.hpp b/Source/IO/Net/SocketStatAverageBps.hpp new file mode 100644 index 00000000..f21fc0e9 --- /dev/null +++ b/Source/IO/Net/SocketStatAverageBps.hpp @@ -0,0 +1,86 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: SocketStatAverageBps.hpp + Date: 2021-10-31 + Author: Reece +***/ +#pragma once + +namespace Aurora::IO::Net +{ + struct SocketStatAverageBps + { + inline AuUInt32 GetBytesPerMS(AuUInt32 timeNow) + { + return (double(bytesTransferred) / (double(frameStart - timeNow) + std::numeric_limits::epsilon())); + } + + inline AuUInt32 GetBytesPerSecondNormalized(AuUInt32 timeNow) + { + AU_LOCK_GUARD(lock); + + // If timeNow isn't at least that of one socket frame, then we just have to return some old data + if (frameZero) return GetLastBytesPerSecond(); + + // Cool we can extrapolate usage using a time weight at least that of one server frame + + // Some useful variables... + auto frameAtPoint = GetBytesPerMS(timeNow); + auto timeElapsed = frameStart - timeNow; + + // Edge case: we aren't receiving much data. if we have more than 1 second of data, we should just average it + if (timeElapsed > 1000) return frameAtPoint / 1000; // from ms + // else assume constant usage will continue to trend for at least another second + // the actual extrapolation + auto weight = double(1000) / double(timeElapsed); + return double(frameAtPoint) * weight; + } + + inline AuUInt32 GetLastBytesPerSecond() + { + AU_LOCK_GUARD(lock); + return (double(lastBytesTransferred) / (double(frameLastEnd - frameLastStart) + std::numeric_limits::epsilon())) / 1000; + } + + inline void Reset() + { + bytesTransferred = 0; + frameStart = 0; + frameLastEnd = 0; + frameLastStart = 0; + lastBytesTransferred = 0; + } + + inline void Add(AuUInt32 timeNow, AuUInt32 bytes) + { + AU_LOCK_GUARD(lock); + auto nextTick = frameLastEnd + 1000; + + if (nextTick < timeNow) + { + frameLastEnd = timeNow; + frameLastStart = std::exchange(frameStart, timeNow); + lastBytesTransferred = std::exchange(bytesTransferred, bytes); + frameZero = true; + } + else + { + frameZero = false; + bytesTransferred += bytes; + } + } + + + AuThreadPrimitives::SpinLock lock; + + AuUInt32 frameStart {}; + AuUInt32 bytesTransferred {}; + + AuUInt32 frameLastStart {}; + AuUInt32 frameLastEnd {}; + AuUInt32 lastBytesTransferred {}; + + bool frameZero {true}; + }; +} \ No newline at end of file diff --git a/Source/IO/Net/SocketStatChannel.hpp b/Source/IO/Net/SocketStatChannel.hpp new file mode 100644 index 00000000..bc6d5a9f --- /dev/null +++ b/Source/IO/Net/SocketStatChannel.hpp @@ -0,0 +1,36 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: SocketStatChannel.hpp + Date: 2021-10-31 + Author: Reece +***/ +#pragma once + +#include "SocketStatAverageBps.hpp" + +namespace Aurora::IO::Net +{ + struct SocketStatChannel + { + SocketStatAverageBps bandwidth; + AuUInt64 total; + + // Interpolated bytes per second, may be less than expected (returning frame -1) + inline AuUInt32 GetAverageBytesPerSecond() + { + return bandwidth.GetLastBytesPerSecond(); + } + + inline AuUInt32 GetBytesPerSecond() + { + return bandwidth.GetBytesPerSecondNormalized(Time::CurrentClockMS()); + } + + inline void Add(AuUInt32 bytes) + { + total += bytes; + bandwidth.Add(Time::CurrentClockMS(), bytes); + } + }; +} \ No newline at end of file diff --git a/Source/Locale/Encoding/ConvertInternal.cpp b/Source/Locale/Encoding/ConvertInternal.cpp index ce76ecaa..24d5a7b7 100644 --- a/Source/Locale/Encoding/ConvertInternal.cpp +++ b/Source/Locale/Encoding/ConvertInternal.cpp @@ -255,11 +255,6 @@ namespace Aurora::Locale::Encoding return ret; } - AuStreamReadWrittenPair_t STLCPToUTF8(ECodePage page, void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max) - { - return DecodeUTF8Internal(in, length, utf8, utf8Max, page); - } - AuStreamReadWrittenPair_t STLCPToUTF8(ECodePage page, const void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max) { return DecodeUTF8Internal(in, length, utf8, utf8Max, page); diff --git a/Source/Locale/Encoding/ConvertInternal.hpp b/Source/Locale/Encoding/ConvertInternal.hpp index c2021c65..11193f6d 100644 --- a/Source/Locale/Encoding/ConvertInternal.hpp +++ b/Source/Locale/Encoding/ConvertInternal.hpp @@ -19,5 +19,4 @@ namespace Aurora::Locale::Encoding AuStreamReadWrittenPair_t STLCPToUTF8(ECodePage page, const void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max); AuStreamReadWrittenPair_t STLUTF8ToCp(ECodePage page, const void *utf8, AuUInt32 utf8Length, void *cp, AuUInt32 cpLen); - AuStreamReadWrittenPair_t STLCPToUTF8(ECodePage page, void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max); } \ No newline at end of file diff --git a/Source/Locale/Encoding/EncoderAdapter.cpp b/Source/Locale/Encoding/EncoderAdapter.cpp index 9b7d5648..d51f0e1c 100644 --- a/Source/Locale/Encoding/EncoderAdapter.cpp +++ b/Source/Locale/Encoding/EncoderAdapter.cpp @@ -31,41 +31,13 @@ namespace Aurora::Locale::Encoding this->decode = decode; } - AuStreamReadWrittenPair_t EncoderAdapter::CPToUTF8(void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max) - { - AuStreamReadWrittenPair_t ret {}; - - if (!length) - { - return {}; - } - - if (!in) - { - return {}; - } - - if (((page == ECodePage::eSysUnk) && - (GetInternalCodePage() == ECodePage::eUTF8)) || - (page == ECodePage::eUTF8)) - { - auto readable = std::min(length, utf8Max); - if (utf8 && in) - { - std::memcpy(utf8, in, readable); - } - return {length, utf8 ? utf8Max : length}; - } - - ret = Win32CPToUTF8(page, in, length, utf8, utf8Max); - if (!ret.first) - { - ret = STLCPToUTF8(page, in, length, utf8, utf8Max); - } - - return ret; + bool EncoderAdapter::TestPage(ECodePage ref) + { + return (((page == ECodePage::eSysUnk) && + (GetInternalCodePage() == ref)) || + (page == ref)); } - + AuStreamReadWrittenPair_t EncoderAdapter::CPToUTF8(const void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Max) { AuStreamReadWrittenPair_t ret {}; @@ -80,21 +52,42 @@ namespace Aurora::Locale::Encoding return {}; } - if (((page == ECodePage::eSysUnk) && - (GetInternalCodePage() == ECodePage::eUTF8)) || - (page == ECodePage::eUTF8)) + // decode using internal and/or optimized apis first + if (TestPage(ECodePage::eUTF8)) { + length = CountUTF8Length({in, length}, true); + auto readable = std::min(length, utf8Max); if (utf8 && in) { std::memcpy(utf8, in, readable); } - return {length, utf8 ? utf8Max : length}; + + return {readable, readable}; + } + + // never remove me. the stl and windows can't bet trusted to fail on conversion failure + if (TestPage(ECodePage::eGBK)) + { + length = GBK::CountGbk(in, length, true); + } + else if (TestPage(ECodePage::eSJIS)) + { + length = SJIS::CountSJIS(in, length, true); + } + else if (TestPage(ECodePage::eUTF16) || TestPage(ECodePage::eUTF16BE)) + { + length = UTF16::Count16(in, length, true); + } + else if ((page == ECodePage::eUTF32) || (page == ECodePage::eUTF32BE)) + { + length &= ~3; } ret = Win32CPToUTF8(page, in, length, utf8, utf8Max); if (!ret.first) { + // TODO: iconv support here ret = STLCPToUTF8(page, in, length, utf8, utf8Max); } @@ -115,9 +108,7 @@ namespace Aurora::Locale::Encoding return {}; } - if (((page == ECodePage::eSysUnk) && - (GetInternalCodePage() == ECodePage::eUTF8)) || - (page == ECodePage::eUTF8)) + if (TestPage(ECodePage::eUTF8)) { auto readable = std::min(utf8Length, cpLen); if (utf8 && cp) @@ -130,6 +121,7 @@ namespace Aurora::Locale::Encoding ret = Win32UTF8ToCp(page, utf8, utf8Length, cp, cpLen); if (!ret.first) { + // TODO: iconv support here ret = STLUTF8ToCp(page, utf8, utf8Length, cp, cpLen); } diff --git a/Source/Locale/Encoding/EncoderAdapter.hpp b/Source/Locale/Encoding/EncoderAdapter.hpp index 641f61f8..8f12188d 100644 --- a/Source/Locale/Encoding/EncoderAdapter.hpp +++ b/Source/Locale/Encoding/EncoderAdapter.hpp @@ -19,8 +19,8 @@ namespace Aurora::Locale::Encoding ~EncoderAdapter(); void Init(ECodePage page, bool decode); - - AuStreamReadWrittenPair_t CPToUTF8(void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Len); + bool TestPage(ECodePage page); + AuStreamReadWrittenPair_t CPToUTF8(const void *in, AuUInt32 length, void *utf8, AuUInt32 utf8Len); AuStreamReadWrittenPair_t UTF8ToCp(const void *utf8, AuUInt32 utf8Length, void *cp, AuUInt32 cpLen); }; diff --git a/Source/Locale/Encoding/EncoderNSL.cpp b/Source/Locale/Encoding/EncoderNSL.cpp index 8bac86dd..b8a58015 100644 --- a/Source/Locale/Encoding/EncoderNSL.cpp +++ b/Source/Locale/Encoding/EncoderNSL.cpp @@ -146,48 +146,6 @@ namespace Aurora::Locale::Encoding #endif } - // TODO(reece): Consider implementing bigendian when I can be bothered - - AuStreamReadWrittenPair_t Win32CPToUTF8(ECodePage page, void *in, AuUInt length, void *utf8, AuUInt32 utf8Max) - { - AuStreamReadWrittenPair_t ret {}; - - #if defined(AU_HAS_MSFT_NATIONALLANGSUPPORT) - switch (page) - { - default: - case ECodePage::eUnsupported: - return {}; - case ECodePage::e18030: - ret = Win32ConvertFromCPToUTF8(CP_CHINESE, in, length, utf8, utf8Max); - break; - case ECodePage::eSysUnk: - ret = Win32ConvertFromCPToUTF8(CP_ACP, in, length, utf8, utf8Max); - break; - case ECodePage::eLatin1: - ret = Win32ConvertFromCPToUTF8(CP_LATIN_1, in, length, utf8, utf8Max); - break; - case ECodePage::eUTF7: - ret = Win32ConvertFromCPToUTF8(CP_UTF7, in, length, utf8, utf8Max); - break; - case ECodePage::e2312: - ret = Win32ConvertFromCPToUTF8(CP_2312_LIMITED_GBK, in, length, utf8, utf8Max); - break; - case ECodePage::eGBK: - ret = Win32ConvertFromCPToUTF8(CP_2312_LIMITED_GBK, in, length, utf8, utf8Max); - break; - case ECodePage::eSJIS: - ret = Win32ConvertFromCPToUTF8(CP_SHIFTJIS, in, length, utf8, utf8Max); - break; - case ECodePage::eUTF16: - ret = Win32ConvertFromUTF16ToUTF8(in, length, utf8, utf8Max); - break; - } - #endif - - return ret; - } - AuStreamReadWrittenPair_t Win32CPToUTF8(ECodePage page, const void *in, AuUInt length, void *utf8, AuUInt32 utf8Max) { AuStreamReadWrittenPair_t ret {}; diff --git a/Source/Locale/Encoding/EncoderNSL.hpp b/Source/Locale/Encoding/EncoderNSL.hpp index df13c5f8..1feabef4 100644 --- a/Source/Locale/Encoding/EncoderNSL.hpp +++ b/Source/Locale/Encoding/EncoderNSL.hpp @@ -14,7 +14,6 @@ namespace Aurora::Locale::Encoding AuStreamReadWrittenPair_t Win32ConvertFromUTF8ToCp(AuUInt32 cp, const void *utf8, AuUInt utf8Length, void *cpBlob, AuUInt32 cpLen); AuStreamReadWrittenPair_t Win32ConvertFromCPToUTF8(AuUInt32 cp, const void *in, AuUInt length, void *utf8, AuUInt32 utf8Len); - AuStreamReadWrittenPair_t Win32CPToUTF8(ECodePage page, void *in, AuUInt length, void *utf8, AuUInt32 utf8Max); AuStreamReadWrittenPair_t Win32CPToUTF8(ECodePage page, const void *in, AuUInt length, void *utf8, AuUInt32 utf8Max); AuStreamReadWrittenPair_t Win32UTF8ToCp(ECodePage page, const void *utf8, AuUInt32 utf8Length, void *cp, AuUInt32 cpLen); } \ No newline at end of file diff --git a/Source/Locale/Encoding/Encoding.cpp b/Source/Locale/Encoding/Encoding.cpp index 211be651..bab80cac 100644 --- a/Source/Locale/Encoding/Encoding.cpp +++ b/Source/Locale/Encoding/Encoding.cpp @@ -11,10 +11,10 @@ namespace Aurora::Locale::Encoding { - AUKN_SYM AuOptional> DecodeBOM(const Memory::MemoryViewRead & binary) + AUKN_SYM BOM DecodeBOM(const Memory::MemoryViewRead & binary) { - #define ADD_PATTERN(str, code) {str, AuArraySize(str) - 1, ECodePage::code} - AuList> bows = + #define ADD_PATTERN(str, code) {str, {ECodePage::code, AuArraySize(str) - 1}} + AuList> bows = { ADD_PATTERN("\xFF\xFE\x00\x00", eUTF32), ADD_PATTERN("\x00\x00\xFE\xFF", eUTF32BE), @@ -30,30 +30,32 @@ namespace Aurora::Locale::Encoding }; #undef ADD_PATTERN - for (const auto &[string, length, category] : bows) + for (const auto &[string, bom] : bows) { - if (binary.length < length) continue; - if (std::memcmp(binary.ptr, string, length) != 0) continue; + if (binary.length < bom.length) continue; + if (std::memcmp(binary.ptr, string, bom.length) != 0) continue; - return AuMakePair(category, length); + return bom; } return {}; } - AUKN_SYM AuStreamReadWrittenPair_t EncodeUTF8(const Memory::MemoryViewRead & utf8, const Memory::MemoryViewWrite & binary, ECodePage page) + // OLD SHIT API + + AUKN_SYM AuStreamReadWrittenPair_t EncodeUTF8(const Memory::MemoryViewRead &utf8, const Memory::MemoryViewWrite & binary, ECodePage page) { TextStreamEncoder re(page); return re.DecodeUTF8(utf8.ptr, utf8.length, binary.ptr, binary.length); } - AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead & binary, const Memory::MemoryViewWrite & utf8, ECodePage page) + AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead &binary, const Memory::MemoryViewWrite & utf8, ECodePage page) { TextStreamProcessor re(page); return re.EncodeUTF8(binary.ptr, binary.length, utf8.ptr, utf8.length); } - AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead & binary, AuString &out, ECodePage page) + AUKN_SYM AuStreamReadWrittenPair_t DecodeUTF8(const Memory::MemoryViewRead &binary, AuString &out, ECodePage page) { auto aaa = DecodeUTF8(binary, {}, page); out.resize(aaa.second); @@ -70,4 +72,74 @@ namespace Aurora::Locale::Encoding out.resize(ret.second); return ret; } + + + // NEW API + + /// Supporting full 6 byte UTF-8, copies or returns the available streams from @param utf8 to @param utf32 + + AUKN_SYM AuStreamReadWrittenPair_t ReadUTF8IntoUTF32ByteString(const Memory::MemoryViewRead &utf8, const Memory::MemoryViewWrite &utf32) + { + const char *begin = utf8.Begin(); + const char *end = utf8.End(); + AuUInt32 *begin2 = utf32.Begin(); + AuUInt32 *end2 = utf32.End(); + UTF32::ReadUtf8ByteString(begin2, end2, begin, end); + return AuStreamReadWrittenPair_t {begin - utf8.Begin(), (begin2 - utf32.Begin()) * sizeof(AuUInt32)}; + } + + + /// Supporting full 6 byte UTF-8, copies or returns the available streams from @param utf32 to @param utf8 + AUKN_SYM AuStreamReadWrittenPair_t ReadUTF32IntoUTF8ByteString(const Memory::MemoryViewRead &utf32, const Memory::MemoryViewWrite &utf8) + { + const AuUInt32 *begin = utf32.Begin(); + const AuUInt32 *end = utf32.End(); + char *dest = utf8.Begin(); + char *destEnd = utf8.End(); + + AuUInt32 counter {}; + const AuUInt32 *cur = begin; + for (; cur < end; cur++) + { + UTF32::WriteCp(*cur, dest, counter, destEnd - dest); + } + + return AuStreamReadWrittenPair_t {(cur - begin) * sizeof(AuUInt32), dest - utf8.Begin()}; + } + + AUKN_SYM void SwapUTF32(const Memory::MemoryViewWrite &utf32) + { + UTF32::SwapU32(utf32.Begin(), utf32.ToCount()); + } + + AUKN_SYM void SwapUTF16(const Memory::MemoryViewWrite &utf32) + { + UTF16::SwapU16(utf32.Begin(), utf32.ToCount()); + } + + AUKN_SYM AuUInt32 CountUTF32Length(const Memory::MemoryViewRead &utf32, bool bytes) + { + return UTF32::Count32(utf32.ptr, utf32.length, bytes); + } + + AUKN_SYM AuUInt32 CountUTF16Length(const Memory::MemoryViewRead &utf16, bool bytes) + { + return UTF16::Count16(utf16.ptr, utf16.length, bytes); + } + + AUKN_SYM AuUInt32 CountUTF8Length(const Memory::MemoryViewRead &utf8, bool bytes) + { + auto pair = ReadUTF8IntoUTF32ByteString(utf8, {}); + return bytes ? pair.first : pair.second / sizeof(AuUInt32); + } + + AUKN_SYM AuUInt32 CountSJISLength(const Memory::MemoryViewRead &sjis, bool bytes) + { + return SJIS::CountSJIS(sjis.ptr, sjis.length, bytes); + } + + AUKN_SYM AuUInt32 CountGBK16Length(const Memory::MemoryViewRead &gbk, bool bytes) + { + return GBK::CountGbk(gbk.ptr, gbk.length, bytes); + } } diff --git a/Source/Locale/Encoding/Encoding.hpp b/Source/Locale/Encoding/Encoding.hpp index e98b3a13..bfa8f226 100644 --- a/Source/Locale/Encoding/Encoding.hpp +++ b/Source/Locale/Encoding/Encoding.hpp @@ -7,117 +7,29 @@ ***/ #pragma once +#include "GBK/GBK.hpp" +#include "SJIS/SJIS.hpp" +#include "UTFn/AuUTF8.hpp" +#include "UTFn/AuUTF16.hpp" +#include "UTFn/AuUTF32.hpp" + namespace Aurora::Locale::Encoding { AuStreamReadWrittenPair_t DecodeUTF8(void *binary, AuUInt32 binaryLength, AuString &out, ECodePage page = ECodePage::eUnsupported); - template - struct TextStreamDecoderImpl + struct TextStreamProcessor { bool readHeader {}; ECodePage page = ECodePage::eUnsupported; ECodePage defaultPage = ECodePage::eUnsupported; EncoderAdapter state; - TextStreamDecoderImpl(ECodePage page = ECodePage::eSysUnk) : defaultPage(page) {} + TextStreamProcessor(ECodePage page = ECodePage::eSysUnk) : defaultPage(page) {} - using TypeIn_t = std::conditional_t; - using TypeCast_t = std::conditional_t; + using TypeIn_t = const void *; + using TypeCast_t = const AuUInt8 *; - - static int GetLenSJISCodePoint(const AuUInt8 *in, AuUInt32 len) - { - if (len == 0) return 0; - auto b = in[0]; - if (b >= 0x80) - { - if (b <= 0xDF) - { - if (len < 2) return 0; - else return 2; - } - else if (b <= 0xEF) - { - if (len < 3) return 0; - else return 3; - } - else - { - if (len < 4) return 0; - else return 4; - } - - } - return 1; - } - - static int GetLenSJISString(const AuUInt8 *in, AuUInt32 len) - { - AuUInt32 i; - for (i = 0; i < len; ) - { - auto next = GetLenSJISCodePoint(in + i, len - i); - if (next == 0) return i; - i += next; - } - return i; - } - - static int GetLenGBKCodePoint(const AuUInt8 *in, AuUInt32 len) - { - if (len == 0) return 0; - auto b = in[0]; - if (b >= 0x80) - { - if (len < 2) return 0; - else return 2; - } - return 1; - } - - static int GetLenGBKString(const AuUInt8 *in, AuUInt32 len) - { - AuUInt32 i; - for (i = 0; i < len; ) - { - auto next = GetLenGBKCodePoint(in + i, len - i); - if (next == 0) return i; - i += next; - } - return i; - } - - #define AU_HIGH_SURROGATE_START 0xd800 - #define AU_HIGH_SURROGATE_END 0xdbff - #define AU_LOW_SURROGATE_START 0xdc00 - #define AU_LOW_SURROGATE_END 0xdfff - #define AU_IS_HIGH_SURROGATE(wch) (((wch) >= AU_HIGH_SURROGATE_START) && ((wch) <= AU_HIGH_SURROGATE_END)) - #define AU_IS_LOW_SURROGATE(wch) (((wch) >= AU_LOW_SURROGATE_START) && ((wch) <= AU_LOW_SURROGATE_END)) - - static int GetLenUC2CodePoint(const AuUInt8 *in, AuUInt32 len) - { - if (len < 2) return 0; - auto a = *reinterpret_cast(in); - if (!AU_IS_HIGH_SURROGATE(a)) return 2; - - if (len < 4) return 0; - auto b = *reinterpret_cast(in + 2); - return AU_IS_LOW_SURROGATE(b) ? 4 : 0; - } - - static int GetLenUC2String(const AuUInt8 *in, AuUInt32 len) - { - AuUInt32 i; - for (i = 0; i < len; ) - { - auto next = GetLenUC2CodePoint(in + i, len - i); - if (next == 0) return i; - i += next; - } - return i; - } - - AuStreamReadWrittenPair_t EncodeUTF8(TypeIn_t binary, AuUInt32 binaryLength, void *utf8, AuUInt32 utfLen) + AuStreamReadWrittenPair_t EncodeUTF8(const void *binary, AuUInt32 binaryLength, void *utf8, AuUInt32 utfLen) { int offset = 0; @@ -128,11 +40,11 @@ namespace Aurora::Locale::Encoding { if (page == ECodePage::eUnsupported) { - auto header = DecodeBOM(Aurora::Memory::MemoryViewRead(binary, binaryLength)); - if (header) + auto header = DecodeBOM(Memory::MemoryViewRead(binary, binaryLength)); + if (header.length) { - page = header->first; - offset = header->second; + page = header.page; + offset = header.length; } else { @@ -155,29 +67,11 @@ namespace Aurora::Locale::Encoding } binaryLength = binaryLength - offset; - - if (page == ECodePage::eGBK) - { - binaryLength = GetLenGBKString(reinterpret_cast(binary) + offset, binaryLength); - } - else if (page == ECodePage::eSJIS) - { - binaryLength = GetLenSJISString(reinterpret_cast(binary) + offset, binaryLength); - } - else if ((page == ECodePage::eUTF16) || (page == ECodePage::eUTF16BE)) - { - binaryLength = GetLenUC2String(reinterpret_cast(binary) + offset, binaryLength); - } - else if ((page == ECodePage::eUTF32) || (page == ECodePage::eUTF32BE)) - { - binaryLength &= ~3; - } - auto real = state.CPToUTF8(reinterpret_cast(binary) + offset, binaryLength, utf8, utfLen); return AuMakePair(real.first + offset, real.second); } - AuStreamReadWrittenPair_t EncodeUTF8(TypeIn_t binary, AuUInt32 binaryLength, AuString &out) + AuStreamReadWrittenPair_t EncodeUTF8(const void *binary, AuUInt32 binaryLength, AuString &out) { auto preemptive = EncodeUTF8(binary, binaryLength, nullptr, 0); if (!AuTryResize(out, preemptive.second)) return {}; @@ -237,6 +131,4 @@ namespace Aurora::Locale::Encoding /// 'TextStreamProcessor', a stateful wrapper around DecodeUTF8 /// Using this you can handle a stateful, optionally bom prefixed, stream /// Initialization (ie: setting a default codepage) is optional - using TextStreamProcessor = typename TextStreamDecoderImpl; - using TextStreamProcessorInternal = typename TextStreamDecoderImpl; } \ No newline at end of file diff --git a/Source/Locale/Encoding/GBK/GBK.hpp b/Source/Locale/Encoding/GBK/GBK.hpp new file mode 100644 index 00000000..4d5e6f97 --- /dev/null +++ b/Source/Locale/Encoding/GBK/GBK.hpp @@ -0,0 +1,36 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: GBK.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Locale::Encoding::GBK +{ + static inline int GetLenGBKCodePoint(const AuUInt8 *in, AuUInt32 len) + { + if (len == 0) return 0; + auto b = in[0]; + if (b >= 0x80) + { + if (len < 2) return 0; + else return 2; + } + return 1; + } + + static inline int CountGbk(const void *base, AuUInt32 length, bool bytes = false) + { + AuUInt32 i {}, cps {}; + for (i = 0; i < length; ) + { + auto next = GetLenGBKCodePoint((const AuUInt8*)base + i, length - i); + if (next == 0) return i; + i += next; + cps++; + } + return bytes ? i : cps; + } +} \ No newline at end of file diff --git a/Source/Locale/Encoding/SJIS/SJIS.hpp b/Source/Locale/Encoding/SJIS/SJIS.hpp new file mode 100644 index 00000000..08b9f9ab --- /dev/null +++ b/Source/Locale/Encoding/SJIS/SJIS.hpp @@ -0,0 +1,67 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: SJIS.hpp + Date: 2021-11-1 + Author: Reece +***/ +#pragma once + +namespace Aurora::Locale::Encoding::SJIS +{ + static inline int GetLenSJISCodePoint(const AuUInt8 *in, AuUInt32 len) + { + if (len == 0) + { + return 0; + } + + auto b = in[0]; + + if (b >= 0x80) + { + if (b <= 0xDF) + { + if (len < 2) + { + return 0; + } + + return 2; + } + else if (b <= 0xEF) + { + if (len < 3) + { + return 0; + } + + return 3; + } + else + { + if (len < 4) + { + return 0; + } + + return 4; + } + + } + return 1; + } + + static inline AuUInt32 CountSJIS(const void *base, AuUInt32 length, bool bytes = false) + { + AuUInt32 i {}, cps {}; + for (; i < length; ) + { + auto next = GetLenSJISCodePoint((const AuUInt8 *)base + i, length - i); + if (next == 0) return i; + cps++; + i += next; + } + return bytes ? i : cps; + } +} \ No newline at end of file diff --git a/Source/Locale/Encoding/UTFn/AuUTF16.hpp b/Source/Locale/Encoding/UTFn/AuUTF16.hpp new file mode 100644 index 00000000..e74fe74a --- /dev/null +++ b/Source/Locale/Encoding/UTFn/AuUTF16.hpp @@ -0,0 +1,53 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: AuUTF16.hpp + Date: 2021-10-31 + Author: Reece +***/ +#pragma once + +namespace Aurora::Locale::Encoding::UTF16 +{ + static void SwapU16(void *base, AuUInt32 count) + { + count *= 2; + for (int i = 0; i < count; i += 2) + { + AuWriteU16BE(base, i, AuReadU16LE(base, i)); + } + } + + #define AU_HIGH_SURROGATE_START 0xd800 + #define AU_HIGH_SURROGATE_END 0xdbff + #define AU_LOW_SURROGATE_START 0xdc00 + #define AU_LOW_SURROGATE_END 0xdfff + #define AU_IS_HIGH_SURROGATE(wch) (((wch) >= AU_HIGH_SURROGATE_START) && ((wch) <= AU_HIGH_SURROGATE_END)) + #define AU_IS_LOW_SURROGATE(wch) (((wch) >= AU_LOW_SURROGATE_START) && ((wch) <= AU_LOW_SURROGATE_END)) + + static int GetLenUC2CodePoint(const AuUInt8 *in, AuUInt32 len) + { + if (len < 2) return 0; + auto a = *reinterpret_cast(in); + if (!AU_IS_HIGH_SURROGATE(a)) return 2; + + if (len < 4) return 0; + auto b = *reinterpret_cast(in + 2); + return AU_IS_LOW_SURROGATE(b) ? 4 : 0; + } + + static int Count16(const void *base, AuUInt32 length, bool bytes = false) + { + AuUInt32 i {}, cps {}; + + for (; i < length; ) + { + auto next = GetLenUC2CodePoint(((const AuUInt8 *)base) + i, length - i); + if (next == 0) return i; + i += next; + cps++; + } + + return bytes ? i : cps; + } +} \ No newline at end of file diff --git a/Source/Locale/Encoding/UTFn/AuUTF32.hpp b/Source/Locale/Encoding/UTFn/AuUTF32.hpp new file mode 100644 index 00000000..5937de6f --- /dev/null +++ b/Source/Locale/Encoding/UTFn/AuUTF32.hpp @@ -0,0 +1,188 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: AuUTF32.hpp + Date: 2021-10-31 + Author: Reece +***/ +#pragma once + +namespace Aurora::Locale::Encoding::UTF32 +{ + static void SwapU32(void *base, AuUInt32 count) + { + count *= 4; + for (int i = 0; i < count; i += 4) + { + AuWriteU32BE(base, i, AuReadU32LE(base, i)); + } + } + + static AuUInt32 Count32(const void *base, AuUInt32 length, bool bytes = false) + { + return bytes ? length & ~3 : (length & ~3) / 4; + } + + static bool WriteCp(AuUInt32 cp, char *& result, AuUInt32 &counter, AuUInt32 max) + { + if (cp < 0x80) + { + if (counter + 1 > max) + { + return false; + } + counter += 1; + *(result++) = static_cast(cp); + } + else if (cp < 0x800) + { + if (counter + 2 > max) + { + return false; + } + counter += 2; + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) + { + if (counter + 3 > max) + { + return false; + } + counter += 3; + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 200000) + { + if (counter + 4 > max) + { + return false; + } + counter += 4; + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 400000) + { + if (counter + 5 > max) + { + return false; + } + counter += 5; + *(result++) = static_cast((cp >> 24) | 0xf8); + *(result++) = static_cast(((cp >> 18) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 80000000) + { + if (counter + 6 > max) + { + return false; + } + counter += 6; + *(result++) = static_cast((cp >> 30) | 0xfc); + *(result++) = static_cast(((cp >> 24) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 18) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else + { + return false; + } + return true; + } + + static bool ReadUtf8ByteString(AuUInt32 *cp, AuUInt32 *&cpEnd, const char *&s, const char *end) + { + if (!s) + { + return false; + } + + if (!cp) + { + cpEnd = cp; + } + + auto &it = s; + + while ((it < end) && // can coomsome + (cp && cp < cpEnd)) + { + AuUInt32 c = 0; + if (*it <= 0x7FU) + { + c = *it; + ++it; + } + else + { + if ((*it & 0xC0U) != 0xC0U) + { + return false; + } + + AuUInt32 nby = 0; + for (AuUInt8 b = *it; (b & 0x80U) != 0; b <<= 1, ++nby) + {} + + if (nby > 6) + { + return false; + } + + if ((end - it) < nby) + { + return false; + } + + c = *it & (AuUInt8(0xFFU) >> (nby + 1)); + + for (AuUInt32 i = 1; i < nby; ++i) + { + if ((it[i] & 0xC0U) != 0x80U) + { + return false; + } + + c = (c << 6) | (it[i] & 0x3FU); + } + + it += nby; + } + + if (cp) + { + *cp = c; + cp++; + } + + cpEnd++; + } + } + + static AuUInt32 CountU8Overhead(AuUInt32 cp) + { + if (cp < 0x80) + return 1; + else if (cp < 0x800) + return 2; + else if (cp < 0x10000) + return 3; + else if (cp < 200000) + return 4; + else if (cp < 400000) + return 5; + else if (cp < 80000000) + return 6; + } +} \ No newline at end of file diff --git a/Source/Locale/Encoding/UTFn/AuUTF8.hpp b/Source/Locale/Encoding/UTFn/AuUTF8.hpp new file mode 100644 index 00000000..07b01e02 --- /dev/null +++ b/Source/Locale/Encoding/UTFn/AuUTF8.hpp @@ -0,0 +1,13 @@ +/*** + Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. + + File: AuUTF8.hpp + Date: 2021-10-31 + Author: Reece +***/ +#pragma once + +namespace Aurora::Locale::Encoding::UTF8 +{ + +} \ No newline at end of file diff --git a/Source/Locale/Locale.cpp b/Source/Locale/Locale.cpp index d9f83d31..137dcec1 100644 --- a/Source/Locale/Locale.cpp +++ b/Source/Locale/Locale.cpp @@ -345,7 +345,7 @@ namespace Aurora::Locale gCountryCode = AuToUpper(gCountryCode); gCodeset = AuToUpper(gCodeset); - LogDbg("Initialized localization information (language: {}, country: {}, codeset: {})", gLanguageCode, gCountryCode, gCodeset); + LogDbg("Initialized default localization information (language: {}, country: {}, codeset: {})", gLanguageCode, gCountryCode, gCodeset); } static bool gLockLocale = false; diff --git a/Source/Memory/Heap.cpp b/Source/Memory/Heap.cpp index cfc71910..2f7a2536 100644 --- a/Source/Memory/Heap.cpp +++ b/Source/Memory/Heap.cpp @@ -51,7 +51,7 @@ namespace Aurora::Memory class InternalHeap : public Heap { public: - InternalHeap() : base_(nullptr), mutex_(nullptr), heap_(nullptr), _count(0) + InternalHeap() : base_(nullptr), mutex_(nullptr), heap_(nullptr), count_(0) { } ~InternalHeap(); @@ -90,14 +90,14 @@ namespace Aurora::Memory AuThreadPrimitives::MutexUnique_t mutex_; void *base_ {}; O1HeapInstance *heap_ {}; - int _count {}; - AuUInt length_; - bool _isDangling {}; + int count_ {}; + AuUInt length_ {}; + bool isDangling_ {}; }; InternalHeap::~InternalHeap() { - SysAssertDbgExp(_count == 0); + SysAssertDbgExp(count_ == 0); if (base_) { @@ -149,7 +149,7 @@ namespace Aurora::Memory { if (!heap_) return nullptr; auto ret = o1heapAllocate(heap_, length); - if (ret) _count++; + if (ret) count_++; return ret; } @@ -210,14 +210,14 @@ namespace Aurora::Memory { if (buffer == nullptr) return; o1heapFree(heap_, buffer); - _count--; + count_--; } void InternalHeap::TryRelease() { - if (!_isDangling) return; + if (!isDangling_) return; - if (_count == 0) + if (count_ == 0) { delete this; return; @@ -226,13 +226,13 @@ namespace Aurora::Memory void InternalHeap::RequestTermination() { - if (_count) + if (count_) { LogWarn("Heap life was less than its allocations, waiting for final free"); LogWarn("Reporting using mayday!"); Telemetry::Mayday(); - _isDangling = true; + isDangling_ = true; TryRelease(); } else diff --git a/Source/Process/Process.cpp b/Source/Process/Process.cpp index e3c05a55..95469b03 100644 --- a/Source/Process/Process.cpp +++ b/Source/Process/Process.cpp @@ -21,9 +21,9 @@ namespace Aurora::Process exit(exitcode); #else // ??? - *(AuUInt8 *)0 = 0; - *(AuUInt8 *)0xFFFF = 0; - *(AuUInt8 *)0xFFFFFFFF = 0; + *(AuUInt32 *)0 = exitcode; + *(AuUInt32 *)0xFFFF = exitcode; + *(AuUInt32 *)0xFFFFFFFF = exitcode; #endif } } \ No newline at end of file diff --git a/Source/Processes/Open.Linux.cpp b/Source/Processes/Open.Linux.cpp index 0c41cd5f..1f8cc8e7 100644 --- a/Source/Processes/Open.Linux.cpp +++ b/Source/Processes/Open.Linux.cpp @@ -11,6 +11,7 @@ #include "Open.Linux.hpp" #include +#include namespace Aurora::Processes { @@ -30,6 +31,6 @@ namespace Aurora::Processes AUKN_SYM void OpenFile(const AuString &file) { - LinuxOpenAsync(file); + LinuxOpenAsync(IO::FS::NormalizePathRet(file)); } } \ No newline at end of file diff --git a/Source/Processes/Open.Win32.cpp b/Source/Processes/Open.Win32.cpp index c7d8f4e7..213cd7d5 100644 --- a/Source/Processes/Open.Win32.cpp +++ b/Source/Processes/Open.Win32.cpp @@ -21,7 +21,7 @@ namespace Aurora::Processes static AuList gOpenItems; static AuThreadPrimitives::ConditionMutexUnique_t gCondMutex; static AuThreadPrimitives::ConditionVariableUnique_t gCondVariable; - static AuThreading::Threads::ThreadUnique_t gOpenerThread; + static AuThreads::ThreadUnique_t gOpenerThread; static void RunTasks() { @@ -49,9 +49,9 @@ namespace Aurora::Processes { gCondMutex = AuThreadPrimitives::ConditionMutexUnique(); gCondVariable = AuThreadPrimitives::ConditionVariableUnique(AuUnsafeRaiiToShared(gCondMutex)); - gOpenerThread = AuThreading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(OpenerThread)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}), + gOpenerThread = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(OpenerThread)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}), "COM ShellExecute Runner" )); gOpenerThread->Run(); diff --git a/Source/Processes/Process.Win32.cpp b/Source/Processes/Process.Win32.cpp index cdb743c1..3fc061d8 100644 --- a/Source/Processes/Process.Win32.cpp +++ b/Source/Processes/Process.Win32.cpp @@ -33,6 +33,9 @@ namespace Aurora::Processes } } + + // TODO: where the hell is my release + class ProcessImpl : public IProcess { public: @@ -53,6 +56,7 @@ namespace Aurora::Processes bool Read(bool error, void *buffer, AuUInt32 &len) override; bool Write(const void *buffer, AuUInt32 len) override; + // TODO: what in the hell this is ugly bool Start(enum ESpawnType type, bool fwdOut, bool fwdErr, bool fwdIn) override; private: @@ -71,9 +75,9 @@ namespace Aurora::Processes AuList cargs_; AuString windowsCli_; - Threading::Threads::ThreadUnique_t thread_; - HANDLE process_ {INVALID_HANDLE_VALUE};; - HANDLE hthread_ {INVALID_HANDLE_VALUE};; + AuThreads::ThreadUnique_t thread_; + HANDLE process_ {INVALID_HANDLE_VALUE}; + HANDLE hthread_ {INVALID_HANDLE_VALUE}; AuSInt exitCode_; }; @@ -327,9 +331,9 @@ namespace Aurora::Processes this->exitCode_ = exitCode; }; - this->thread_ = AuThreading::Threads::ThreadUnique(AuThreading::Threads::ThreadInfo( - AuMakeShared(AuThreading::Threads::IThreadVectorsFunctional::OnEntry_t(std::bind(a)), - AuThreading::Threads::IThreadVectorsFunctional::OnExit_t{}) + this->thread_ = AuThreads::ThreadUnique(AuThreads::ThreadInfo( + AuMakeShared(AuThreads::IThreadVectorsFunctional::OnEntry_t(std::bind(a)), + AuThreads::IThreadVectorsFunctional::OnExit_t{}) )); if (!this->thread_) diff --git a/Source/Threading/Threads/OSThread.cpp b/Source/Threading/Threads/OSThread.cpp index 131bf82a..387ba8f7 100644 --- a/Source/Threading/Threads/OSThread.cpp +++ b/Source/Threading/Threads/OSThread.cpp @@ -58,9 +58,16 @@ namespace Aurora::Threading::Threads OSThread::~OSThread() { + bool bDetached {}; + bool bDetachedSuccess {}; + if (contextUsed_) { - if (!detached_) + if (detached_) + { + bDetached = true; + } + else { Exit(); WaitFor(terminated_.get()); @@ -69,7 +76,25 @@ namespace Aurora::Threading::Threads terminated_.reset(); FreeOSContext(); - HookReleaseThreadResources(); + + if (bDetached) + { + if (this->tlsReferenceThread_) + { + AU_LOCK_GUARD(this->tlsLock_); + AU_LOCK_GUARD(this->tlsReferenceThread_->tlsLock_); + + std::exchange(this->tlsReferenceThread_->tls_, this->tls_); + std::exchange(this->tlsReferenceThread_->threadFeatures_, this->threadFeatures_); + + bDetachedSuccess = true; + } + } + + if (bDetachedSuccess || !bDetached) + { + HookReleaseThreadResources(); + } } // Called whenever an aurora thread is exiting @@ -92,9 +117,13 @@ namespace Aurora::Threading::Threads } } - void OSThread::AddLastHopeTlsHook(const AuSPtr &feature) + void OSThread::AddLastHopeTlsHook(const AuSPtr &feature) { // TODO: mutex + if (!feature) + { + return; + } AuTryInsert(threadFeatures_, feature); } @@ -248,13 +277,14 @@ namespace Aurora::Threading::Threads void OSThread::SetAffinity(AuUInt32 mask) { + if (mask == 0) mask = 0xFFFFFFFF; affinityProcessMask_ = mask; UpdateAffinity(mask); } void OSThread::SetPrio(EThreadPrio prio) { - prio_ = prio; + if (!EThreadPrioIsValid(prio)) return; UpdatePrio(prio); } @@ -292,7 +322,7 @@ namespace Aurora::Threading::Threads return 0; }; - auto ok = _beginthreadex(nullptr, info_.stackSize.value_or(0), OSEP_f, this, 0, 0); + auto ok = _beginthreadex(nullptr, info_.stackSize, OSEP_f, this, 0, 0); if (ok == -1L) { SysPushErrorGen("Couldn't initialize thread: {}", GetName()); @@ -318,9 +348,9 @@ namespace Aurora::Threading::Threads return false; } - if (info_.stackSize.has_value()) + if (info_.stackSize) { - ret = pthread_attr_setstacksize(&tattr, std::min(AuUInt32(PTHREAD_STACK_MIN), info_.stackSize.value())); + ret = pthread_attr_setstacksize(&tattr, std::min(AuUInt32(PTHREAD_STACK_MIN), info_.stackSize)); if (ret != 0) { SysPushErrorGen("Couldn't create thread: {}", GetName()); @@ -346,6 +376,11 @@ namespace Aurora::Threading::Threads void OSThread::_ThreadEP() { + // Poke TLS reference thread entity + // TODO: we need an internal OSThread *TryPokeTLSThread() + auto osThread = static_cast(GetThread()); + this->tlsReferenceThread_ = osThread; + OSAttach(); task_(); Exit(true); @@ -449,7 +484,7 @@ namespace Aurora::Threading::Threads this->unixThreadId_ = 0; // !!!! #endif UpdatePrio(prio_); - UpdateAffinity(affinityProcessMask_); + SetAffinity(affinityProcessMask_); UpdateName(); } @@ -529,6 +564,19 @@ namespace Aurora::Threading::Threads // fall through on error } + else if (prio_ == EThreadPrio::ePrioRT) + { + int policyNonRT = + #if defined(AURORA_IS_XNU_DERIVED) + SCHED_FIFO; + #else + SCHED_OTHER; + #endif + + sched_param param {}; + param.sched_priority = sched_get_priority_min(policyNonRT); + pthread_setschedparam(handle_, policyNonRT, ¶m); + } const int *val; if (!AuTryFind(kNiceMap, prio, val)) @@ -540,7 +588,9 @@ namespace Aurora::Threading::Threads { setpriority(PRIO_PROCESS, tid, *val); } - #endif + #endif + + prio_ = prio; } void OSThread::UpdateAffinity(AuUInt32 mask) diff --git a/Source/Threading/Threads/OSThread.hpp b/Source/Threading/Threads/OSThread.hpp index 39cd40fb..2c1f3fa7 100644 --- a/Source/Threading/Threads/OSThread.hpp +++ b/Source/Threading/Threads/OSThread.hpp @@ -62,6 +62,7 @@ namespace Aurora::Threading::Threads private: Primitives::SpinLock tlsLock_; AuSPtr tls_; + OSThread * tlsReferenceThread_ {}; AuString name_; ThreadInfo info_; AuUInt32 affinityProcessMask_ = 0xFFFFFFFF;