AuroraRuntime/Source/Threading/Threads/AuOSThread.cpp

/***
    Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.

    File: AuOSThread.cpp
    Date: 2021-6-12
    Author: Reece
***/
#include <Source/RuntimeInternal.hpp>
#include "AuThreads.hpp"
#include <Source/Threading/AuWaitFor.hpp>
#include "AuOSThread.hpp"
#include "AuThreadHandles.hpp"
#include "TLSView.hpp"

#if defined(AURORA_IS_LINUX_DERIVED)
    #include <sys/resource.h>
    #include <cxxabi.h>
#endif

#if defined(AURORA_HAS_PTHREADS)
    #include <sched.h>
#endif

#if defined(AURORA_PLATFORM_WIN32)
    #include <Aux_ulib.h>
#endif

#if defined(AURORA_IS_MODERNNT_DERIVED)
    static auto const kThreadPowerThrottling = Aurora::THREAD_INFORMATION_CLASS::ThreadPowerThrottling;

    struct THREAD_POWER_THROTTLING_STATE2
    {
        ULONG Version;
        ULONG ControlMask;
        ULONG StateMask;
    };
#endif

#if defined(AURORA_COMPILER_CLANG)
    // warning: enumeration values 'kEnumCount' and 'kEnumInvalid' not handled in switch [-Wswitch
    #pragma clang diagnostic ignored "-Wswitch"
    // Yea, I don't give a shit.
#endif

#include "AuSpawnThread.hpp"

namespace Aurora
{
    void RuntimeLateClean();
}

namespace Aurora::Threading::Threads
{
    static struct DtorFlipper
    {
        DtorFlipper() : bActive(true)
        {
        }

        ~DtorFlipper()
        {
            bActive = false;
        }
        bool bActive;

    } gGlobal;

    static ThreadInfo gDummyThreadInfo;

#if defined(AURORA_IS_POSIX_DERIVED)

    static void HandleSigAbortThread(int a)
    {
        ((OSThread *)HandleCurrent())->InternalKillForceNtfy();
        SysPanic("Couldn't terminate thread context");
    }

#endif
    
    OSThread::OSThread(const ThreadInfo &info) : info_(info),
        epExecEvent(false, false, true)
    {
        this->name_ = info.name.value_or("Aurora Thread");

        // maybe we should atomic exchange compare these when needed frogthink
        this->terminatedSignalLs_ = AuLoop::NewLSEvent(true, false, true);
        this->terminateSignalLs_  = AuLoop::NewLSEvent(true, false, true);

        this->terminated_      = AuThreadPrimitives::EventShared(true, false, true);
        this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true);

        SysAssert(this->terminatedSignalLs_ && this->terminateSignalLs_ && this->terminated_ && this->terminateSignal_);
    }

    OSThread::OSThread() : info_(gDummyThreadInfo),
        epExecEvent(false, false, true)
    {
        this->name_ = "Main Thread";

        this->terminated_      = AuThreadPrimitives::EventShared(true, false, true);
        this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true);

        SysAssert(this->terminated_ && this->terminateSignal_);
        this->bNotOwned = true;
    }

    OSThread::OSThread(AuUInt64 id) : info_(gDummyThreadInfo),
        epExecEvent(false, false, true)
    {
        this->name_ = "System Thread";
        this->handle_ = reinterpret_cast<decltype(handle_)>(id);
        this->bNotOwned = true;
    }

    bool DeadTest();

    OSThread::~OSThread()
    {
        bool bDetached {};
        bool bDetachedSuccess {};

        if (this->bNotOwned)
        {
        #if defined(AURORA_IS_MODERNNT_DERIVED)
            auto hHandle = (HANDLE)this->handle_;
            AuWin32CloseHandle(hHandle);
        #endif
            return;
        }

        if (!gGlobal.bActive)
        {
            return;
        }

#if 0
        if (DeadTest())
        {
            return;
        }
#endif
        if (this->contextUsed_)
        {
            this->epExecEvent->Wait();

            {
                AU_LOCK_GUARD(this->pFlag->mutex);
                this->pFlag->bLock = true;
            }

            if (this->detached_)
            {
                bDetached = true;
            }
            else
            {
                if (gRuntimeRunLevel <= 3)
                {
                    Exit();
                }
                else if (gRuntimeRunLevel >= 5)
                {
                    // Application is dead
                }
                else
                {
                    if (this->terminated_)
                    {
                        this->exiting_ = true;
                        if (this->terminated_->LockMS(5'000))
                        {
                            return;
                        }
                    }

                    // Kill the current OS thread instance
                    TeminateOSContext(false);

                    this->exitOnlyOnce_->Unlock();
                
                #if defined(AURORA_IS_POSIX_DERIVED)
                    if (this->terminated_)
                    {
                        this->terminated_->Lock();
                    }
                #endif
                }
            }
        }
        else
        {
            if (auto pTerminated = AuExchange(this->terminatedSignalLs_, {}))
            {
                pTerminated->Set();
            }

            if (auto pTerminated = AuExchange(this->terminated_, {}))
            {
                pTerminated->Set();
            }
        }

        FreeOSContext();
        
        if (bDetached)
        {
            if (this->tlsReferenceThread_)
            {
                AU_LOCK_GUARD(this->tlsLock_);
                AU_LOCK_GUARD(this->tlsReferenceThread_->tlsLock_);

                SetThreadKey(this->tlsReferenceThread_->tls_);
                AuExchange(this->tlsReferenceThread_->tls_, this->tls_);
                AuExchange(this->tlsReferenceThread_->threadFeatures_, this->threadFeatures_);
                
                bDetachedSuccess = true;
            }
        }

        if (bDetachedSuccess || !bDetached)
        {
            HookReleaseThreadResources();
        }
    }

    // Called whenever an aurora thread is exiting
    void OSThread::HookOnExit()
    {

    }

    // Called whenever a thread object is released, or on thread termination
    // Whichever is soonest
    // It is possible a user may wish to keep a thread handle open
    // Is is expected for a system thread to release on dtor unlike our aurora threads 
    void OSThread::HookReleaseThreadResources()
    {
        AU_LOCK_GUARD(this->tlsLock_);
        for (const auto &feature : this->threadFeatures_)
        {
            feature->Cleanup();
        }
    }

    AuUInt64 OSThread::SetNoUnwindTerminateExitWatchDogTimeoutInMS(AuUInt64 uMS)
    {
        if (!uMS)
        {
            uMS = AuSToMS<AuUInt64>(15);
        }

        qwExitTimeout = uMS;
        return qwExitTimeout;
    }

    void OSThread::AddLastHopeTlsHook(const AuSPtr<AuThreads::IThreadFeature> &feature)
    {
        AU_LOCK_GUARD(this->tlsLock_);
        if (!feature)
        {
            return;
        }
        AuTryInsert(this->threadFeatures_, feature);
    }

    void OSThread::Detach()
    {
        this->detached_ = true;
    }

    AuSPtr<IWaitable> OSThread::AsWaitable()
    {
        if (!this->terminated_)
        {
            this->terminated_ = AuThreadPrimitives::EventShared(true, false, true);
        }
        return this->terminated_;
    }

    void OSThread::SendExitSignal()
    {
        this->exiting_ = true;

        if (this->terminateSignalLs_)
        {
            this->terminateSignalLs_->Set();
        }

        if (this->terminateSignal_)
        {
            this->terminateSignal_->Set();
        }
    }

    void OSThread::UnsafeForceTerminateSignal()
    {
        TeminateOSContext(false);
    }

    bool OSThread::Run()
    {
        if (!this->terminated_)
        {
            SysPanic("::Run called on system thread");
        }

        if (AuExchange(this->contextUsed_, true))
        {
            return false;
        }

        this->terminated_->Reset();

        return ExecuteNewOSContext([=]()
        {
            try
            {
                // this functional backends are being deprecated
                if (this->info_.callbacks)
                {
                    this->info_.callbacks->OnEntry(this);
                }
            }
            catch (...)
            {
                Debug::PrintError();
            }
        });
    }

    void OSThread::Exit()
    {
        this->PrivateUserDataClear();
        
        while ((!this->terminated_) || (!this->terminated_->TryLock()))
        {
            if (Exit(false)) break;
        }
    }

    bool OSThread::Exit(bool willReturnToOS, bool isEOL)
    {
        if (GetThread() == this)
        {
            if (!this->InternalKill(false))
            {
                return false;
            }

            // release handle + sigterm
            if (!willReturnToOS)
            {
                TeminateOSContext(true);
                while (true)
                {
                    ContextYield();
                }
            }
        }
        else
        {
            // exit signal 
            this->exiting_ = true;

            if (!this->terminated_)
            {
                return true;
            }


            // attempt to join with the thread once it has exited, or timeout 
            if (this->terminated_->LockMS(this->qwExitTimeout))
            {
                return true;
            }

            // Do not force terminate if we're marked as dead and still running
            // The thread must've requested suicide and got stuck in a lengthy clean up effort
            if (!this->exitOnlyOnce_->TryLock())
            {
                AuLogWarn("Watchdog error - OS thread context didn't finish in {} MS, but he should exiting now.", this->qwExitTimeout);
                return false;
            }

            AuLogWarn("Watchdog error - OS thread context didn't finish in {} MS, forcefully terminating without a watchdog overlooking onExit", this->qwExitTimeout);

            // Kill the current OS thread instance
            TeminateOSContext(false);

            // Dispatch kill callback from within an emulated thread context (no switch, just just change thread tls)
            ExecuteInDeadThread([=]()
            {
                this->InternalKill(true);
            });

            this->exitOnlyOnce_->Unlock();
        }
        return true;
    }

    bool OSThread::Exiting()
    {
        return this->exiting_;
    }

    void OSThread::SetName(const AuString &name)
    {
        if (gRuntimeConfig.threadingConfig.bNoThreadNames)
        {
            return;
        }
        this->name_ = name;
        this->UpdateName();
    }

    EThreadThrottle OSThread::GetThrottle()
    {
        return this->throttle_;
    }

    EThreadPriority OSThread::GetPriority()
    {
        return this->prio_;
    }

    AuHwInfo::CpuBitId OSThread::GetMask()
    {
        return this->mask_;
    }

    AuString OSThread::GetName()
    {
        return this->name_;
    }

    void OSThread::SetAffinity(const HWInfo::CpuBitId &mask)
    {
        auto zero = HWInfo::CpuBitId();
        
        if (mask == zero ||
            mask == zero.Not())
        {
            this->mask_ = HWInfo::GetCPUInfo().maskAllCores;
            this->userManagingAffinity_ = false;

            UpdatePrio(this->throttle_, this->prio_);
        }
        else
        {
            this->mask_ = mask;
            this->userManagingAffinity_ = true;
        }

        UpdateAffinity(this->mask_);
        AffinityPrioThrottleTickAmendECores();
    }

    void OSThread::SetPriority(EThreadPriority prio)
    {
        if (!EThreadPriorityIsValid(prio))
        {
            return;
        }

        UpdatePrio(this->throttle_, prio);
        AffinityPrioThrottleTickAmendECores();
    }

    void OSThread::SetThrottle(EThreadThrottle throttle)
    {
        if (!EThreadThrottleIsValid(throttle))
        {
            return;
        }

        UpdatePrio(throttle, this->prio_);
        this->throttle_ = throttle;
        AffinityPrioThrottleTickAmendECores();
    }

    AuSPtr<TLSView> OSThread::GetTlsView()
    {
        SysPanic("Deprecated Concept");
    }

    bool OSThread::ExecuteNewOSContext(AuFunction<void()> task)
    {
        this->task_ = task;

        this->handle_ = 0;

        auto ret = SpawnThread([this]()
        {
            while ((!this->handle_) ||
                #if defined(INVALID_HANDLE_VALUE)
                   (this->handle_ == INVALID_HANDLE_VALUE)
                #else
                   0
                #endif
                   )
            {
                AuThreading::ContextYield();
            }

            if (!this->tls_)
            {
                this->tls_ = GetThreadKey();
            }

            this->_ThreadEP();
        }, GetName(), this->info_.stackSize);
        
        if (ret.first)
        {
            this->handle_ = (decltype(handle_))ret.second;
        }

        return ret.first;
    }

    void OSThread::AffinityPrioThrottleTickAmendECores()
    {
        auto zero = HWInfo::CpuBitId();
        auto &cpuInfo = AuHwInfo::GetCPUInfo();
        auto sysEcores = cpuInfo.maskECores;
        bool bIsMaskProbablyDefault = this->mask_ == zero || this->mask_ == zero.Not();
        bool bSystemHasNoECores = sysEcores == zero;
        EThreadPriority prio = this->prio_;

        if (this->throttle_ == EThreadThrottle::eEfficient &&
            bSystemHasNoECores)
        {
            if (bIsMaskProbablyDefault)
            {
                sysEcores = cpuInfo.maskAllCores;
            }

            /**
             * We have a lot of code that will specify an efficiency throttle and then
             * arbitrarily use prio levels to (hopefully) order work in the os scheduler.
             * Systems without ecores would see a higher prio work than normal tasks if
             * we do nothing about this. On systems without ecores, efficiency throttle 
             * will just lock the prio to low.
            */
            prio = EThreadPriority::ePrioLow;

            UpdatePrio(this->throttle_, prio);
            UpdateAffinity(sysEcores);
        }
    }

    void AttachSignalKiller();

    void OSThread::_ThreadEP()
    {
        // Poke TLS reference thread entity
        // TODO: we need an internal OSThread *TryPokeTLSThread()
        auto osThread = static_cast<OSThread *>(GetThread());
        this->tlsReferenceThread_ = osThread;
        bool bFailing {};

        OSAttach();

    #if defined(AURORA_IS_POSIX_DERIVED)
        this->bLongJmpOnce = false;
        if (setjmp(env) != 0)
        {
            Exit(true);
            return;
        }

        this->bSupportsAltKill = true;
    #endif

        auto pFlag = this->pFlag;

        auto pA = this->terminatedSignalLs_;
        auto pB = this->terminated_;

        try
        {
            if (auto task = task_)
            {
                this->epExecEvent->Set();
                task();
            }
        }
#if !defined(AURORA_IS_POSIX_DERIVED)
        catch (...)
        {
            SysPushErrorHAL("OS Thread Aborted");
        }

        AU_LOCK_GUARD(pFlag->mutex);
        if (pFlag->bLock)
        {
            if (pA)
            {
                pA->Set();
            }
            if (pB)
            {
                pB->Set();
            }
            return;
        }

        this->SignalDeath();
        Exit(true);
#else
    #if defined(AURORA_COMPILER_GCC)
        catch (abi::__forced_unwind&)
        {
            throw;
        }
    #endif
        catch (...)
        {
            AU_LOCK_GUARD(pFlag->mutex);
            if (pFlag->bLock)
            {
                return;
            }

            bFailing = true;
            if (!Aurora::kIsDebugBuild)
            {
                SysPushErrorHAL("OS Thread Aborted");
            }

            // "Safer" update 
            this->HookOnExit();

            this->SignalDeath();
        }

        AU_LOCK_GUARD(pFlag->mutex);
        if (pFlag->bLock)
        {
            if (pA)
            {
                pA->Set();
            }
            if (pB)
            {
                pB->Set();
            }
            return;
        }

        if (!bFailing)
        {
            Exit(true);
        }
#endif
    }

    void OSThread::ExecuteInDeadThread(AuFunction<void()> callback)
    {
        auto old = HandleCurrent();

        if (this == old) [[unlikely]]
        {
            callback();
            return;
        }

        if (old)
        {
            static_cast<OSThread *>(old)->OSDeatach();
        }
        
        auto uOldHandle = GetThreadKey();
        this->OSAttach();
        
        callback();

        if (old)
        {
            HandleRegister(old);
            static_cast<OSThread *>(old)->OSDeatach();
        }
        else [[unlikely]]
        {
            HandleRemove();
        }

        SetThreadKey(uOldHandle);
    }

    void OSThread::UpdateName()
    {
        if (gRuntimeConfig.threadingConfig.bNoThreadNames)
        {
            return;
        }

    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if ((AuBuild::kCurrentPlatform == AuBuild::EPlatform::ePlatformWin32) && (!AuSwInfo::IsWindows10OrGreater()))
        {
            static const DWORD kMSVCExceptionSetName = 0x406D1388;

        #pragma pack(push,8)
            typedef struct tagTHREADNAME_INFO
            {
                DWORD dwType;
                LPCSTR szName;
                DWORD dwThreadID;
                DWORD dwFlags;
            } THREADNAME_INFO;
        #pragma pack(pop)

            if (!IsDebuggerPresent())
            {
                return;
            }

            THREADNAME_INFO info;
            info.dwType = 0x1000;
            info.szName = this->name_.c_str();
            if (pGetThreadId)
            {
                info.dwThreadID = pGetThreadId(this->handle_);
            }
            else if (this->unixThreadId_ == GetCurrentThreadId())
            {
                info.dwThreadID = this->unixThreadId_;
            }
            else
            {
                return;
            }

            info.dwFlags = 0;

            auto raise = AuStaticCast<void(__cdecl *)(THREADNAME_INFO &)>([](THREADNAME_INFO &info)
            {
                __try
                {
                    RaiseException(kMSVCExceptionSetName, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR *)&info);
                }
                __except (EXCEPTION_EXECUTE_HANDLER)
                {
                }
            });

            raise(info);
        }
        else
        {
            if (pSetThreadDescription)
            {
                pSetThreadDescription(this->handle_, AuLocale::ConvertFromUTF8(this->name_).c_str());
            }
        }

    #elif defined(AURORA_HAS_PTHREADS)

        if (!this->handle_)
        {
            return;
        }
        
        pthread_setname_np(this->handle_, this->name_.c_str());
    
    #endif
    }

    static void AttachSignalKiller()
    {
    #if defined(AURORA_IS_POSIX_DERIVED)
        struct sigaction action =
        {
            .sa_handler = HandleSigAbortThread,
            .sa_flags = 0
        };
        ::sigemptyset(&action.sa_mask);
        ::sigaction(SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate, &action, nullptr);
    #endif
    }

    void OSThread::MakeMain()
    {
        this->bNotOwned = true;
        this->name_ = "Main System Thread";
        this->OSAttach();
    }

    void OSThread::OSAttach()
    {
        HandleRegister(this);
    #if defined(AURORA_IS_LINUX_DERIVED)
        this->unixThreadId_ = gettid();
    #elif defined(AURORA_IS_XNU_DERIVED)
        this->unixThreadId_ = pthread_getthreadid_np();
    #elif defined(AURORA_IS_BSD_DERIVED)
        #if __FreeBSD_version < 900031
            long lwpid;
            thr_self(&lwpid);
            this->unixThreadId_ = lwpid;
        #elif __FreeBSD_version > 900030
            this->unixThreadId_ = pthread_getthreadid_np();
        #else
            static_assert(false);
        #endif
    #elif defined(AURORA_HAS_PTHREADS)
        this->unixThreadId_ = 0; // !!!!
    #endif

    #if defined(AURORA_PLATFORM_WIN32)
        this->unixThreadId_ = GetCurrentThreadId();
    #endif

        if (this->tls_)
        {
            SetThreadKey(this->tls_);
        }
        
        UpdatePrio(this->throttle_, this->prio_);
        SetAffinity(this->mask_);
        AffinityPrioThrottleTickAmendECores();
        UpdateName();
    }

    static AuHashMap<EThreadPriority, int> kNiceMap 
    {
        {
            EThreadPriority::ePrioRT, -19
        },
        {
            EThreadPriority::ePrioAboveHigh, -15
        },
        {
            EThreadPriority::ePrioHigh, -11
        },
        {
            EThreadPriority::ePrioNormal, -2
        },
        {
            EThreadPriority::ePrioLow, 5
        },
        {
            EThreadPriority::ePrioLowest, 15
        }
    };

#if defined(AURORA_IS_MODERNNT_DERIVED)
    static const AuHashMap<EThreadPriority, int> kWin32Map
    {
        {
            EThreadPriority::ePrioRT, THREAD_PRIORITY_TIME_CRITICAL
        },
        {
            EThreadPriority::ePrioAboveHigh, THREAD_PRIORITY_HIGHEST
        },
        {
            EThreadPriority::ePrioHigh, THREAD_PRIORITY_ABOVE_NORMAL
        },
        {
            EThreadPriority::ePrioNormal, THREAD_PRIORITY_NORMAL
        },
        {
            EThreadPriority::ePrioLow, THREAD_PRIORITY_BELOW_NORMAL
        },
        {
            EThreadPriority::ePrioLowest, THREAD_PRIORITY_LOWEST
        }
    };
#endif


#if defined(AURORA_IS_XNU_DERIVED)
    static const AuHashMap<AuPair<EThreadThrottle, EThreadPriority>, AuPair<AuUInt32, int>> kXnuQoSLevel
    {
        // Rt
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 5)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
        },

        // Perf
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 8)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 3)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 0)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
        },

        // Efficient
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 10)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 1)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -1)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -2)
        }
    };
#endif

    void OSThread::UpdatePrio(EThreadThrottle throttle, EThreadPriority prio)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        const int *val;
        if (!AuTryFind(kWin32Map, prio, val))
        {
            return;
        }

        if (!SetThreadPriority(this->handle_, *val))
        {
            SysPushErrorHAL("Couldn't update thread priority");
        }

        THREAD_POWER_THROTTLING_STATE2 throttlingState {};
        throttlingState.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;

        switch (throttle)
        {
        case EThreadThrottle::eNormal:
            throttlingState.ControlMask = 0;
            throttlingState.StateMask = 0;
            break;
        case EThreadThrottle::ePerformance:
            throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            throttlingState.StateMask = 0;
            break;
        case EThreadThrottle::eEfficient:
            throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            throttlingState.StateMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            break;
        }

        if (pSetThreadInformation)
        {
            pSetThreadInformation(this->handle_,
                                  kThreadPowerThrottling,
                                  &throttlingState,
                                  sizeof(throttlingState));
        }
      
    //#elif defined(AURORA_IS_XNU_DERIVED)

    #elif defined(AURORA_HAS_PTHREADS)
        sched_param param {};
        int policyNonRT {};
        int schedA {};
        int scheduler {};
    #if defined(AURORA_IS_LINUX_DERIVED)
        rlimit rl;
    #endif

    #if defined(AURORA_IS_POSIX_DERIVED)
        static const auto GetPrioLevel = [](int sched, EThreadPriority prio)
        {
            const int *val;
            if (!AuTryFind(kNiceMap, prio, val))
            {
                return (int)sched_get_priority_max(sched);
            }

            int min = sched_get_priority_min(sched);
            int max = sched_get_priority_max(sched);

            float f = ((1.0 - ((float(*val) + 19.0) / (19.0 + 20.0))) * float(max - min)) + float(min);
            return (int)f;
        };
    #endif

        if (!this->handle_)
        {
            return;
        }

        if (prio == EThreadPriority::ePrioRT)
        {
        #if defined(AURORA_IS_POSIX_DERIVED)

        #if defined(SCHED_RR)
            param.sched_priority = sched_get_priority_max((schedA = SCHED_RR));
        #else
            param.sched_priority = sched_get_priority_max((schedA = SCHED_FIFO));
        #endif

            if (::sched_setscheduler(this->unixThreadId_, schedA, &param) == 0)
            {
                goto gtfo;
            }

            {

            #if defined(AURORA_IS_LINUX_DERIVED)
                if (getrlimit(RLIMIT_RTPRIO, &rl) == 0)
                {
                    bool bWarnOnce {};
                    rl.rlim_max = param.sched_priority;

                    if (rl.rlim_max != RLIM_INFINITY)
                    {
                        AuLogWarn("We will not use RT-Kit to keep your Linux machine alive through XDG_FREEEEEEDESKTOP_HOPES_AND_DREAMS");
                        AuLogWarn("Hint: try (1) : in bash, elevate using: '[doas/sudo] prlimit --rtprio=unlimited --pid=$$' before running the main executable");
                        AuLogWarn("Hint: try (2) : append '* hard rtprio unlimited' to /etc/security/limits.conf");
                        AuLogWarn("Hint: try (3) : check /etc/systemd/[system/user].conf");
                        bWarnOnce = true;
                        rl.rlim_cur = rl.rlim_max;
                    }
                    else
                    {
                        rl.rlim_cur = rl.rlim_max;
                    }

                    param.sched_priority = rl.rlim_cur;

                #if defined(SCHED_RR)
                    if (::sched_setscheduler(this->unixThreadId_, SCHED_RR, &param) == 0)
                    {
                        goto gtfo;
                    }
                #endif

                    if (::setrlimit(RLIMIT_RTPRIO, &rl) != 0 &&
                        !bWarnOnce)
                    {
                        AuLogWarn("We will not use RT-Kit to keep your Linux machine alive through XDG_FREEEEEEDESKTOP_HOPES_AND_DREAMS");
                        AuLogWarn("Hint: try (1) : in bash, elevate using: '[doas/sudo] prlimit --rtprio=unlimited --pid=$$' before running the main executable");
                        AuLogWarn("Hint: try (2) : append '* hard rtprio unlimited' to /etc/security/limits.conf");
                        AuLogWarn("Hint: try (3) : check /etc/systemd/[system/user].conf");
                    }

                #if defined(SCHED_RR)
                    if (::sched_setscheduler(this->unixThreadId_, SCHED_RR, &param) == 0)
                    {
                        goto gtfo;
                    }
                #endif

                    param.sched_priority = sched_get_priority_max(SCHED_FIFO);

                    if (::sched_setscheduler(this->unixThreadId_, SCHED_FIFO, &param) == 0)
                    {
                        goto gtfo;
                    }
                }
                else
            #endif
                {
                    param.sched_priority = sched_get_priority_max(SCHED_FIFO);

                    if (::sched_setscheduler(this->unixThreadId_, SCHED_FIFO, &param) == 0)
                    {
                        goto gtfo;
                    }
                }
            }

        #else
        #if defined(SCHED_RR)
            param.sched_priority = sched_get_priority_min(SCHED_RR);

            if (pthread_setschedparam(this->handle_, SCHED_RR, &param) == 0)
            {
                goto gtfo;
            }
        #endif

            param.sched_priority = GetPrioLevel(SCHED_FIFO, prio);

            if (pthread_setschedparam(this->handle_, SCHED_FIFO, &param) == 0)
            {
                goto gtfo;
            }
        #endif

            // fall through on error
        }
    #if defined(AURORA_IS_LINUX_DERIVED)
        else if (throttle == EThreadThrottle::eEfficient)
        {
            param.sched_priority = GetPrioLevel(SCHED_IDLE, prio);

            if (::sched_setscheduler(this->unixThreadId_, SCHED_IDLE, &param) == 0)
            {
                goto gtfo;
            }

            // fail
            return;
        }
        else if (this->throttle_ == EThreadThrottle::eEfficient ||
                 this->prio_ == EThreadPriority::ePrioRT)
    #else
        else if (this->prio_ == EThreadPriority::ePrioRT)
    #endif
        {
        #if defined(AURORA_IS_POSIX_DERIVED)
            policyNonRT = 
            #if defined(AURORA_IS_XNU_DERIVED)
                SCHED_FIFO;
            #else
                SCHED_OTHER;
            #endif  

            param.sched_priority = GetPrioLevel(policyNonRT, prio);

            if (::sched_setscheduler(this->unixThreadId_, policyNonRT, &param) == 0)
            {
                goto gtfo;
            }

            return;
        #endif
        }

        {
            scheduler = ::sched_getscheduler(this->unixThreadId_);
            param.sched_priority = GetPrioLevel(scheduler, prio);

            if (::sched_setscheduler(this->unixThreadId_, scheduler, &param) == 0)
            {
                goto gtfo;
            }
        }
     
        // TODO: We used to set the legacy unix niceness value here.
        AuLogWarn("Couldn't set affinity");
    #endif

    gtfo:
    #if defined(AURORA_IS_LINUX_DERIVED) || defined(AURORA_IS_BSD_DERIVED)
        switch (throttle)
        {
        case EThreadThrottle::eNormal:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskAllCores;
            break;
        case EThreadThrottle::ePerformance:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskPCores;
            break;
        case EThreadThrottle::eEfficient:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskECores;
            break;
        }

        UpdateAffinity(this->mask_);
    #endif

        this->prio_ = prio;
    }

    void OSThread::UpdateAffinity(const HWInfo::CpuBitId &mask)
    {

    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if ((AuBuild::kCurrentPlatform != AuBuild::EPlatform::ePlatformWin32) ||
            (AuSwInfo::IsWindows10OrGreater()))
        {
            auto sets = mask.CpuBitCount() ? 
                        mask.ToCpuSets() :
                        AuHwInfo::GetCPUInfo().maskAllCores.ToCpuSets();

            if (pSetThreadSelectedCpuSets)
            {
                if (pSetThreadSelectedCpuSets(this->handle_, sets.data(), sets.size()))
                {
                    return;
                }
                
                SysPushErrorUnavailableError("SetThreadSelectedCpuSets is expected on modern NT (CoreOS?) excluding Windows; or Win10+");
            }
        }

    #if defined(AURORA_PLATFORM_WIN32)
        if (pSetThreadGroupAffinity)
        {
            GROUP_AFFINITY affinityGroup { 0 };
            mask.ToMsWin7GroupAffinity(&affinityGroup);
            if (pSetThreadGroupAffinity(this->handle_, &affinityGroup, nullptr))
            {
                return;
            }
        }

        if (mask.CpuBitCount() <= 32)
        {
            DWORD_PTR uProcMask {};
            DWORD_PTR uSysMask {};
            AuUInt32 uProcmask { AuUInt32(mask.lower) };

            if (::GetProcessAffinityMask(GetCurrentProcess(), &uProcMask, &uSysMask))
            {
                uProcmask &= uProcMask;
            }

            if (::SetThreadAffinityMask(this->handle_, uProcmask))
            {
                return;
            }
        }
    #endif

        SysPushErrorUnavailableError("Couldn't set thread affinity");
        return;

    #elif defined(AURORA_HAS_PTHREADS)

        if (!this->handle_)        
        {
             return;
        }

        auto mask2 = mask.And(this->throttleMask_);

        if (mask2.CpuBitCount() == 0)
        {
            switch (this->throttle_)
            {
            case EThreadThrottle::eNormal:
                mask2 = AuHwInfo::GetCPUInfo().maskAllCores;
                break;
            case EThreadThrottle::ePerformance:
                mask2 = AuHwInfo::GetCPUInfo().maskPCores;
                break;
            case EThreadThrottle::eEfficient:
                mask2 = AuHwInfo::GetCPUInfo().maskECores;
                break;
            }
        }

        cpu_set_t cpuset;
        CPU_ZERO(&cpuset);
        
        AuUInt8 index {};
        while (mask2.CpuBitScanForward(index, index))
        {
            CPU_SET(index, &cpuset);
            index++;
        }

        if (!CPU_COUNT(&cpuset))
        {
            return;
        }

        if (pthread_setaffinity_np(this->handle_, sizeof(cpuset), &cpuset) != 0)
        {
            SysPushErrorHAL("Couldn't set affinity mask");
        }

        return;

    #endif
    }

    void OSThread::OSDeatach()
    {
    #if defined(AURORA_IS_POSIX_DERIVED)
        this->bSupportsAltKill = false;
        //struct sigaction action =
        //{
        //    .sa_handler = SIG_DFL
        //};
        //::sigemptyset(&action.sa_mask);
        //::sigaction(SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate, &action, nullptr);
    #endif
    }

    bool OSThread::InternalKill(bool locked)
    {
        this->PrivateUserDataClear();

        if (this->terminated_)
        {
            if (!locked)
            {
                if (!this->exitOnlyOnce_->TryLock())
                {
                    return false;
                }
            }
        }

        HookOnExit();

        try
        {
            // dispatch kill callback
            if (this->info_.callbacks)
            {
                this->info_.callbacks->OnExit(this);
            }
        }
        catch (...)
        {
            Debug::PrintError();
            AuLogWarn("Couldn't deinitialize thread");
            AuLogWarn("The smart thing to do at this point would be to panic");
            AuLogWarn("...but we could continue");
            AuLogWarn("Carrying on despite the potential for data integrity loss and memory leaks");
            Telemetry::Mayday();
        }

        HookReleaseThreadResources();

        this->SignalDeath();

        return true;
    }
    
    void OSThread::SignalDeath()
    {
        if (this->terminatedSignalLs_)
        {
            this->terminatedSignalLs_->Set();
        }

        if (this->terminated_)
        {
            this->exitOnlyOnce_->Unlock();
            this->terminated_->Set();
        }
    }

    bool OSThread::InternalKillForceNtfy()
    {
        this->SignalDeath();
  

        // Great C++ ABI guys...
    #if defined(AURORA_HAS_PTHREADS)
        {
            if (gettid() == getpid())
            {
                AuProcess::Exit(0);
                return true;
            }
            ::longjmp(env, 1);
        }
        ::pthread_exit(nullptr);
    #endif

        return true;
    }

    void OSThread::TeminateOSContext(bool calledFromThis)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if (calledFromThis)
        {
            ::ExitThread(0);
        }
        else
        {
            ::TerminateThread(this->handle_, 0);
        }

    #elif defined(AURORA_HAS_PTHREADS)

        if (calledFromThis)
        {
            ::longjmp(env, 1);
            ::pthread_exit(nullptr);
        }
        else
        {   
            // pthreads is fun. thats not how unix works...
            // pthread_kill(this->handle_, SIGKILL);
            // remember signal inheritance is a cluster fuck & the tree will be walked
            // this is giving me flashbacks to hacking in apcs into the kernel
            // gross...
            // let's let nptl handle it
            //pthread_cancel(this->handle_);
            // ...doesnt work with c++ bc catch handlers get in the way. FUCK
            
            if (this->bSupportsAltKill)
            {
                ::pthread_kill(this->handle_, SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate);
            }
            else
            {
                ::pthread_cancel(this->handle_);
            }
        }

    #else
    
        SysPanic("Not implemented");
    
    #endif
    }

    void OSThread::FreeOSContext()
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        AuWin32CloseHandle(this->handle_);
    #endif
    }

    AuSPtr<AuLoop::ILoopSource> OSThread::AsLoopSource()
    {
        if (!this->terminatedSignalLs_)
        {
            this->terminatedSignalLs_ = AuLoop::NewLSEvent(true, false, true);
        }
        return this->terminatedSignalLs_;
    }
        
    AuSPtr<IWaitable> OSThread::GetShutdownSignalWaitable()
    {
        return this->terminateSignal_;
    }

    AuSPtr<AuLoop::ILoopSource> OSThread::GetShutdownSignalLS()
    {
        return this->terminateSignalLs_;
    }

    void InitThreading()
    {
    #if defined(AURORA_PLATFORM_WIN32)
        AuxUlibInitialize();
    #endif

        AttachSignalKiller();
        
        if (auto pThread = GetThread())
        {
            AuStaticCast<OSThread>(pThread)->MakeMain();
        }
    }
}