AuroraRuntime/Source/Threading/Threads/AuOSThread.cpp

/***
    Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.

    File: AuOSThread.cpp
    Date: 2021-6-12
    Author: Reece
***/
#include <Source/RuntimeInternal.hpp>
#include "AuThreads.hpp"
#include <Source/Threading/AuWaitFor.hpp>
#include "AuOSThread.hpp"
#include "AuThreadHandles.hpp"
#include "TLSView.hpp"

#if defined(AURORA_IS_LINUX_DERIVED)
    #include <sys/resource.h>
    #include <cxxabi.h>
#endif

#if defined(AURORA_HAS_PTHREADS)
    #include <sched.h>
#endif

#if defined(AURORA_PLATFORM_WIN32)
    #include <Aux_ulib.h>
#endif

#if defined(AURORA_IS_MODERNNT_DERIVED)
    static auto const kThreadPowerThrottling = Aurora::THREAD_INFORMATION_CLASS::ThreadPowerThrottling;

    struct THREAD_POWER_THROTTLING_STATE2
    {
        ULONG Version;
        ULONG ControlMask;
        ULONG StateMask;
    };
#endif

#include "AuSpawnThread.hpp"

namespace Aurora
{
    void RuntimeLateClean();
}

namespace Aurora::Threading::Threads
{
    static struct DtorFlipper
    {
        DtorFlipper() : bActive(true)
        {
        }

        ~DtorFlipper()
        {
            bActive = false;
        }
        bool bActive;

    } gGlobal;

    static ThreadInfo gDummyThreadInfo;

#if defined(AURORA_IS_POSIX_DERIVED)
    static const auto kSignalAbort = SIGUSR1;

    static void HandleSigAbortThread(int a)
    {
        ((OSThread *)HandleCurrent())->InternalKillForceNtfy();
        SysPanic("Couldn't terminate thread context");
    }

#endif
    
    OSThread::OSThread(const ThreadInfo &info) : info_(info)
    {
        this->name_ = info.name.value_or("Aurora Thread");

        // maybe we should atomic exchange compare these when needed frogthink
        this->terminatedSignalLs_ = AuLoop::NewLSEvent(true, false, true);
        this->terminateSignalLs_  = AuLoop::NewLSEvent(true, false, true);

        this->terminated_      = AuThreadPrimitives::EventShared(true, false, true);
        this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true);

        SysAssert(this->terminatedSignalLs_ && this->terminateSignalLs_ && this->terminated_ && this->terminateSignal_);
    }

    OSThread::OSThread() : info_(gDummyThreadInfo)
    {
        this->name_ = "Main Thread";

        this->terminated_      = AuThreadPrimitives::EventShared(true, false, true);
        this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true);

        SysAssert(this->terminated_ && this->terminateSignal_);
    }

    OSThread::OSThread(AuUInt64 id) : info_(gDummyThreadInfo)
    {
        this->name_ = "System Thread";
        this->handle_ = reinterpret_cast<decltype(handle_)>(id);
        this->bNotOwned = true;
    }

    bool DeadTest();

    OSThread::~OSThread()
    {
        bool bDetached {};
        bool bDetachedSuccess {};

        if (this->bNotOwned)
        {
        #if defined(AURORA_IS_MODERNNT_DERIVED)
            auto hHandle = (HANDLE)this->handle_;
            AuWin32CloseHandle(hHandle);
        #endif
            return;
        }

        if (!gGlobal.bActive)
        {
            return;
        }

#if 0
        if (DeadTest())
        {
            return;
        }
#endif
        if (this->contextUsed_)
        {
            if (this->detached_)
            {
                bDetached = true;
            }
            else
            {
                if (gRuntimeRunLevel <= 3)
                {
                    Exit();
                }
                else if (gRuntimeRunLevel >= 5)
                {
                    // Application is dead
                }
                else
                {
                    if (this->terminated_)
                    {
                        this->exiting_ = true;
                        if (this->terminated_->LockMS(5'000))
                        {
                            return;
                        }
                    }

                    // Kill the current OS thread instance
                    TeminateOSContext(false);

                    if (this->exitOnlyOnce_)
                    {
                        this->exitOnlyOnce_->Unlock();
                    }
                
                #if defined(AURORA_IS_POSIX_DERIVED)
                    if (this->terminated_)
                    {
                        WaitFor(this->terminated_.get());
                    }
                #endif
                }
            }
        }

        this->terminated_.reset();
        FreeOSContext();
        
        if (bDetached)
        {
            if (this->tlsReferenceThread_)
            {
                AU_LOCK_GUARD(this->tlsLock_);
                AU_LOCK_GUARD(this->tlsReferenceThread_->tlsLock_);

                SetThreadKey(this->tlsReferenceThread_->tls_);
                AuExchange(this->tlsReferenceThread_->tls_, this->tls_);
                AuExchange(this->tlsReferenceThread_->threadFeatures_, this->threadFeatures_);
                
                bDetachedSuccess = true;
            }
        }

        if (bDetachedSuccess || !bDetached)
        {
            HookReleaseThreadResources();
        }
    }

    // Called whenever an aurora thread is exiting
    void OSThread::HookOnExit()
    {

    }

    // Called whenever a thread object is released, or on thread termination
    // Whichever is soonest
    // It is possible a user may wish to keep a thread handle open
    // Is is expected for a system thread to release on dtor unlike our aurora threads 
    void OSThread::HookReleaseThreadResources()
    {
        AU_LOCK_GUARD(this->tlsLock_);
        for (const auto &feature : this->threadFeatures_)
        {
            feature->Cleanup();
        }
    }

    void OSThread::AddLastHopeTlsHook(const AuSPtr<AuThreads::IThreadFeature> &feature)
    {
        AU_LOCK_GUARD(this->tlsLock_);
        if (!feature)
        {
            return;
        }
        AuTryInsert(this->threadFeatures_, feature);
    }

    void OSThread::Detach()
    {
        this->detached_ = true;
    }

    AuSPtr<IWaitable> OSThread::AsWaitable()
    {
        return this->terminated_;
    }

    void OSThread::SendExitSignal()
    {
        this->exiting_ = true;

        if (this->terminateSignalLs_)
        {
            this->terminateSignalLs_->Set();
        }

        if (this->terminateSignal_)
        {
            this->terminateSignal_->Set();
        }
    }

    void OSThread::UnsafeForceTerminateSignal()
    {
        TeminateOSContext(false);
    }

    bool OSThread::Run()
    {
        if (!this->terminated_)
        {
            SysPanic("::Run called on system thread");
        }

        if (AuExchange(this->contextUsed_, true))
        {
            return false;
        }

        this->terminated_->Reset();

        return ExecuteNewOSContext([=]()
        {
            try
            {
                // this functional backends are being deprecated
                if (this->info_.callbacks)
                {
                    this->info_.callbacks->OnEntry(this);
                }
            }
            catch (...)
            {
                Debug::PrintError();
            }
        });
    }

    void OSThread::Exit()
    {
        this->PrivateUserDataClear();
        
        while ((!this->terminated_) || (!this->terminated_->TryLock()))
        {
            if (Exit(false)) break;
        }
    }

    bool OSThread::Exit(bool willReturnToOS, bool isEOL)
    {
        if (GetThread() == this)
        {
            if (!this->InternalKill(false))
            {
                return false;
            }

            // release handle + sigterm
            if (!willReturnToOS)
            {
                TeminateOSContext(true);
                while (true)
                {
                    YieldToOtherThread();
                }
            }
        }
        else
        {
            // exit signal 
            this->exiting_ = true;

            if (!this->terminated_)
            {
                return true;
            }

            if (!this->exitOnlyOnce_)
            {
                return true;
            }

            // attempt to join with the thread once it has exited, or timeout 
            if (WaitFor(this->terminated_.get(), 15 * 1000))
            {
                return true;
            }

            // Do not force terminate if we're marked as dead and still running
            // The thread must've requested suicide and got stuck in a lengthy clean up effort
            if (!this->exitOnlyOnce_->TryLock())
            {
                AuLogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, but he should exiting now.");
                return false;
            }

            AuLogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, forcefully terminating without a watchdog overlooking onExit");

            // Kill the current OS thread instance
            TeminateOSContext(false);

            // Dispatch kill callback from within an emulated thread context (no switch, just just change thread tls)
            ExecuteInDeadThread([=]()
            {
                this->InternalKill(true);
            });

            this->exitOnlyOnce_->Unlock();
        }
        return true;
    }

    bool OSThread::Exiting()
    {
        return this->exiting_;
    }

    void OSThread::SetName(const AuString &name)
    {
        if (gRuntimeConfig.threadingConfig.bNoThreadNames)
        {
            return;
        }
        this->name_ = name;
        this->UpdateName();
    }

    EThreadThrottle OSThread::GetThrottle()
    {
        return this->throttle_;
    }

    EThreadPriority OSThread::GetPriority()
    {
        return this->prio_;
    }

    AuHwInfo::CpuBitId OSThread::GetMask()
    {
        return this->mask_;
    }

    AuString OSThread::GetName()
    {
        return this->name_;
    }

    void OSThread::SetAffinity(const HWInfo::CpuBitId &mask)
    {
        auto zero = HWInfo::CpuBitId();
        
        if (mask == zero ||
            mask == zero.Not())
        {
            this->mask_ = HWInfo::GetCPUInfo().maskAllCores;
            this->userManagingAffinity_ = false;

            UpdatePrio(this->throttle_, this->prio_);
        }
        else
        {
            this->mask_ = mask;
            this->userManagingAffinity_ = true;
        }

        UpdateAffinity(this->mask_);
        AffinityPrioThrottleTickAmendECores();
    }

    void OSThread::SetPriority(EThreadPriority prio)
    {
        if (!EThreadPriorityIsValid(prio))
        {
            return;
        }

        UpdatePrio(this->throttle_, prio);
        AffinityPrioThrottleTickAmendECores();
    }

    void OSThread::SetThrottle(EThreadThrottle throttle)
    {
        if (!EThreadThrottleIsValid(throttle))
        {
            return;
        }

        this->throttle_ = throttle;
        UpdatePrio(this->throttle_, this->prio_);
        AffinityPrioThrottleTickAmendECores();
    }

    AuSPtr<TLSView> OSThread::GetTlsView()
    {
        SysPanic("Deprecated Concept");
    }

    bool OSThread::ExecuteNewOSContext(AuFunction<void()> task)
    {
        this->task_ = task;

        this->handle_ = 0;

        auto ret = SpawnThread([this]()
        {
            while ((!this->handle_) ||
                #if defined(INVALID_HANDLE_VALUE)
                   (this->handle_ == INVALID_HANDLE_VALUE)
                #else
                   0
                #endif
                   )
            {
                AuThreading::ContextYield();
            }

            if (!this->tls_)
            {
                this->tls_ = GetThreadKey();
            }

            this->_ThreadEP();
        }, GetName(), this->info_.stackSize);
        
        if (ret.first)
        {
            this->handle_ = (decltype(handle_))ret.second;
        }

        return ret.first;
    }

    void OSThread::AffinityPrioThrottleTickAmendECores()
    {
        auto zero = HWInfo::CpuBitId();
        auto &cpuInfo = AuHwInfo::GetCPUInfo();
        auto sysEcores = cpuInfo.maskECores;
        bool bIsMaskProbablyDefault = this->mask_ == zero || this->mask_ == zero.Not();
        bool bSystemHasNoECores = sysEcores == zero;
        EThreadPriority prio = this->prio_;

        if (this->throttle_ == EThreadThrottle::eEfficient &&
            bSystemHasNoECores)
        {
            if (bIsMaskProbablyDefault)
            {
                sysEcores = cpuInfo.maskAllCores;
            }

            /**
             * We have a lot of code that will specify an efficiency throttle and then
             * arbitrarily use prio levels to (hopefully) order work in the os scheduler.
             * Systems without ecores would see a higher prio work than normal tasks if
             * we do nothing about this. On systems without ecores, efficiency throttle 
             * will just lock the prio to low.
            */
            prio = EThreadPriority::ePrioLow;

            UpdatePrio(this->throttle_, prio);
            UpdateAffinity(sysEcores);
        }
    }

    void OSThread::_ThreadEP()
    {
        // Poke TLS reference thread entity
        // TODO: we need an internal OSThread *TryPokeTLSThread()
        auto osThread = static_cast<OSThread *>(GetThread());
        this->tlsReferenceThread_ = osThread;
        bool bFailing {};

        OSAttach();

    #if defined(AURORA_IS_POSIX_DERIVED)
        this->bLongJmpOnce = false;
        if (setjmp(env) != 0)
        {
            Exit(true);
            return;
        }
    #endif

        try
        {
            if (task_)
            {
                task_();
            }
        }

#if !defined(AURORA_IS_POSIX_DERIVED)
        catch (...)
        {
            SysPushErrorHAL("OS Thread Aborted");
        }
        Exit(true);
#else
    #if defined(AURORA_COMPILER_GCC)
        catch (abi::__forced_unwind&)
        {
            throw;
        }
    #endif
        catch (...)
        {
            bFailing = true;
            if (!Aurora::kIsDebugBuild)
            {
                SysPushErrorHAL("OS Thread Aborted");
            }

            // "Safer" update 
            this->HookOnExit();


            if (this->terminated_)
            {
                if (this->exitOnlyOnce_)
                {
                    this->exitOnlyOnce_->Unlock();
                }
                this->terminated_->Set();
            } 

            if (this->terminatedSignalLs_)
            {
                this->terminatedSignalLs_->Set();
            }
        }

        if (!bFailing)
        {
            Exit(true);
        }
#endif
    }

    void OSThread::ExecuteInDeadThread(AuFunction<void()> callback)
    {
        auto old = HandleCurrent();

        if (this == old) [[unlikely]]
        {
            callback();
            return;
        }

        if (old)
        {
            static_cast<OSThread *>(old)->OSDeatach();
        }
        
        auto uOldHandle = GetThreadKey();
        this->OSAttach();
        
        callback();

        if (old)
        {
            HandleRegister(old);
            static_cast<OSThread *>(old)->OSDeatach();
        }
        else [[unlikely]]
        {
            HandleRemove();
        }

        SetThreadKey(uOldHandle);
    }

    void OSThread::UpdateName()
    {
        if (gRuntimeConfig.threadingConfig.bNoThreadNames)
        {
            return;
        }

    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if ((AuBuild::kCurrentPlatform == AuBuild::EPlatform::ePlatformWin32) && (!AuSwInfo::IsWindows10OrGreater()))
        {
            static const DWORD kMSVCExceptionSetName = 0x406D1388;

        #pragma pack(push,8)
            typedef struct tagTHREADNAME_INFO
            {
                DWORD dwType;
                LPCSTR szName;
                DWORD dwThreadID;
                DWORD dwFlags;
            } THREADNAME_INFO;
        #pragma pack(pop)

            if (!IsDebuggerPresent())
            {
                return;
            }

            THREADNAME_INFO info;
            info.dwType = 0x1000;
            info.szName = this->name_.c_str();
            if (pGetThreadId)
            {
                info.dwThreadID = pGetThreadId(this->handle_);
            }
            else if (this->unixThreadId_ == GetCurrentThreadId())
            {
                info.dwThreadID = this->unixThreadId_;
            }
            else
            {
                return;
            }

            info.dwFlags = 0;

            auto raise = AuStaticCast<void(__cdecl *)(THREADNAME_INFO &)>([](THREADNAME_INFO &info)
            {
                __try
                {
                    RaiseException(kMSVCExceptionSetName, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR *)&info);
                }
                __except (EXCEPTION_EXECUTE_HANDLER)
                {
                }
            });

            raise(info);
        }
        else
        {
            if (pSetThreadDescription)
            {
                pSetThreadDescription(this->handle_, AuLocale::ConvertFromUTF8(this->name_).c_str());
            }
        }

    #elif defined(AURORA_HAS_PTHREADS)

        if (!this->handle_)
        {
            return;
        }
        
        pthread_setname_np(this->handle_, this->name_.c_str());
    
    #endif
    }

    static void AttachSignalKiller()
    {
    #if defined(AURORA_IS_POSIX_DERIVED)
        struct sigaction action =
        {
            .sa_handler = HandleSigAbortThread,
        };
        ::sigemptyset(&action.sa_mask);
        ::sigaction(kSignalAbort, &action, nullptr);
    #endif
    }

    void OSThread::OSAttach()
    {
        this->bSupportsAltKill = true;

        HandleRegister(this);
    #if defined(AURORA_IS_LINUX_DERIVED)
        this->unixThreadId_ = gettid();
    #elif defined(AURORA_IS_XNU_DERIVED)
        this->unixThreadId_ = pthread_getthreadid_np();
    #elif defined(AURORA_IS_BSD_DERIVED)
        #if __FreeBSD_version < 900031
            long lwpid;
            thr_self(&lwpid);
            this->unixThreadId_ = lwpid;
        #elif __FreeBSD_version > 900030
            this->unixThreadId_ = pthread_getthreadid_np();
        #else
            static_assert(false);
        #endif
    #elif defined(AURORA_HAS_PTHREADS)
        this->unixThreadId_ = 0; // !!!!
    #endif

    #if defined(AURORA_PLATFORM_WIN32)
        this->unixThreadId_ = GetCurrentThreadId();
    #endif

        if (this->tls_)
        {
            SetThreadKey(this->tls_);
        }
        UpdatePrio(this->throttle_, this->prio_);
        SetAffinity(this->mask_);
        AffinityPrioThrottleTickAmendECores();
        UpdateName();
        AttachSignalKiller();
    }

    static AuHashMap<EThreadPriority, int> kNiceMap 
    {
        {
            EThreadPriority::ePrioRT, -19
        },
        {
            EThreadPriority::ePrioAboveHigh, -19
        },
        {
            EThreadPriority::ePrioHigh, -11
        },
        {
            EThreadPriority::ePrioNormal, -2
        },
        {
            EThreadPriority::ePrioLow, 5
        },
        {
            EThreadPriority::ePrioLowest, 15
        }
    };

#if defined(AURORA_IS_MODERNNT_DERIVED)
    static const AuHashMap<EThreadPriority, int> kWin32Map
    {
        {
            EThreadPriority::ePrioRT, THREAD_PRIORITY_TIME_CRITICAL
        },
        {
            EThreadPriority::ePrioAboveHigh, THREAD_PRIORITY_HIGHEST
        },
        {
            EThreadPriority::ePrioHigh, THREAD_PRIORITY_ABOVE_NORMAL
        },
        {
            EThreadPriority::ePrioNormal, THREAD_PRIORITY_NORMAL
        },
        {
            EThreadPriority::ePrioLow, THREAD_PRIORITY_BELOW_NORMAL
        },
        {
            EThreadPriority::ePrioLowest, THREAD_PRIORITY_LOWEST
        }
    };
#endif


#if defined(AURORA_IS_XNU_DERIVED)
    static const AuHashMap<AuPair<EThreadThrottle, EThreadPriority>, AuPair<AuUInt32, int>> kXnuQoSLevel
    {
        // Rt
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 5)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0)
        },
        {
            AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
        },

        // Perf
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 8)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 3)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 0)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1)
        },
        {
            AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
        },

        // Efficient
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 10)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 1)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -1)
        },
        {
            AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -2)
        }
    };
#endif

    void OSThread::UpdatePrio(EThreadThrottle throttle, EThreadPriority prio)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        const int *val;
        if (!AuTryFind(kWin32Map, prio, val))
        {
            return;
        }

        if (!SetThreadPriority(this->handle_, *val))
        {
            SysPushErrorHAL("Couldn't update thread priority");
        }

        THREAD_POWER_THROTTLING_STATE2 throttlingState {};
        throttlingState.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;

        switch (throttle)
        {
        case EThreadThrottle::eNormal:
            throttlingState.ControlMask = 0;
            throttlingState.StateMask = 0;
            break;
        case EThreadThrottle::ePerformance:
            throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            throttlingState.StateMask = 0;
            break;
        case EThreadThrottle::eEfficient:
            throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            throttlingState.StateMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
            break;
        }

        if (pSetThreadInformation)
        {
            pSetThreadInformation(this->handle_,
                                  kThreadPowerThrottling,
                                  &throttlingState,
                                  sizeof(throttlingState));
        }
      
    //#elif defined(AURORA_IS_XNU_DERIVED)


    #elif defined(AURORA_HAS_PTHREADS)

        if (!this->handle_)
        {
            return;
        }

        if (prio == EThreadPriority::ePrioRT)
        {
            sched_param param {};
            param.sched_priority = sched_get_priority_min(SCHED_RR);

            if (pthread_setschedparam(this->handle_, SCHED_RR, &param) == 0)
            {
                return;
            }

            // fall through on error
        }
        else if (this->prio_ == EThreadPriority::ePrioRT)
        {
            int policyNonRT = 
            #if defined(AURORA_IS_XNU_DERIVED)
                SCHED_FIFO;
            #else
                SCHED_OTHER;
            #endif  

            sched_param param {};
            param.sched_priority = sched_get_priority_min(policyNonRT);
            pthread_setschedparam(this->handle_, policyNonRT, &param);
        }

        const int *val;
        if (!AuTryFind(kNiceMap, prio, val))
        {
            return;
        }

        if (auto tid = unixThreadId_)
        {
            // TODO: per thread beaviour is a linux bug
            setpriority(PRIO_PROCESS, tid, *val);
        }
    #endif

    #if defined(AURORA_IS_LINUX_DERIVED) || defined(AURORA_IS_BSD_DERIVED)
        switch (throttle)
        {
        case EThreadThrottle::eNormal:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskAllCores;
            break;
        case EThreadThrottle::ePerformance:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskPCores;
            break;
        case EThreadThrottle::eEfficient:
            this->throttleMask_ = AuHwInfo::GetCPUInfo().maskECores;
            break;
        }

        UpdateAffinity(this->mask_);
    #endif

        this->prio_ = prio;
    }

    void OSThread::UpdateAffinity(const HWInfo::CpuBitId &mask)
    {

    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if ((AuBuild::kCurrentPlatform != AuBuild::EPlatform::ePlatformWin32) ||
            (AuSwInfo::IsWindows10OrGreater()))
        {
            auto sets = mask.CpuBitCount() ? 
                        mask.ToCpuSets() :
                        AuHwInfo::GetCPUInfo().maskAllCores.ToCpuSets();

            if (pSetThreadSelectedCpuSets)
            {
                if (pSetThreadSelectedCpuSets(this->handle_, sets.data(), sets.size()))
                {
                    return;
                }
                
                SysPushErrorUnavailableError("SetThreadSelectedCpuSets is expected on modern NT (CoreOS?) excluding Windows; or Win10+");
            }
        }

    #if defined(AURORA_PLATFORM_WIN32)
        if (pSetThreadGroupAffinity)
        {
            GROUP_AFFINITY affinityGroup { 0 };
            mask.ToMsWin7GroupAffinity(&affinityGroup);
            if (pSetThreadGroupAffinity(this->handle_, &affinityGroup, nullptr))
            {
                return;
            }
        }

        SetThreadAffinityMask(this->handle_, mask.lower);
    #endif

        SysPushErrorUnavailableError("Couldn't set thread affinity");
        return;

    #elif defined(AURORA_HAS_PTHREADS)

        if (!this->handle_)        
        {
             return;
        }

        auto mask2 = mask.And(this->throttleMask_);

        if (mask2.CpuBitCount() == 0)
        {
            switch (this->throttle_)
            {
            case EThreadThrottle::eNormal:
                mask2 = AuHwInfo::GetCPUInfo().maskAllCores;
                break;
            case EThreadThrottle::ePerformance:
                mask2 = AuHwInfo::GetCPUInfo().maskPCores;
                break;
            case EThreadThrottle::eEfficient:
                mask2 = AuHwInfo::GetCPUInfo().maskECores;
                break;
            }
        }

        cpu_set_t cpuset;
        CPU_ZERO(&cpuset);
        
        AuUInt8 index {};
        while (mask2.CpuBitScanForward(index, index))
        {
            CPU_SET(index, &cpuset);
            index++;
        }

        if (!CPU_COUNT(&cpuset))
        {
            return;
        }

        if (pthread_setaffinity_np(this->handle_, sizeof(cpuset), &cpuset) != 0)
        {
            SysPushErrorHAL("Couldn't set affinity mask");
        }

        return;

    #endif
    }

    void OSThread::OSDeatach()
    {
    #if defined(AURORA_IS_POSIX_DERIVED)
        struct sigaction action =
        {
            .sa_handler = SIG_DFL
        };
        ::sigemptyset(&action.sa_mask);
        ::sigaction(kSignalAbort, &action, nullptr);
    #endif
    }

    bool OSThread::InternalKill(bool locked)
    {
        this->PrivateUserDataClear();

        if (this->terminated_)
        {
            if (!locked)
            {
                if (!this->exitOnlyOnce_)
                {
                    return true;
                }

                if (!this->exitOnlyOnce_->TryLock())
                {
                    return false;
                }
            }
        }

        HookOnExit();

        try
        {
            // dispatch kill callback
            if (this->info_.callbacks)
            {
                this->info_.callbacks->OnExit(this);
            }
        }
        catch (...)
        {
            Debug::PrintError();
            AuLogWarn("Couldn't deinitialize thread");
            AuLogWarn("The smart thing to do at this point would be to panic");
            AuLogWarn("...but we could continue");
            AuLogWarn("Carrying on despite the potential for data integrity loss and memory leaks");
            Telemetry::Mayday();
        }

        HookReleaseThreadResources();

        if (this->terminatedSignalLs_)
        {
            this->terminatedSignalLs_->Set();
        }

        if (this->terminated_)
        {
            this->exitOnlyOnce_->Unlock();
            this->terminated_->Set(); // must be set last, after which point we cannot use this!
        } 

        return true;
    }

    bool OSThread::InternalKillForceNtfy()
    {
        if (this->terminated_)
        {
            if (this->exitOnlyOnce_)
            {
                this->exitOnlyOnce_->Unlock();
            }
            this->terminated_->Set();
        }

        if (this->terminatedSignalLs_)
        {
            this->terminatedSignalLs_->Set();
        }

        // Great C++ ABI guys...
    #if defined(AURORA_HAS_PTHREADS)
        {
            if (gettid() == getpid())
            {
                AuProcess::Exit(0);
                return true;
            }
            ::longjmp(env, 1);
        }
        ::pthread_exit(nullptr);
    #endif

        return true;
    }

    void OSThread::TeminateOSContext(bool calledFromThis)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (this->handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if (calledFromThis)
        {
            ::ExitThread(0);
        }
        else
        {
            ::TerminateThread(this->handle_, 0);
        }

    #elif defined(AURORA_HAS_PTHREADS)

        if (calledFromThis)
        {
            ::longjmp(env, 1);
            ::pthread_exit(nullptr);
        }
        else
        {   
            // pthreads is fun. thats not how unix works...
            // pthread_kill(this->handle_, SIGKILL);
            // remember signal inheritance is a cluster fuck & the tree will be walked
            // this is giving me flashbacks to hacking in apcs into the kernel
            // gross...
            // let's let nptl handle it
            //pthread_cancel(this->handle_);
            // ...doesnt work with c++ bc catch handlers get in the way. FUCK
            
            if (this->bSupportsAltKill)
            {
                ::pthread_kill(this->handle_, kSignalAbort);
            }
            else
            {
                ::pthread_cancel(this->handle_);
            }
        }

    #else
    
        SysPanic("Not implemented");
    
    #endif
    }

    void OSThread::FreeOSContext()
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        AuWin32CloseHandle(this->handle_);
    #endif
    }

    AuSPtr<AuLoop::ILoopSource> OSThread::AsLoopSource()
    {
        return this->terminateSignalLs_;
    }
        
    AuSPtr<IWaitable> OSThread::GetShutdownSignalWaitable()
    {
        return this->terminateSignal_;
    }

    AuSPtr<AuLoop::ILoopSource> OSThread::GetShutdownSignalLS()
    {
        return this->terminateSignalLs_;
    }

    void InitThreading()
    {
    #if defined(AURORA_PLATFORM_WIN32)
        AuxUlibInitialize();
    #endif

        AttachSignalKiller();
    }
}