/*** Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: AuOSThread.cpp Date: 2021-6-12 Author: Reece ***/ #include #include "AuThreads.hpp" #include #include "AuOSThread.hpp" #include "AuThreadHandles.hpp" #include "TLSView.hpp" #if defined(AURORA_IS_LINUX_DERIVED) #include #include #endif #if defined(AURORA_HAS_PTHREADS) #include #endif #if defined(AURORA_PLATFORM_WIN32) #include #endif #if defined(AURORA_IS_MODERNNT_DERIVED) static auto const kThreadPowerThrottling = Aurora::THREAD_INFORMATION_CLASS::ThreadPowerThrottling; struct THREAD_POWER_THROTTLING_STATE2 { ULONG Version; ULONG ControlMask; ULONG StateMask; }; #endif #if defined(AURORA_COMPILER_CLANG) // warning: enumeration values 'kEnumCount' and 'kEnumInvalid' not handled in switch [-Wswitch #pragma clang diagnostic ignored "-Wswitch" // Yea, I don't give a shit. #endif #include "AuSpawnThread.hpp" namespace Aurora { void RuntimeLateClean(); } namespace Aurora::Threading::Threads { static struct DtorFlipper { DtorFlipper() : bActive(true) { } ~DtorFlipper() { bActive = false; } bool bActive; } gGlobal; static ThreadInfo gDummyThreadInfo; #if defined(AURORA_IS_POSIX_DERIVED) static void HandleSigAbortThread(int a) { ((OSThread *)HandleCurrent())->InternalKillForceNtfy(); SysPanic("Couldn't terminate thread context"); } #endif OSThread::OSThread(const ThreadInfo &info) : info_(info), epExecEvent(false, false, true) { this->name_ = info.name.value_or("Aurora Thread"); // maybe we should atomic exchange compare these when needed frogthink this->terminatedSignalLs_ = AuLoop::NewLSEvent(true, false, true); this->terminateSignalLs_ = AuLoop::NewLSEvent(true, false, true); this->terminated_ = AuThreadPrimitives::EventShared(true, false, true); this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true); SysAssert(this->terminatedSignalLs_ && this->terminateSignalLs_ && this->terminated_ && this->terminateSignal_); this->InitThreadCreateTime(); } OSThread::OSThread() : info_(gDummyThreadInfo), epExecEvent(false, false, true) { this->name_ = "Main Thread"; this->terminated_ = AuThreadPrimitives::EventShared(true, false, true); this->terminateSignal_ = AuThreadPrimitives::EventShared(true, false, true); SysAssert(this->terminated_ && this->terminateSignal_); this->bNotOwned = true; this->InitThreadCreateTime(); } OSThread::OSThread(AuUInt64 id) : info_(gDummyThreadInfo), epExecEvent(false, false, true) { this->name_ = "System Thread"; this->handle_ = reinterpret_cast(id); this->bNotOwned = true; } bool DeadTest(); OSThread::~OSThread() { bool bDetached {}; bool bDetachedSuccess {}; if (this->bNotOwned) { #if defined(AURORA_IS_MODERNNT_DERIVED) auto hHandle = (HANDLE)this->handle_; AuWin32CloseHandle(hHandle); #endif return; } if (!gGlobal.bActive) { return; } #if 0 if (DeadTest()) { return; } #endif if (this->contextUsed_) { this->epExecEvent->Wait(); { AU_LOCK_GUARD(this->pFlag->mutex); this->pFlag->bLock = true; } if (this->detached_) { bDetached = true; } else { if (gRuntimeRunLevel <= 3) { Exit(); } else if (gRuntimeRunLevel >= 5) { // Application is dead } else { if (this->terminated_) { this->exiting_ = true; if (this->terminated_->LockMS(5'000)) { return; } } // Kill the current OS thread instance TeminateOSContext(false); this->exitOnlyOnce_->Unlock(); #if defined(AURORA_IS_POSIX_DERIVED) if (this->terminated_) { this->terminated_->Lock(); } #endif } } } else { if (auto pTerminated = AuExchange(this->terminatedSignalLs_, {})) { pTerminated->Set(); } if (auto pTerminated = AuExchange(this->terminated_, {})) { pTerminated->Set(); } } FreeOSContext(); if (bDetached) { if (this->tlsReferenceThread_) { AU_LOCK_GUARD(this->tlsLock_); AU_LOCK_GUARD(this->tlsReferenceThread_->tlsLock_); SetThreadKey(this->tlsReferenceThread_->tls_); AuExchange(this->tlsReferenceThread_->tls_, this->tls_); AuExchange(this->tlsReferenceThread_->threadFeatures_, this->threadFeatures_); bDetachedSuccess = true; } } if (bDetachedSuccess || !bDetached) { HookReleaseThreadResources(); } } // Called whenever an aurora thread is exiting void OSThread::HookOnExit() { } // Called whenever a thread object is released, or on thread termination // Whichever is soonest // It is possible a user may wish to keep a thread handle open // Is is expected for a system thread to release on dtor unlike our aurora threads void OSThread::HookReleaseThreadResources() { AU_LOCK_GUARD(this->tlsLock_); for (const auto &feature : this->threadFeatures_) { feature->Cleanup(); } } AuUInt64 OSThread::SetNoUnwindTerminateExitWatchDogTimeoutInMS(AuUInt64 uMS) { if (!uMS) { uMS = AuSToMS(15); } qwExitTimeout = uMS; return qwExitTimeout; } void OSThread::AddLastHopeTlsHook(const AuSPtr &feature) { AU_LOCK_GUARD(this->tlsLock_); if (!feature) { return; } AuTryInsert(this->threadFeatures_, feature); } void OSThread::Detach() { this->detached_ = true; } AuSPtr OSThread::AsWaitable() { if (!this->terminated_) { this->terminated_ = AuThreadPrimitives::EventShared(true, false, true); } return this->terminated_; } void OSThread::SendExitSignal() { this->exiting_ = true; if (this->terminateSignalLs_) { this->terminateSignalLs_->Set(); } if (this->terminateSignal_) { this->terminateSignal_->Set(); } } void OSThread::UnsafeForceTerminateSignal() { TeminateOSContext(false); } bool OSThread::Run() { if (!this->terminated_) { SysPanic("::Run called on system thread"); } if (AuExchange(this->contextUsed_, true)) { return false; } this->terminated_->Reset(); return ExecuteNewOSContext([=]() { try { // this functional backends are being deprecated if (this->info_.callbacks) { this->info_.callbacks->OnEntry(this); } } catch (...) { Debug::PrintError(); } }); } void OSThread::Exit() { this->PrivateUserDataClear(); while ((!this->terminated_) || (!this->terminated_->TryLock())) { if (Exit(false)) break; } } bool OSThread::Exit(bool willReturnToOS, bool isEOL) { if (GetThread() == this) { if (!this->InternalKill(false)) { return false; } // release handle + sigterm if (!willReturnToOS) { TeminateOSContext(true); while (true) { ContextYield(); } } } else { // exit signal this->exiting_ = true; if (!this->terminated_) { return true; } // attempt to join with the thread once it has exited, or timeout if (this->terminated_->LockMS(this->qwExitTimeout)) { return true; } // Do not force terminate if we're marked as dead and still running // The thread must've requested suicide and got stuck in a lengthy clean up effort if (!this->exitOnlyOnce_->TryLock()) { AuLogWarn("Watchdog error - OS thread context didn't finish in {} MS, but he should exiting now.", this->qwExitTimeout); return false; } AuLogWarn("Watchdog error - OS thread context didn't finish in {} MS, forcefully terminating without a watchdog overlooking onExit", this->qwExitTimeout); // Kill the current OS thread instance TeminateOSContext(false); // Dispatch kill callback from within an emulated thread context (no switch, just just change thread tls) ExecuteInDeadThread([=]() { this->InternalKill(true); }); this->exitOnlyOnce_->Unlock(); } return true; } bool OSThread::Exiting() { return this->exiting_; } void OSThread::SetName(const AuString &name) { if (gRuntimeConfig.threadingConfig.bNoThreadNames) { return; } this->name_ = name; this->UpdateName(); } EThreadThrottle OSThread::GetThrottle() { return this->throttle_; } EThreadPriority OSThread::GetPriority() { return this->prio_; } AuHwInfo::CpuBitId OSThread::GetMask() { return this->mask_; } AuString OSThread::GetName() { return this->name_; } void OSThread::SetAffinity(const HWInfo::CpuBitId &mask) { auto zero = HWInfo::CpuBitId(); if (mask == zero || mask == zero.Not()) { this->mask_ = HWInfo::GetCPUInfo().maskAllCores; this->userManagingAffinity_ = false; UpdatePrio(this->throttle_, this->prio_); } else { this->mask_ = mask; this->userManagingAffinity_ = true; } UpdateAffinity(this->mask_); AffinityPrioThrottleTickAmendECores(); } void OSThread::SetPriority(EThreadPriority prio) { if (!EThreadPriorityIsValid(prio)) { return; } UpdatePrio(this->throttle_, prio); AffinityPrioThrottleTickAmendECores(); } void OSThread::SetThrottle(EThreadThrottle throttle) { if (!EThreadThrottleIsValid(throttle)) { return; } UpdatePrio(throttle, this->prio_); this->throttle_ = throttle; AffinityPrioThrottleTickAmendECores(); } bool OSThread::ExecuteNewOSContext(AuFunction task) { this->task_ = task; this->handle_ = 0; auto ret = SpawnThread([this]() { while ((!this->handle_) || #if defined(INVALID_HANDLE_VALUE) (this->handle_ == INVALID_HANDLE_VALUE) #else 0 #endif ) { AuThreading::ContextYield(); } if (!this->tls_) { this->tls_ = GetThreadKey(); } this->_ThreadEP(); }, GetName(), this->info_.stackSize); if (ret.first) { this->handle_ = (decltype(handle_))ret.second; } return ret.first; } void OSThread::AffinityPrioThrottleTickAmendECores() { auto zero = HWInfo::CpuBitId(); auto &cpuInfo = AuHwInfo::GetCPUInfo(); auto sysEcores = cpuInfo.maskECores; bool bIsMaskProbablyDefault = this->mask_ == zero || this->mask_ == zero.Not(); bool bSystemHasNoECores = sysEcores == zero; EThreadPriority prio = this->prio_; if (this->throttle_ == EThreadThrottle::eEfficient && bSystemHasNoECores) { if (bIsMaskProbablyDefault) { sysEcores = cpuInfo.maskAllCores; } /** * We have a lot of code that will specify an efficiency throttle and then * arbitrarily use prio levels to (hopefully) order work in the os scheduler. * Systems without ecores would see a higher prio work than normal tasks if * we do nothing about this. On systems without ecores, efficiency throttle * will just lock the prio to low. */ prio = EThreadPriority::ePrioLow; UpdatePrio(this->throttle_, prio); UpdateAffinity(sysEcores); } } void AttachSignalKiller(); void OSThread::InitThreadCreateTime() { uClockCreationTime[0] = AuTime::CurrentClockNS(); uClockCreationTime[1] = AuTime::SteadyClockNS(); uClockCreationTime[2] = AuTime::ProcessClockNS(); } AuUInt64 OSThread::GetThreadCreationTime(Time::EClock eClock) { if (AuTime::EClockIsValid(eClock)) { SysPushErrorArg(); return {}; } switch (eClock) { case Time::EClock::eWall: return uClockCreationTime[0]; case Time::EClock::eSteady: return uClockCreationTime[1]; case Time::EClock::eProcessTime: return uClockCreationTime[2]; default: SysPushErrorArg(); return {}; } } void OSThread::_ThreadEP() { // Poke TLS reference thread entity // TODO: we need an internal OSThread *TryPokeTLSThread() auto osThread = static_cast(GetThread()); this->tlsReferenceThread_ = osThread; bool bFailing {}; OSAttach(); #if defined(AURORA_IS_POSIX_DERIVED) this->bLongJmpOnce = false; if (setjmp(env) != 0) { Exit(true); return; } this->bSupportsAltKill = true; #endif auto pFlag = this->pFlag; auto pA = this->terminatedSignalLs_; auto pB = this->terminated_; this->InitThreadCreateTime(); try { if (auto task = task_) { this->epExecEvent->Set(); task(); } } #if !defined(AURORA_IS_POSIX_DERIVED) catch (...) { SysPushErrorHAL("OS Thread Aborted"); } AU_LOCK_GUARD(pFlag->mutex); if (pFlag->bLock) { if (pA) { pA->Set(); } if (pB) { pB->Set(); } return; } this->SignalDeath(); Exit(true); #else #if defined(AURORA_COMPILER_GCC) catch (abi::__forced_unwind&) { throw; } #endif catch (...) { AU_LOCK_GUARD(pFlag->mutex); if (pFlag->bLock) { return; } bFailing = true; if (!Aurora::kIsDebugBuild) { SysPushErrorHAL("OS Thread Aborted"); } // "Safer" update this->HookOnExit(); this->SignalDeath(); } AU_LOCK_GUARD(pFlag->mutex); if (pFlag->bLock) { if (pA) { pA->Set(); } if (pB) { pB->Set(); } return; } if (!bFailing) { Exit(true); } #endif } void OSThread::ExecuteInDeadThread(AuFunction callback) { auto old = HandleCurrent(); if (this == old) [[unlikely]] { callback(); return; } if (old) { static_cast(old)->OSDeatach(); } auto uOldHandle = GetThreadKey(); this->OSAttach(); callback(); if (old) { HandleRegister(old); static_cast(old)->OSDeatach(); } else [[unlikely]] { HandleRemove(); } SetThreadKey(uOldHandle); } void OSThread::UpdateName() { if (gRuntimeConfig.threadingConfig.bNoThreadNames) { return; } #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if ((AuBuild::kCurrentPlatform == AuBuild::EPlatform::ePlatformWin32) && (!AuSwInfo::IsWindows10OrGreater())) { static const DWORD kMSVCExceptionSetName = 0x406D1388; #pragma pack(push,8) typedef struct tagTHREADNAME_INFO { DWORD dwType; LPCSTR szName; DWORD dwThreadID; DWORD dwFlags; } THREADNAME_INFO; #pragma pack(pop) if (!IsDebuggerPresent()) { return; } THREADNAME_INFO info; info.dwType = 0x1000; info.szName = this->name_.c_str(); if (pGetThreadId) { info.dwThreadID = pGetThreadId(this->handle_); } else if (this->unixThreadId_ == GetCurrentThreadId()) { info.dwThreadID = this->unixThreadId_; } else { return; } info.dwFlags = 0; auto raise = AuStaticCast([](THREADNAME_INFO &info) { __try { RaiseException(kMSVCExceptionSetName, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR *)&info); } __except (EXCEPTION_EXECUTE_HANDLER) { } }); raise(info); } else { if (pSetThreadDescription) { pSetThreadDescription(this->handle_, AuLocale::ConvertFromUTF8(this->name_).c_str()); } } #elif defined(AURORA_HAS_PTHREADS) if (!this->handle_) { return; } pthread_setname_np(this->handle_, this->name_.c_str()); #endif } static void AttachSignalKiller() { #if defined(AURORA_IS_POSIX_DERIVED) struct sigaction action = { .sa_handler = HandleSigAbortThread, .sa_flags = 0 }; ::sigemptyset(&action.sa_mask); ::sigaction(SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate, &action, nullptr); #endif } void OSThread::MakeMain() { this->bNotOwned = true; this->name_ = "Main System Thread"; this->OSAttach(); } void OSThread::OSAttach() { HandleRegister(this); #if defined(AURORA_IS_LINUX_DERIVED) this->unixThreadId_ = gettid(); #elif defined(AURORA_IS_XNU_DERIVED) this->unixThreadId_ = pthread_getthreadid_np(); #elif defined(AURORA_IS_BSD_DERIVED) #if __FreeBSD_version < 900031 long lwpid; thr_self(&lwpid); this->unixThreadId_ = lwpid; #elif __FreeBSD_version > 900030 this->unixThreadId_ = pthread_getthreadid_np(); #else static_assert(false); #endif #elif defined(AURORA_HAS_PTHREADS) this->unixThreadId_ = 0; // !!!! #endif #if defined(AURORA_PLATFORM_WIN32) this->unixThreadId_ = GetCurrentThreadId(); #endif if (this->tls_) { SetThreadKey(this->tls_); } UpdatePrio(this->throttle_, this->prio_); SetAffinity(this->mask_); AffinityPrioThrottleTickAmendECores(); UpdateName(); } static AuHashMap kNiceMap { { EThreadPriority::ePrioRT, -19 }, { EThreadPriority::ePrioAboveHigh, -15 }, { EThreadPriority::ePrioHigh, -11 }, { EThreadPriority::ePrioNormal, -2 }, { EThreadPriority::ePrioLow, 5 }, { EThreadPriority::ePrioLowest, 15 } }; #if defined(AURORA_IS_MODERNNT_DERIVED) static const AuHashMap kWin32Map { { EThreadPriority::ePrioRT, THREAD_PRIORITY_TIME_CRITICAL }, { EThreadPriority::ePrioAboveHigh, THREAD_PRIORITY_HIGHEST }, { EThreadPriority::ePrioHigh, THREAD_PRIORITY_ABOVE_NORMAL }, { EThreadPriority::ePrioNormal, THREAD_PRIORITY_NORMAL }, { EThreadPriority::ePrioLow, THREAD_PRIORITY_BELOW_NORMAL }, { EThreadPriority::ePrioLowest, THREAD_PRIORITY_LOWEST } }; #endif #if defined(AURORA_IS_XNU_DERIVED) static const AuHashMap, AuPair> kXnuQoSLevel { // Rt { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 5) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) }, // Perf { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 8) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 3) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 0) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) }, // Efficient { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 10) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 1) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -1) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -2) } }; #endif void OSThread::UpdatePrio(EThreadThrottle throttle, EThreadPriority prio) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } const int *val; if (!AuTryFind(kWin32Map, prio, val)) { return; } if (!SetThreadPriority(this->handle_, *val)) { SysPushErrorHAL("Couldn't update thread priority"); } THREAD_POWER_THROTTLING_STATE2 throttlingState {}; throttlingState.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION; switch (throttle) { case EThreadThrottle::eNormal: throttlingState.ControlMask = 0; throttlingState.StateMask = 0; break; case EThreadThrottle::ePerformance: throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; throttlingState.StateMask = 0; break; case EThreadThrottle::eEfficient: throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; throttlingState.StateMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; break; } if (pSetThreadInformation) { pSetThreadInformation(this->handle_, kThreadPowerThrottling, &throttlingState, sizeof(throttlingState)); } //#elif defined(AURORA_IS_XNU_DERIVED) #elif defined(AURORA_HAS_PTHREADS) sched_param param {}; int policyNonRT {}; int schedA {}; int scheduler {}; #if defined(AURORA_IS_LINUX_DERIVED) rlimit rl; #endif #if defined(AURORA_IS_POSIX_DERIVED) static const auto GetPrioLevel = [](int sched, EThreadPriority prio) { const int *val; if (!AuTryFind(kNiceMap, prio, val)) { return (int)sched_get_priority_max(sched); } int min = sched_get_priority_min(sched); int max = sched_get_priority_max(sched); float f = ((1.0 - ((float(*val) + 19.0) / (19.0 + 20.0))) * float(max - min)) + float(min); return (int)f; }; #endif if (!this->handle_) { return; } if (prio == EThreadPriority::ePrioRT) { #if defined(AURORA_IS_POSIX_DERIVED) #if defined(SCHED_RR) param.sched_priority = sched_get_priority_max((schedA = SCHED_RR)); #else param.sched_priority = sched_get_priority_max((schedA = SCHED_FIFO)); #endif if (::sched_setscheduler(this->unixThreadId_, schedA, ¶m) == 0) { goto gtfo; } { #if defined(AURORA_IS_LINUX_DERIVED) if (getrlimit(RLIMIT_RTPRIO, &rl) == 0) { bool bWarnOnce {}; rl.rlim_max = param.sched_priority; if (rl.rlim_max != RLIM_INFINITY) { AuLogWarn("We will not use RT-Kit to keep your Linux machine alive through XDG_FREEEEEEDESKTOP_HOPES_AND_DREAMS"); AuLogWarn("Hint: try (1) : in bash, elevate using: '[doas/sudo] prlimit --rtprio=unlimited --pid=$$' before running the main executable"); AuLogWarn("Hint: try (2) : append '* hard rtprio unlimited' to /etc/security/limits.conf"); AuLogWarn("Hint: try (3) : check /etc/systemd/[system/user].conf"); bWarnOnce = true; rl.rlim_cur = rl.rlim_max; } else { rl.rlim_cur = rl.rlim_max; } param.sched_priority = rl.rlim_cur; #if defined(SCHED_RR) if (::sched_setscheduler(this->unixThreadId_, SCHED_RR, ¶m) == 0) { goto gtfo; } #endif if (::setrlimit(RLIMIT_RTPRIO, &rl) != 0 && !bWarnOnce) { AuLogWarn("We will not use RT-Kit to keep your Linux machine alive through XDG_FREEEEEEDESKTOP_HOPES_AND_DREAMS"); AuLogWarn("Hint: try (1) : in bash, elevate using: '[doas/sudo] prlimit --rtprio=unlimited --pid=$$' before running the main executable"); AuLogWarn("Hint: try (2) : append '* hard rtprio unlimited' to /etc/security/limits.conf"); AuLogWarn("Hint: try (3) : check /etc/systemd/[system/user].conf"); } #if defined(SCHED_RR) if (::sched_setscheduler(this->unixThreadId_, SCHED_RR, ¶m) == 0) { goto gtfo; } #endif param.sched_priority = sched_get_priority_max(SCHED_FIFO); if (::sched_setscheduler(this->unixThreadId_, SCHED_FIFO, ¶m) == 0) { goto gtfo; } } else #endif { param.sched_priority = sched_get_priority_max(SCHED_FIFO); if (::sched_setscheduler(this->unixThreadId_, SCHED_FIFO, ¶m) == 0) { goto gtfo; } } } #else #if defined(SCHED_RR) param.sched_priority = sched_get_priority_min(SCHED_RR); if (pthread_setschedparam(this->handle_, SCHED_RR, ¶m) == 0) { goto gtfo; } #endif param.sched_priority = GetPrioLevel(SCHED_FIFO, prio); if (pthread_setschedparam(this->handle_, SCHED_FIFO, ¶m) == 0) { goto gtfo; } #endif // fall through on error } #if defined(AURORA_IS_LINUX_DERIVED) else if (throttle == EThreadThrottle::eEfficient) { param.sched_priority = GetPrioLevel(SCHED_IDLE, prio); if (::sched_setscheduler(this->unixThreadId_, SCHED_IDLE, ¶m) == 0) { goto gtfo; } // fail return; } else if (this->throttle_ == EThreadThrottle::eEfficient || this->prio_ == EThreadPriority::ePrioRT) #else else if (this->prio_ == EThreadPriority::ePrioRT) #endif { #if defined(AURORA_IS_POSIX_DERIVED) policyNonRT = #if defined(AURORA_IS_XNU_DERIVED) SCHED_FIFO; #else SCHED_OTHER; #endif param.sched_priority = GetPrioLevel(policyNonRT, prio); if (::sched_setscheduler(this->unixThreadId_, policyNonRT, ¶m) == 0) { goto gtfo; } return; #endif } { scheduler = ::sched_getscheduler(this->unixThreadId_); param.sched_priority = GetPrioLevel(scheduler, prio); if (::sched_setscheduler(this->unixThreadId_, scheduler, ¶m) == 0) { goto gtfo; } } // TODO: We used to set the legacy unix niceness value here. AuLogWarn("Couldn't set affinity"); #endif gtfo: #if defined(AURORA_IS_LINUX_DERIVED) || defined(AURORA_IS_BSD_DERIVED) switch (throttle) { case EThreadThrottle::eNormal: this->throttleMask_ = AuHwInfo::GetCPUInfo().maskAllCores; break; case EThreadThrottle::ePerformance: this->throttleMask_ = AuHwInfo::GetCPUInfo().maskPCores; break; case EThreadThrottle::eEfficient: this->throttleMask_ = AuHwInfo::GetCPUInfo().maskECores; break; } UpdateAffinity(this->mask_); #endif this->prio_ = prio; } void OSThread::UpdateAffinity(const HWInfo::CpuBitId &mask) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if ((AuBuild::kCurrentPlatform != AuBuild::EPlatform::ePlatformWin32) || (AuSwInfo::IsWindows10OrGreater())) { auto sets = mask.CpuBitCount() ? mask.ToCpuSets() : AuHwInfo::GetCPUInfo().maskAllCores.ToCpuSets(); if (pSetThreadSelectedCpuSets) { if (pSetThreadSelectedCpuSets(this->handle_, sets.data(), sets.size())) { return; } SysPushErrorUnavailableError("SetThreadSelectedCpuSets is expected on modern NT (CoreOS?) excluding Windows; or Win10+"); } } #if defined(AURORA_PLATFORM_WIN32) if (pSetThreadGroupAffinity) { GROUP_AFFINITY affinityGroup { 0 }; mask.ToMsWin7GroupAffinity(&affinityGroup); if (pSetThreadGroupAffinity(this->handle_, &affinityGroup, nullptr)) { return; } } if (mask.CpuBitCount() <= 32) { DWORD_PTR uProcMask {}; DWORD_PTR uSysMask {}; AuUInt32 uProcmask { AuUInt32(mask.lower) }; if (::GetProcessAffinityMask(GetCurrentProcess(), &uProcMask, &uSysMask)) { uProcmask &= uProcMask; } if (::SetThreadAffinityMask(this->handle_, uProcmask)) { return; } } #endif SysPushErrorUnavailableError("Couldn't set thread affinity"); return; #elif defined(AURORA_HAS_PTHREADS) if (!this->handle_) { return; } auto mask2 = mask.And(this->throttleMask_); if (mask2.CpuBitCount() == 0) { switch (this->throttle_) { case EThreadThrottle::eNormal: mask2 = AuHwInfo::GetCPUInfo().maskAllCores; break; case EThreadThrottle::ePerformance: mask2 = AuHwInfo::GetCPUInfo().maskPCores; break; case EThreadThrottle::eEfficient: mask2 = AuHwInfo::GetCPUInfo().maskECores; break; } } cpu_set_t cpuset; CPU_ZERO(&cpuset); AuUInt8 index {}; while (mask2.CpuBitScanForward(index, index)) { CPU_SET(index, &cpuset); index++; } if (!CPU_COUNT(&cpuset)) { return; } if (pthread_setaffinity_np(this->handle_, sizeof(cpuset), &cpuset) != 0) { SysPushErrorHAL("Couldn't set affinity mask"); } return; #endif } void OSThread::OSDeatach() { #if defined(AURORA_IS_POSIX_DERIVED) this->bSupportsAltKill = false; //struct sigaction action = //{ // .sa_handler = SIG_DFL //}; //::sigemptyset(&action.sa_mask); //::sigaction(SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate, &action, nullptr); #endif } bool OSThread::InternalKill(bool locked) { this->PrivateUserDataClear(); if (this->terminated_) { if (!locked) { if (!this->exitOnlyOnce_->TryLock()) { return false; } } } HookOnExit(); try { // dispatch kill callback if (this->info_.callbacks) { this->info_.callbacks->OnExit(this); } } catch (...) { Debug::PrintError(); AuLogWarn("Couldn't deinitialize thread"); AuLogWarn("The smart thing to do at this point would be to panic"); AuLogWarn("...but we could continue"); AuLogWarn("Carrying on despite the potential for data integrity loss and memory leaks"); Telemetry::Mayday(); } HookReleaseThreadResources(); this->SignalDeath(); return true; } void OSThread::SignalDeath() { if (this->terminatedSignalLs_) { this->terminatedSignalLs_->Set(); } if (this->terminated_) { this->exitOnlyOnce_->Unlock(); this->terminated_->Set(); } } bool OSThread::InternalKillForceNtfy() { this->SignalDeath(); // Great C++ ABI guys... #if defined(AURORA_HAS_PTHREADS) { if (gettid() == getpid()) { AuProcess::Exit(0); return true; } ::longjmp(env, 1); } ::pthread_exit(nullptr); #endif return true; } void OSThread::TeminateOSContext(bool calledFromThis) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if (calledFromThis) { ::ExitThread(0); } else { ::TerminateThread(this->handle_, 0); } #elif defined(AURORA_HAS_PTHREADS) if (calledFromThis) { ::longjmp(env, 1); ::pthread_exit(nullptr); } else { // pthreads is fun. thats not how unix works... // pthread_kill(this->handle_, SIGKILL); // remember signal inheritance is a cluster fuck & the tree will be walked // this is giving me flashbacks to hacking in apcs into the kernel // gross... // let's let nptl handle it //pthread_cancel(this->handle_); // ...doesnt work with c++ bc catch handlers get in the way. FUCK if (this->bSupportsAltKill) { ::pthread_kill(this->handle_, SIGRTMIN + gRuntimeConfig.linuxConfig.uSignalTerminate); } else { ::pthread_cancel(this->handle_); } } #else SysPanic("Not implemented"); #endif } void OSThread::FreeOSContext() { #if defined(AURORA_IS_MODERNNT_DERIVED) AuWin32CloseHandle(this->handle_); #endif } AuSPtr OSThread::AsLoopSource() { if (!this->terminatedSignalLs_) { this->terminatedSignalLs_ = AuLoop::NewLSEvent(true, false, true); } return this->terminatedSignalLs_; } AuSPtr OSThread::GetShutdownSignalWaitable() { return this->terminateSignal_; } AuSPtr OSThread::GetShutdownSignalLS() { return this->terminateSignalLs_; } void InitThreading() { #if defined(AURORA_PLATFORM_WIN32) AuxUlibInitialize(); #endif AttachSignalKiller(); if (auto pThread = GetThread()) { AuStaticCast(pThread)->MakeMain(); } } }