/*** Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: OSThread.cpp Date: 2021-6-12 Author: Reece ***/ #include #include "Threads.hpp" #include "OSThread.hpp" #include "../WaitFor.hpp" #include "ThreadHandles.hpp" #include "TLSView.hpp" #if defined(AURORA_IS_LINUX_DERIVED) #include #endif #if defined(AURORA_HAS_PTHREADS) #include #endif #if defined(AURORA_PLATFORM_WIN32) #include #endif #include "SpawnThread.hpp" namespace Aurora::Threading::Threads { static ThreadInfo gDummyThreadInfo; OSThread::OSThread(const ThreadInfo &info) : info_(info) { this->name_ = info.name.value_or("Aurora Thread"); this->terminated_ = Primitives::EventShared(true, false); // maybe we should atomic exchange compare these when needed frogthink this->terminateSignal_ = Primitives::EventShared(true, false, true); this->terminatedSignalLs_ = Loop::NewLSEvent(true, false); this->terminateSignalLs_ = Loop::NewLSEvent(true, false, true); this->exitOnlyOnce_ = Primitives::CriticalSectionUnique(); SysAssert(this->terminated_ ? true : false, "out of memory"); } OSThread::OSThread() : info_(gDummyThreadInfo) { this->name_ = "Main Thread"; this->terminated_ = Primitives::EventShared(true, false); this->exitOnlyOnce_ = Primitives::CriticalSectionUnique(); } OSThread::OSThread(AuUInt64 id) : info_(gDummyThreadInfo) { this->name_ = "System Thread"; this->handle_ = reinterpret_cast(id); } OSThread::~OSThread() { bool bDetached {}; bool bDetachedSuccess {}; if (this->contextUsed_) { if (this->detached_) { bDetached = true; } else { Exit(); WaitFor(this->terminated_.get()); } } this->terminated_.reset(); FreeOSContext(); if (bDetached) { if (this->tlsReferenceThread_) { AU_LOCK_GUARD(this->tlsLock_); AU_LOCK_GUARD(this->tlsReferenceThread_->tlsLock_); AuExchange(this->tlsReferenceThread_->tls_, this->tls_); AuExchange(this->tlsReferenceThread_->threadFeatures_, this->threadFeatures_); bDetachedSuccess = true; } } if (bDetachedSuccess || !bDetached) { HookReleaseThreadResources(); } } // Called whenever an aurora thread is exiting void OSThread::HookOnExit() { } // Called whenever a thread object is released, or on thread termination // Whichever is soonest // It is possible a user may wish to keep a thread handle open // Is is expected for a system thread to release on dtor unlike our aurora threads void OSThread::HookReleaseThreadResources() { tls_.reset(); for (const auto &feature : this->threadFeatures_) { feature->Cleanup(); } } void OSThread::AddLastHopeTlsHook(const AuSPtr &feature) { // TODO: mutex if (!feature) { return; } AuTryInsert(this->threadFeatures_, feature); } void OSThread::Detach() { this->detached_ = true; } AuSPtr OSThread::AsWaitable() { return this->terminated_; } void OSThread::SendExitSignal() { this->exiting_ = true; if (this->terminateSignalLs_) { this->terminateSignalLs_->Set(); } if (this->terminateSignal_) { this->terminateSignal_->Set(); } } bool OSThread::Run() { if (!this->terminated_) { SysPanic("::Run called on system thread"); } if (AuExchange(this->contextUsed_, true)) { return false; } this->terminated_->Reset(); return ExecuteNewOSContext([=]() { try { // this functional backends are being deprecated if (this->info_.callbacks) { this->info_.callbacks->OnEntry(this); } } catch (...) { Debug::PrintError(); } }); } void OSThread::Exit() { while ((!this->terminated_) || (!this->terminated_->TryLock())) { if (Exit(false)) break; } } bool OSThread::Exit(bool willReturnToOS) { if (GetThread() == this) { if (!this->InternalKill(false)) { return false; } // release handle + sigterm if (!willReturnToOS) { TeminateOSContext(true); while (true) { YieldToOtherThread(); } } } else { // exit signal this->exiting_ = true; if (!this->terminated_) { return true; } // attempt to join with the thread once it has exited, or timeout if (WaitFor(this->terminated_.get(), 15 * 1000)) { return true; } // Do not force terminate if we're marked as dead and still running // The thread must've requested suicide and got stuck in a lengthy clean up effort if (!this->exitOnlyOnce_->TryLock()) { AuLogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, but he should exiting now."); return false; } AuLogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, forcefully terminating without a watchdog overlooking onExit"); // Kill the current OS thread instance TeminateOSContext(false); // Dispatch kill callback from within an emulated thread context (no switch, just just change thread tls) ExecuteInDeadThread([=]() { this->InternalKill(true); }); this->exitOnlyOnce_->Unlock(); } return true; } bool OSThread::Exiting() { return this->exiting_; } void OSThread::SetName(const AuString &name) { this->name_ = name; } EThreadThrottle OSThread::GetThrottle() { return this->throttle_; } EThreadPriority OSThread::GetPriority() { return this->prio_; } AuHwInfo::CpuBitId OSThread::GetMask() { return this->mask_; } AuString OSThread::GetName() { return this->name_; } void OSThread::SetAffinity(const HWInfo::CpuBitId &mask) { auto zero = HWInfo::CpuBitId(); if (mask == zero || mask == zero.Not()) { this->mask_ = HWInfo::GetCPUInfo().maskAllCores; this->userManagingAffinity_ = false; UpdatePrio(this->throttle_, this->prio_); } else { this->mask_ = mask; this->userManagingAffinity_ = true; } UpdateAffinity(this->mask_); } void OSThread::SetPriority(EThreadPriority prio) { if (!EThreadPriorityIsValid(prio)) { return; } UpdatePrio(this->throttle_, prio); } void OSThread::SetThrottle(EThreadThrottle throttle) { if (!EThreadThrottleIsValid(throttle)) { return; } this->throttle_ = throttle; UpdatePrio(this->throttle_, this->prio_); } AuSPtr OSThread::GetTlsView() { if (!this->tls_) { this->tls_ = AuMakeShared(); } return this->tls_; } bool OSThread::ExecuteNewOSContext(AuFunction task) { this->task_ = task; auto ret = SpawnThread([this]() { this->_ThreadEP(); }, GetName(), this->info_.stackSize); if (ret.first) { this->handle_ = (decltype(handle_))ret.second; } return ret.first; } void OSThread::_ThreadEP() { // Poke TLS reference thread entity // TODO: we need an internal OSThread *TryPokeTLSThread() auto osThread = static_cast(GetThread()); this->tlsReferenceThread_ = osThread; OSAttach(); try { task_(); } catch (...) { SysPushErrorHAL("OS Thread Aborted"); } Exit(true); } void OSThread::ExecuteInDeadThread(AuFunction callback) { auto old = HandleCurrent(); if (this == old) [[unlikely]] { callback(); return; } if (old) { static_cast(old)->OSDeatach(); } this->OSAttach(); callback(); if (old) { HandleRegister(old); static_cast(old)->OSDeatach(); } else [[unlikely]] { HandleRemove(); } } void OSThread::UpdateName() { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if ((AuBuild::kCurrentPlatform == AuBuild::EPlatform::ePlatformWin32) && (!AuSwInfo::IsWindows10OrGreater())) { static const DWORD kMSVCExceptionSetName = 0x406D1388; #pragma pack(push,8) typedef struct tagTHREADNAME_INFO { DWORD dwType; LPCSTR szName; DWORD dwThreadID; DWORD dwFlags; } THREADNAME_INFO; #pragma pack(pop) if (!IsDebuggerPresent()) { return; } THREADNAME_INFO info; info.dwType = 0x1000; info.szName = this->name_.c_str(); info.dwThreadID = ::GetThreadId(this->handle_); info.dwFlags = 0; auto raise = AuStaticCast([](THREADNAME_INFO &info) { __try { RaiseException(kMSVCExceptionSetName, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR *)&info); } __except (EXCEPTION_EXECUTE_HANDLER) { } }); raise(info); } else { static HRESULT(WINAPI * SetThreadDescription_f)(HANDLE, PCWSTR); if (!SetThreadDescription_f) { SetThreadDescription_f = AuReinterpretCast(GetProcAddress(GetModuleHandleW(L"Kernel32.dll"), "SetThreadDescription")); } if (SetThreadDescription_f) { SetThreadDescription_f(this->handle_, AuLocale::ConvertFromUTF8(this->name_).c_str()); } } #elif defined(AURORA_HAS_PTHREADS) pthread_setname_np(this->handle_, this->name_.c_str()); #endif } void OSThread::OSAttach() { HandleRegister(this); #if defined(AURORA_IS_LINUX_DERIVED) this->unixThreadId_ = gettid(); #elif defined(AURORA_IS_XNU_DERIVED) this->unixThreadId_ = pthread_getthreadid_np(); #elif defined(AURORA_IS_BSD_DERIVED) #if __FreeBSD_version < 900031 long lwpid; thr_self(&lwpid); this->unixThreadId_ = lwpid; #elif __FreeBSD_version > 900030 this->unixThreadId_ = pthread_getthreadid_np(); #else static_assert(false); #endif #elif defined(AURORA_HAS_PTHREADS) this->unixThreadId_ = 0; // !!!! #endif UpdatePrio(this->throttle_, this->prio_); SetAffinity(this->mask_); UpdateName(); } static AuHashMap kNiceMap { { EThreadPriority::ePrioRT, -19 }, { EThreadPriority::ePrioAboveHigh, -19 }, { EThreadPriority::ePrioHigh, -11 }, { EThreadPriority::ePrioNormal, -2 }, { EThreadPriority::ePrioLow, 5 }, { EThreadPriority::ePrioLowest, 15 } }; #if defined(AURORA_IS_MODERNNT_DERIVED) static const AuHashMap kWin32Map { { EThreadPriority::ePrioRT, THREAD_PRIORITY_TIME_CRITICAL }, { EThreadPriority::ePrioAboveHigh, THREAD_PRIORITY_HIGHEST }, { EThreadPriority::ePrioHigh, THREAD_PRIORITY_ABOVE_NORMAL }, { EThreadPriority::ePrioNormal, THREAD_PRIORITY_NORMAL }, { EThreadPriority::ePrioLow, THREAD_PRIORITY_BELOW_NORMAL }, { EThreadPriority::ePrioLowest, THREAD_PRIORITY_LOWEST } }; #endif #if defined(AURORA_IS_XNU_DERIVED) static const AuHashMap, AuPair> kXnuQoSLevel { // Rt { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 5) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0) }, { AuMakePair(EThreadThrottle::eNormal, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) }, // Perf { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 10) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 8) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 3) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_HIGH, 0) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 1) }, { AuMakePair(EThreadThrottle::ePerformance, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0) }, // Efficient { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioRT), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 10) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioAboveHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 1) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioHigh), AuMakePair(DISPATCH_QUEUE_PRIORITY_LOW, 0) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioNormal), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLow), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -1) }, { AuMakePair(EThreadThrottle::eEfficient, EThreadPriority::ePrioLowest), AuMakePair(DISPATCH_QUEUE_PRIORITY_BACKGROUND, -2) } }; #endif void OSThread::UpdatePrio(EThreadThrottle throttle, EThreadPriority prio) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } const int *val; if (!AuTryFind(kWin32Map, prio, val)) { return; } if (!SetThreadPriority(this->handle_, *val)) { SysPushErrorHAL("Couldn't update thread priority"); } THREAD_POWER_THROTTLING_STATE throttlingState {}; throttlingState.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION; switch (throttle) { case EThreadThrottle::eNormal: throttlingState.ControlMask = 0; throttlingState.StateMask = 0; break; case EThreadThrottle::ePerformance: throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; throttlingState.StateMask = 0; break; case EThreadThrottle::eEfficient: throttlingState.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; throttlingState.StateMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; break; } SetThreadInformation(this->handle_, ThreadPowerThrottling, &throttlingState, sizeof(throttlingState)); //#elif defined(AURORA_IS_XNU_DERIVED) #elif defined(AURORA_HAS_PTHREADS) if (!this->handle_) { return; } if (prio == EThreadPriority::ePrioRT) { sched_param param {}; param.sched_priority = sched_get_priority_min(SCHED_RR); if (pthread_setschedparam(this->handle_, SCHED_RR, ¶m) == 0) { return; } // fall through on error } else if (this->prio_ == EThreadPriority::ePrioRT) { int policyNonRT = #if defined(AURORA_IS_XNU_DERIVED) SCHED_FIFO; #else SCHED_OTHER; #endif sched_param param {}; param.sched_priority = sched_get_priority_min(policyNonRT); pthread_setschedparam(this->handle_, policyNonRT, ¶m); } const int *val; if (!AuTryFind(kNiceMap, prio, val)) { return; } if (auto tid = unixThreadId_) { // TODO: per thread beaviour is a linux bug setpriority(PRIO_PROCESS, tid, *val); } #endif #if defined(AURORA_IS_LINUX_DERIVED) || defined(AURORA_IS_BSD_DERIVED) switch (throttle) { case EThreadThrottle::eNormal: break; case EThreadThrottle::ePerformance: this->throttleMask_ = AuHwInfo::GetCPUInfo().maskPCores; break; case EThreadThrottle::eEfficient: this->throttleMask_ = AuHwInfo::GetCPUInfo().maskECores; break; } #endif this->prio_ = prio; } void OSThread::UpdateAffinity(const HWInfo::CpuBitId &mask) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if ((AuBuild::kCurrentPlatform != AuBuild::EPlatform::ePlatformWin32) || (AuSwInfo::IsWindows10OrGreater())) { static BOOL(WINAPI * SetThreadSelectedCpuSets_f)(HANDLE, const ULONG *, ULONG); if (!SetThreadSelectedCpuSets_f) { SetThreadSelectedCpuSets_f = AuReinterpretCast(GetProcAddress(GetModuleHandleW(L"Kernel32.dll"), "SetThreadSelectedCpuSets")); } auto sets = mask.ToCpuSets(); if (SetThreadSelectedCpuSets_f) { if (SetThreadSelectedCpuSets_f(this->handle_, sets.data(), sets.size())) { return; } SysPushErrorUnavailableError("SetThreadSelectedCpuSets is expected on modern NT (CoreOS?) excluding Windows; or Win10+"); } } #if defined(AURORA_PLATFORM_WIN32) GROUP_AFFINITY affinityGroup {0}; mask.ToMsWin7GroupAffinity(&affinityGroup); if (SetThreadGroupAffinity(this->handle_, &affinityGroup, nullptr)) { return; } #endif SysPushErrorUnavailableError("Couldn't set thread affinity"); return; #elif defined(AURORA_HAS_PTHREADS) auto mask2 = mask.And(this->throttleMask_); if (mask2.CpuBitCount() == 0) { switch (this->throttle_) { case EThreadThrottle::eNormal: break; case EThreadThrottle::ePerformance: mask2 = AuHwInfo::GetCPUInfo().maskPCores; break; case EThreadThrottle::eEfficient: mask2 = AuHwInfo::GetCPUInfo().maskECores; break; } } cpu_set_t cpuset; CPU_ZERO(&cpuset); AuUInt8 index {}; while (mask2.CpuBitScanForward(index, index)) { CPU_SET(index, &cpuset); index++; } if (pthread_setaffinity_np(this->handle_, sizeof(cpuset), &cpuset) != 0) { SysPushErrorHAL("Couldn't set affinity mask"); } return; #endif } void OSThread::OSDeatach() { } bool OSThread::InternalKill(bool locked) { if (this->terminated_) { if (!locked) { if (!this->exitOnlyOnce_->TryLock()) { return false; } } } HookOnExit(); try { // dispatch kill callback if (this->info_.callbacks) { this->info_.callbacks->OnExit(this); } } catch (...) { Debug::PrintError(); AuLogWarn("Couldn't deinitialize thread"); AuLogWarn("The smart thing to do at this point would be to panic"); AuLogWarn("...but we could continue"); AuLogWarn("Carrying on despite the potential for data integrity loss and memory leaks"); Telemetry::Mayday(); } HookReleaseThreadResources(); if (this->terminated_) { this->exitOnlyOnce_->Unlock(); this->terminated_->Set(); } if (this->terminatedSignalLs_) { this->terminatedSignalLs_->Set(); } return true; } void OSThread::TeminateOSContext(bool calledFromThis) { #if defined(AURORA_IS_MODERNNT_DERIVED) if (this->handle_ == INVALID_HANDLE_VALUE) { return; } if (calledFromThis) { ExitThread(0); } else { TerminateThread(this->handle_, 0); } #elif defined(AURORA_HAS_PTHREADS) pthread_kill(this->handle_, SIGTERM); #else SysPanic("Not implemented"); #endif } void OSThread::FreeOSContext() { #if defined(AURORA_IS_MODERNNT_DERIVED) AuWin32CloseHandle(this->handle_); #endif } AuSPtr OSThread::AsLoopSource() { return this->terminateSignalLs_; } AuSPtr OSThread::GetShutdownSignalWaitable() { return this->terminateSignal_; } AuSPtr OSThread::GetShutdownSignalLS() { return this->terminateSignalLs_; } void InitThreading() { #if defined(AURORA_PLATFORM_WIN32) AuxUlibInitialize(); #endif } }