/*** Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Loop.cpp Date: 2021-9-21 Author: Reece ***/ #include #include "Loop.hpp" #include "ILoopSourceEx.hpp" #include "LSIOHandle.hpp" #include "LSTimerNoKernelScheduler.hpp" #include namespace Aurora::IO::Loop { // Write guards the transition period just after ::select-like and during { {onTriggered(...); } { reset spurious } } // By read-locking, depending on the platform, you provide some insurance that WaitMultipleLoopSources didn't // temporarily lock the resource. Under POSIX, this RWLock is fully respected. Under the platform agnostic path, // it is only partially respected. Under Win32, it's not really respected, but these issues can be worked around, // by using the interop-ready loop sources; under win32s fast path, interop-ready loop sources can be atomically // AND locked via the native nt mechanism. AuRWRenterableLock gWaitForMultipleALLLock; bool WaitMultipleAndOldImpl(const AuList> &lsList, AuList> &signaled, AuUInt64 uFlags, AuOptional optTimeoutMS, bool bSpin, bool bHasTimeOut, bool bSleepForever, bool bZeroTick, AuUInt64 uTimeoutEnd, bool bAvoidKrn, AuUInt32 uBaseFlags); bool WaitMultipleAndNewImpl(const AuList> &lsList, AuList> &signaled, AuUInt64 uFlags, AuOptional optTimeoutMS, bool bSpin, bool bHasTimeOut, bool bSleepForever, bool bZeroTick, AuUInt64 uTimeoutEnd, bool bAlert, bool bBreakAPCs); #if !defined(AURORA_IS_MODERNNT_DERIVED) AUKN_SYM AuSPtr NewLSWin32Source(bool) { return {}; } #endif #if !defined(AURORA_IS_XNU_DERIVED) AUKN_SYM AuSPtr NewLSAppleSource() { return {}; } #endif AUKN_SYM AuSPtr NewLSFile(const AuSPtr &pFileTransaction) { if (!pFileTransaction) { SysPushErrorArg(); return {}; } return pFileTransaction->NewLoopSource(); } AUKN_SYM AuSPtr NewStdIn() { return AuConsole::StdInBufferLoopSource(); } AUKN_SYM AuSPtr NewLSAsync(AuAsync::WorkerPId_t workerPid) { if (!workerPid) { return AuAsync::GetAsyncApp()->WorkerToLoopSource(workerPid); } return workerPid.GetPool()->WorkerToLoopSource(workerPid); } #if defined(AURORA_IS_MODERNNT_DERIVED) || defined(AURORA_IS_POSIX_DERIVED) AuList> WaitMultipleOrObjects(const AuList> &objects, bool bAlert, bool bBreakAPCs, bool bZeroTick, AuUInt32 timeout, bool bAllowOthers, bool &bTooMany); bool WaitMultipleAndObjects(const AuList> &objects, bool bAlert, bool bBreakAPCs, bool bZeroTick, AuUInt64 qwTimeoutAbs, bool &bTooMany, bool &bTimeout); #endif AuList> WaitMultipleOrObjectsFallback(const AuList> &objects, AuUInt32 timeout, bool bAlert, bool bBreak, bool bZeroTick, bool bAllowOthers, bool &bTimeout); void ResetLoopSourceFalseAlarm(const AuSPtr &pLoopSource) { if (!pLoopSource) { return; } if (pLoopSource->GetType() == ELoopSource::eSourceWin32) { return; } if (auto pSemaphore = AuDynamicCast(pLoopSource)) { // TOOD:Hack: AU_LOCK_GUARD(pSemaphore->cs); if (pSemaphore->maxIterationsOrZero_) { AuAtomicSub(&pSemaphore->count_, 1u); } pSemaphore->UpdateTimeSteadyNs(pSemaphore->targetTime_ - pSemaphore->reschedStepNsOrZero_); pSemaphore->Set(); return; } if (auto pMutex = AuDynamicCast(pLoopSource)) { pMutex->Unlock(); return; } if (auto pEvent = AuDynamicCast(pLoopSource)) { pEvent->Set(); return; } if (auto pSemaphore = AuDynamicCast(pLoopSource)) { pSemaphore->AddOne(); return; } } AUKN_SYM bool WaitMultipleLoopSources(const AuList> &lsList, AuList> &signaled, bool bAny, AuOptionalEx optTimeoutMS) { return WaitMultipleLoopSourcesEx(lsList, signaled, bAny ? kWaitMultipleFlagAny : 0, optTimeoutMS); } AUKN_SYM bool WaitMultipleLoopSourcesEx(const AuList> &lsList, AuList> &signaled, AuUInt64 uFlags, AuOptional optTimeoutMS) { signaled.clear(); bool bAny { bool(uFlags & kWaitMultipleFlagAny) }; bool bSpin { !(uFlags & kWaitMultipleFlagNoSpin) }; bool bAvoidKrn { bool(uFlags & kWaitMultipleFlagAvoidKern) }; bool bZeroTick { optTimeoutMS && optTimeoutMS.Value() == 0 }; bool bHasTimeOut { optTimeoutMS && optTimeoutMS.Value() }; bool bBreakAPCs { bool(uFlags & kWaitMultipleFlagBreakAfterAPC) }; bool bNoAlert { bool(uFlags & kWaitMultipleFlagNoIOCallbacks) }; bool bAlert { !bNoAlert }; bool bSleepForever { !optTimeoutMS }; AuUInt32 uBaseFlags {}; if (lsList.empty()) { if (bNoAlert) { return true; } else { if (bZeroTick) { return IOYield(); } else if (bHasTimeOut) { return IOYieldFor(optTimeoutMS.Value()); } else { return IOYieldFor(0); } } } if (bNoAlert) { uBaseFlags |= kFlagLSTryNoIOAlerts; } if (lsList.size() == 1) { AuUInt8 uFlags {}; auto pSource = lsList[0]; if (!pSource) { signaled.push_back({}); return true; } if (!bSpin) { uFlags = kFlagLSTryNoSpin; } uFlags |= uBaseFlags; bool bStatus {}; if (bSleepForever) { bStatus = pSource->WaitOnExt(uFlags, 0); } else if (bHasTimeOut) { bStatus = pSource->WaitOnExt(uFlags, optTimeoutMS.value()); } else { bStatus = pSource->IsSignaledExt(uFlags); } if (bStatus) { signaled.push_back(pSource); } return bStatus; } AuUInt64 uTimeoutEnd = bHasTimeOut ? AuTime::SteadyClockNS() + AuMSToNS(optTimeoutMS.value()) : 0; AU_DEBUG_MEMCRUNCH; signaled.reserve(lsList.size()); if (!bAny) { // try to sleep somewhat atomically. // under windows, this isn't quite possible with in process primitives. we just have to hope for interop-ready + kernel fast path. // under linux, it's fine, because we can select or poll, and then { lock { ... } } if (gRuntimeConfig.ioConfig.bUseSelectWaitAllStrat) { bool bTooMany {}; bool bTimeout {}; bool bRet = WaitMultipleAndObjects(lsList, bAlert, bBreakAPCs, false, uTimeoutEnd, bTooMany, bTimeout); if (!bTooMany) { if (bRet) { // explicit everyone became alert via select-like and said OK to our vtable query signaled = lsList; return true; } else if (bTimeout) { // explicit timeout timestamp exceeded return false; } else { // Win32 message loop needs tending to or APC dispatch return true; } } } // try to sleep on one item in the list, and then atomically acquire the rest if that one succeeded. // it can be observed that a single item in this lsList dips to non-signaled momentarily. // the same may also happen with local (not native) win32 loop sources under WaitMultipleAndObjects. // (interop ready/win32/win32-fast path is safe) // (local or interop/linux/posix-fast path is safe) // (local/win32 technically breaks WaitForMultipleObject docs) // (any/win32/>64 handles generic path technically breaks WaitForMultipleObject docs) // (any/*/generic path technically breaks WaitForMultipleObject docs) // our implementation is good enough to work on real world code, but it technically can break some multi-mutex wait and multi-consumer event-happened queries code. if (gRuntimeConfig.ioConfig.bUseOldIOWaitAllAlg) { return WaitMultipleAndOldImpl(lsList, signaled, uFlags, optTimeoutMS, bSpin, bHasTimeOut, bSleepForever, bZeroTick, uTimeoutEnd, bAvoidKrn, uBaseFlags); } else { return WaitMultipleAndNewImpl(lsList, signaled, uFlags, optTimeoutMS, bSpin, bHasTimeOut, bSleepForever, bZeroTick, uTimeoutEnd, bAlert, bBreakAPCs); } } else { bool bTimedout {}; AuList> signalTemp; auto lsList2 = lsList; bool bAnyFound {}; auto DoTheThing = [&](bool bLastTick) { for (auto itr = lsList2.begin(); itr != lsList2.end(); ) { if (signalTemp.size() && uFlags & kWaitMultipleFlagBreakAfterOne) { break; } auto pSource = *itr; if (!pSource) { signalTemp.push_back({}); itr = lsList2.erase(itr); continue; } auto eType = pSource->GetType(); if (eType == ELoopSource::eSourceFastMutex || eType == ELoopSource::eSourceFastSemaphore || eType == ELoopSource::eSourceFastEvent) { bAnyFound = true; if (pSource->IsSignaledExt(uBaseFlags | kFlagLSTryNoSpin)) { signalTemp.push_back(pSource); itr = lsList2.erase(itr); } else { if (bLastTick && bZeroTick) { itr = lsList2.erase(itr); } else { itr++; } } } else { itr++; } } }; if (bSpin && gRuntimeConfig.threadingConfig.bPlatformIsSMPProcessorOptimized && !bZeroTick) { AuThreadPrimitives::DoTryIf([&]() { DoTheThing(false); if (!bAnyFound) { return true; } return bool(signalTemp.size()); }); if (bAnyFound) { DoTheThing(true); } } else { DoTheThing(true); } bool bAllowOthers = !(uFlags & kWaitMultipleFlagBreakAfterOne); if (lsList2.size() && (bAllowOthers || signalTemp.empty())) { bZeroTick |= bool(signalTemp.size()); AuUInt32 uTimeoutMS {}; bool bTooMany {}; do { if (uTimeoutEnd) { auto uStartTime = Time::SteadyClockNS(); if (uStartTime >= uTimeoutEnd) { bZeroTick = true; } uTimeoutMS = AuNSToMS(uTimeoutEnd - uStartTime); if (!uTimeoutMS) { bZeroTick = true; } } #if defined(AURORA_IS_MODERNNT_DERIVED) if (AuBuild::kCurrentVendor == AuBuild::EVendor::eGenericMicrosoft && lsList2.size() < MAXIMUM_WAIT_OBJECTS) { signaled = WaitMultipleOrObjects(lsList2, bAlert, bBreakAPCs, bZeroTick, uTimeoutMS, bAllowOthers, bTooMany); bTimedout = uTimeoutEnd && uTimeoutMS && !bZeroTick ? Time::SteadyClockNS() >= uTimeoutEnd : false; } else #elif defined(AURORA_IS_POSIX_DERIVED) if (true) { signaled = WaitMultipleOrObjects(lsList2, bAlert, bBreakAPCs,bZeroTick, uTimeoutMS, bAllowOthers, bTooMany); bTimedout = uTimeoutEnd && uTimeoutMS && !bZeroTick ? Time::SteadyClockNS() >= uTimeoutEnd : false; } else #endif { bTooMany = true; } } while (!bTimedout && !bTooMany && !bZeroTick && signaled.empty() && (!bBreakAPCs || bNoAlert)); if (bTooMany) { signaled = WaitMultipleOrObjectsFallback(lsList2, uTimeoutMS, bAlert, bBreakAPCs, bZeroTick, bAllowOthers, bTimedout); bTimedout &= !bZeroTick; } } signaled.insert(signaled.end(), signalTemp.begin(), signalTemp.end()); if (bTimedout) { return false; } else if (bBreakAPCs && bAlert) { return true; } else { return signaled.size(); } } return false; } bool WaitMultipleAndNewImpl(const AuList> &lsList, AuList> &signaled, AuUInt64 uBaseFlags, AuOptional optTimeoutMS, bool bSpin, bool bHasTimeOut, bool bSleepForever, bool bZeroTick, AuUInt64 uTimeoutEnd, bool bAlert, bool bBreakAPCs) { AuUInt32 uTick {}; while (true) { AuUInt32 uChecked {}; bool bStatus {}; // Sleep on one (warn: this one primitive will appear to be non-signaled until we can write-acquire gWaitForMultipleALLLock) // (... : select, poll, etc based paths do not have this issue!) { bool bIsMutex {}; ELoopSource eType; AuUInt8 uFlags {}; if (!bSpin) { uFlags |= kFlagLSTryNoSpin; } if (!bAlert) { uFlags |= kFlagLSTryNoIOAlerts; } auto a = (uTick / lsList.size()) % 2; uChecked = uTick++ % lsList.size(); auto pCurrent = lsList[uChecked]; AuUInt32 uLocalCounter {}; while (!pCurrent) { uChecked = uTick++ % lsList.size(); pCurrent = lsList[pCurrent]; if ((uLocalCounter ++) > lsList.size()) { return false; } } eType = pCurrent->GetType(); bIsMutex = eType == ELoopSource::eSourceMutex || eType == ELoopSource::eSourceFastMutex; if (bIsMutex ^ bool(a)) { continue; } if (bHasTimeOut || bSleepForever) { bStatus = pCurrent->WaitOnAbsExt(uFlags, uTimeoutEnd); if (!bStatus) { if (uTimeoutEnd && AuNSToMS(uTimeoutEnd) <= AuTime::SteadyClockMS()) { return false; } else if (bAlert && bBreakAPCs) { return true; } } } else { bStatus = pCurrent->IsSignaledExt(uFlags | kFlagLSTryNoIOAlerts); if (!bStatus && bAlert) { if (IOYield()) { return true; } } } if (bIsMutex) { //uTick = 0; } } if (bStatus) { // Warning: this does not protect the first acquisition ! AU_LOCK_GLOBAL_GUARD(gWaitForMultipleALLLock->AsWritable()); AuUInt32 uBreakA {}; AuUInt32 uBreakIndex {}; { for (AU_ITERATE_N(a, 2)) { bool dBreak {}; for (AU_ITERATE_N(i, lsList.size())) { bool bIsMutex {}; ELoopSource eType; if (i == uChecked) { continue; } auto &pCurrent = lsList[i]; if (!pCurrent) { continue; } eType = pCurrent->GetType(); bIsMutex = eType == ELoopSource::eSourceMutex || eType == ELoopSource::eSourceFastMutex; if (bIsMutex ^ bool(a)) { continue; } if (!pCurrent->IsSignaledExt(kFlagLSTryNoSpin | kFlagLSTryNoIOAlerts)) { dBreak = true; uBreakIndex = i; break; } } if (dBreak) { uBreakA = a + 1; bStatus = false; break; } } } if (!bStatus) { for (AU_ITERATE_N(a, uBreakA)) { for (AU_ITERATE_N(i, uBreakIndex)) { auto &pCurrent = lsList[i]; bool bIsMutex {}; ELoopSource eType; if (!pCurrent) { continue; } if (i == uChecked) { continue; } eType = pCurrent->GetType(); bIsMutex = eType == ELoopSource::eSourceMutex || eType == ELoopSource::eSourceFastMutex; if (bIsMutex ^ bool(a)) { continue; } ResetLoopSourceFalseAlarm(pCurrent); } } if (auto pSource = lsList[uChecked]) { ResetLoopSourceFalseAlarm(pSource); } if (uTimeoutEnd && uTimeoutEnd <= AuTime::SteadyClockNS()) { return false; } } else { signaled = lsList; return true; } } } } bool WaitMultipleAndOldImpl(const AuList> &lsList, AuList> &signaled, AuUInt64 uFlags, AuOptional optTimeoutMS, bool bSpin, bool bHasTimeOut, bool bSleepForever, bool bZeroTick, AuUInt64 uTimeoutEnd, bool bAvoidKrn, AuUInt32 uBaseFlags) { AuList reverseList; { AuUInt32 uStartingOffset { 1 }; auto &entryZero = lsList[0]; if (!entryZero) { signaled.push_back({}); } if (entryZero) { bool bStatus {}; auto eType = entryZero->GetType(); AuUInt8 uFlags {}; if (!bSpin) { uFlags = kFlagLSTryNoSpin; } uFlags |= uBaseFlags; if (eType == ELoopSource::eSourceMutex || eType == ELoopSource::eSourceFastMutex) { bStatus = false; uStartingOffset = 0; goto mainAllSleep; } else if (bSleepForever) { bStatus = entryZero->WaitOnExt(uFlags, 0); } else if (bHasTimeOut) { bStatus = entryZero->WaitOnExt(uFlags, optTimeoutMS.value()); } else { bStatus = entryZero->IsSignaledExt(uFlags); } if (!bStatus) { goto next; } else { reverseList.push_back(0); signaled.push_back(entryZero); bZeroTick = true; } } mainAllSleep: if (lsList.size() > uStartingOffset && (!bAvoidKrn || signaled.empty())) { for (AU_ITERATE_N(a, 2)) { bool dBreak {}; for (AU_ITERATE_N_TO_X(i, uStartingOffset, lsList.size())) { AuUInt32 uTimeoutMS {}; bool bIsMutex {}; ELoopSource eType; auto &pCurrent = lsList[i]; eType = pCurrent->GetType(); bIsMutex = eType == ELoopSource::eSourceMutex || eType == ELoopSource::eSourceFastMutex; if (bIsMutex ^ bool(a)) { continue; } if (uTimeoutEnd && !bZeroTick) { auto uStartTime = Time::SteadyClockNS(); if (uStartTime >= uTimeoutEnd) { bZeroTick = true; } uTimeoutMS = AuNSToMS(uTimeoutEnd - uStartTime); if (!uTimeoutMS) { bZeroTick = true; } } if (bZeroTick) { if (bSpin) { if (!pCurrent->IsSignaledExt(uBaseFlags)) { dBreak = true; break; } bSpin = false; } else { if (!pCurrent->IsSignaledExt(uBaseFlags | kFlagLSTryNoSpin)) { dBreak = true; break; } } } else { if (bSpin) { if (!pCurrent->WaitOnAbsExt(uBaseFlags, uTimeoutEnd)) { dBreak = true; break; } // TBD bSpin = false; } else { if (!pCurrent->WaitOnAbsExt(kFlagLSTryNoSpin | uBaseFlags, uTimeoutEnd)) { dBreak = true; break; } } } reverseList.push_back(i); signaled.push_back(pCurrent); } if (dBreak || signaled.size() != lsList.size()) { break; } } } next: bool bReturnStatus { true }; if (signaled.size() != lsList.size()) { bReturnStatus = false; signaled.clear(); for (const auto &uIndex : reverseList) { ResetLoopSourceFalseAlarm(lsList[uIndex]); } } return bReturnStatus; } } AuList> WaitMultipleOrObjectsFallback(const AuList> &objects, AuUInt32 timeout, bool bAlert, bool bBreak, bool bZeroTick, bool bAllowOthers, bool &bTimeout) { AuList> loopSourceExs; AuList> triggered; if (objects.empty()) { return {}; } bTimeout = false; #if defined(AURORA_IS_MODERNNT_DERIVED) // Optimization / can be ignored / based impl details of NTs LoopQueue { AuUInt32 uBaseFlags { 0 }; if (!bAlert) { uBaseFlags |= kFlagLSTryNoIOAlerts; } // Check past the first loopqueues chug batch for (AU_ITERATE_N_TO_X(i, MAXIMUM_WAIT_OBJECTS - 1, objects.size())) { if (objects[i]->IsSignaledExt(uBaseFlags)) { triggered.push_back(objects[i]); } } // If we got something if (triggered.size()) { // Check everyone for (AU_ITERATE_N_TO_X(i, 0, MAXIMUM_WAIT_OBJECTS - 1)) { if (objects[i]->IsSignaledExt(uBaseFlags)) { triggered.push_back(objects[i]); } } // and then return return triggered; } // otherwise, we just need to gross chug poll. } #endif auto pQueue = AuLoop::NewLoopQueue(); if (!pQueue) { return {}; } pQueue->ConfigureDoIOApcCallbacks(bAlert); pQueue->ConfigureBreakAnyAfterAPC(bBreak); try { loopSourceExs.reserve(objects.size()); triggered.reserve(triggered.size()); for (const auto &source : objects) { if (!source) { continue; } if (!pQueue->SourceAdd(source)) { return {}; } if (source->GetType() == ELoopSource::eSourceWin32) { continue; } if (auto pLoopSourceEx = AuDynamicCast(source)) { if (!AuTryInsert(loopSourceExs, pLoopSourceEx)) { return {}; } } } } catch (...) { return {}; } auto pListener = AuMakeSharedThrow([&](const AuSPtr &source) { triggered.push_back(source); return false; }); if (!pQueue->AddCallback(pListener)) { return {}; } for (const auto &source : loopSourceExs) { source->OnPresleep(); } if (bZeroTick) { (void)pQueue->PumpNonblocking(); } else { bTimeout = !pQueue->WaitAny(timeout); } for (AU_ITERATE_N(i, loopSourceExs.size())) { auto pLoopSource = loopSourceExs[i]; if (bAllowOthers || triggered.empty()) { if (std::find(triggered.begin(), triggered.end(), pLoopSource) == triggered.end()) { auto eType = pLoopSource->GetType(); if (eType == ELoopSource::eSourceFastMutex || eType == ELoopSource::eSourceFastSemaphore || eType == ELoopSource::eSourceFastEvent) { if (pLoopSource->IsSignaledNoSpinIfUserland()) { triggered.push_back(pLoopSource); } } } } pLoopSource->OnFinishSleep(); } if (!bAllowOthers && triggered.size() > 1) { for (AU_ITERATE_N_TO_X(i, 1, triggered.size())) { ResetLoopSourceFalseAlarm(triggered[i]); } return { triggered[0] }; } else { return triggered; } } AUKN_SYM AuInt64 DbgLoopSourceToReadFd(AuSPtr pLoopSource) { if (!pLoopSource) { return -1; } auto pSourceEx = AuDynamicCast(pLoopSource); if (!pSourceEx) { return -1; } if (!pSourceEx->Singular()) { return -1; } return pSourceEx->GetHandle(); } AUKN_SYM AuInt64 DbgLoopSourceToWriteFd(AuSPtr pLoopSource) { if (!pLoopSource) { return -1; } if (auto pIOHandle = AuDynamicCast(pLoopSource)) { return pIOHandle->pHandle->GetOSWriteHandleSafe().OrElse([&]() { return pIOHandle->pHandle->GetOSReadHandleSafe(); }).ValueOr(AuUInt(-1)); } auto pSourceEx = AuDynamicCast(pLoopSource); if (!pSourceEx) { return -1; } if (!pSourceEx->Singular()) { return -1; } #if defined(AURORA_IS_POSIX_DERIVED) return pSourceEx->GetWriteHandle(); #else return pSourceEx->GetHandle(); #endif } }