2022-11-17 07:46:07 +00:00
|
|
|
/***
|
|
|
|
Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.
|
|
|
|
|
|
|
|
File: AuConditionVariable.Win32.cpp
|
|
|
|
Date: 2021-6-12
|
|
|
|
Author: Reece
|
|
|
|
***/
|
|
|
|
#include <Source/RuntimeInternal.hpp>
|
|
|
|
#include "AuConditionVariable.Generic.hpp"
|
2023-03-15 00:35:29 +00:00
|
|
|
#include <Time/Time.hpp>
|
2023-03-21 03:18:09 +00:00
|
|
|
#include "SMTYield.hpp"
|
2022-11-17 07:46:07 +00:00
|
|
|
|
|
|
|
#if !defined(_AURUNTIME_GENERICCV)
|
|
|
|
|
2023-03-15 00:35:29 +00:00
|
|
|
#if !defined(NTSTATUS_TIMEOUT)
|
|
|
|
#define NTSTATUS_TIMEOUT 0x102
|
|
|
|
#endif
|
|
|
|
|
2022-11-17 07:46:07 +00:00
|
|
|
namespace Aurora::Threading::Primitives
|
|
|
|
{
|
|
|
|
ConditionVariableImpl::ConditionVariableImpl(const AuSPtr<IConditionMutex> &pMutex) :
|
2023-03-15 00:35:29 +00:00
|
|
|
mutex_(AuStaticCast<Win32ConditionMutex>(pMutex))
|
2022-11-17 07:46:07 +00:00
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if defined(AURORA_FORCE_SRW_LOCKS)
|
|
|
|
::InitializeConditionVariable(&this->winCond_);
|
|
|
|
#endif
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
AuSPtr<IConditionMutex> ConditionVariableImpl::GetMutex()
|
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
return this->mutex_;
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool ConditionVariableImpl::WaitForSignal(AuUInt32 uTimeout)
|
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if !defined(AURORA_FORCE_SRW_LOCKS)
|
2023-03-15 00:35:29 +00:00
|
|
|
return WaitForSignalNS(AuMSToNS<AuUInt64>(uTimeout));
|
2023-03-16 18:25:23 +00:00
|
|
|
#else
|
|
|
|
auto bOK = ::SleepConditionVariableSRW(&this->winCond_, reinterpret_cast<PSRWLOCK>(this->mutex_->GetOSHandle()), uTimeout ? uTimeout : INFINITE, 0);
|
|
|
|
|
|
|
|
if (!bOK)
|
|
|
|
{
|
|
|
|
SysAssert(GetLastError() == ERROR_TIMEOUT, "SleepConditionVariable failure");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
#endif
|
2023-03-15 00:35:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool ConditionVariableImpl::WaitForSignalNS(AuUInt64 qwTimeout)
|
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if !defined(AURORA_FORCE_SRW_LOCKS)
|
2023-03-15 00:35:29 +00:00
|
|
|
bool bRet { true };
|
|
|
|
auto pThatMutex = reinterpret_cast<NT4Mutex *>(&this->mutex_->lock_);
|
2022-11-17 07:46:07 +00:00
|
|
|
|
2023-03-15 00:35:29 +00:00
|
|
|
if (qwTimeout)
|
2022-11-17 07:46:07 +00:00
|
|
|
{
|
2023-06-11 16:52:50 +00:00
|
|
|
#if defined(AU_TRUST_NT_KERNEL_SCHED_TIMEOUT)
|
2023-03-15 00:35:29 +00:00
|
|
|
auto uEndTimeSteady = AuTime::SteadyClockNS() + qwTimeout;
|
2023-03-16 18:25:23 +00:00
|
|
|
#endif
|
2023-03-15 00:35:29 +00:00
|
|
|
auto uEndTimeWall = AuTime::CurrentClockNS() + qwTimeout;
|
|
|
|
auto uTargetTimeNt = AuTime::ConvertTimestampNs(uEndTimeWall);
|
2023-03-16 22:18:07 +00:00
|
|
|
bool bIOU {};
|
2023-03-15 00:35:29 +00:00
|
|
|
|
2023-03-15 08:43:46 +00:00
|
|
|
while (true)
|
2023-03-15 00:35:29 +00:00
|
|
|
{
|
2023-03-15 08:28:16 +00:00
|
|
|
LARGE_INTEGER word;
|
|
|
|
word.QuadPart = uTargetTimeNt;
|
2023-03-15 17:54:59 +00:00
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
if (bRet)
|
|
|
|
{
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
auto uNow = this->wlist;
|
|
|
|
auto waiting = uNow >> 2u;
|
2023-05-30 11:46:38 +00:00
|
|
|
auto uNext = ((waiting + 1) << 2u) | (!bool(waiting)) | (uNow & 1);
|
2023-03-15 08:28:16 +00:00
|
|
|
|
|
|
|
if (AuAtomicCompareExchange(&this->wlist, uNext, uNow) == uNow)
|
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2023-05-30 11:46:38 +00:00
|
|
|
|
2023-05-31 04:34:36 +00:00
|
|
|
this->mutex_->Unlock();
|
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
bRet = pNtWaitForKeyedEvent(gKeyedEventHandle, &this->wlist, 0, &word) != NTSTATUS_TIMEOUT;
|
2023-05-31 04:34:36 +00:00
|
|
|
|
|
|
|
this->mutex_->Lock();
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
2023-04-23 18:39:54 +00:00
|
|
|
else /* unblock NtReleaseKeyedEvent after an atomic this->wlist change <-> NtReleaseKeyedEvent race condition. */
|
|
|
|
{ /* this->wlist waiters should still be accounting for us, leading to a NtReleaseKeyedEvent block condition*/
|
2023-05-31 04:34:36 +00:00
|
|
|
this->mutex_->Unlock();
|
2023-03-15 08:28:16 +00:00
|
|
|
LARGE_INTEGER word;
|
|
|
|
word.QuadPart = 0;
|
2023-06-11 23:40:24 +00:00
|
|
|
bRet = pNtWaitForKeyedEvent(gKeyedEventHandle, &this->wlist, 0, nullptr) != NTSTATUS_TIMEOUT;
|
2023-05-31 04:34:36 +00:00
|
|
|
this->mutex_->Lock();
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 17:54:59 +00:00
|
|
|
#if !defined(AU_TRUST_NT_KERNEL_SCHED_TIMEOUT)
|
|
|
|
if (!bRet)
|
|
|
|
#else
|
2023-03-15 08:28:16 +00:00
|
|
|
if (!bRet && uEndTimeSteady <= AuTime::SteadyClockNS())
|
2023-03-15 17:54:59 +00:00
|
|
|
#endif
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
auto uNow = this->wlist;
|
|
|
|
auto uOld = (uNow >> 2u);
|
|
|
|
|
|
|
|
if (uOld == 0)
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
// broadcast has woken everyone up
|
2023-03-15 08:28:16 +00:00
|
|
|
|
2023-03-15 17:54:59 +00:00
|
|
|
if (bIOU || CheckOut(bRet)) // the cope acquire
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
// in which case we're good
|
|
|
|
return true;
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
2023-03-15 16:06:58 +00:00
|
|
|
else
|
|
|
|
{
|
2023-04-23 18:39:54 +00:00
|
|
|
// ...and now we might owe NtReleaseKeyedEvent a thread >:(
|
2023-03-15 16:06:58 +00:00
|
|
|
bRet = false;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-23 18:39:54 +00:00
|
|
|
// go for an atomic decrement while racing against ::Signal and ::Broadcast
|
2023-03-15 16:06:58 +00:00
|
|
|
auto waiting = uOld - 1u;
|
|
|
|
auto uNext = waiting << 2u;
|
|
|
|
|
|
|
|
if (AuAtomicCompareExchange(&this->wlist, uNext, uNow) == uNow)
|
|
|
|
{
|
|
|
|
// break if successful
|
2023-03-15 17:54:59 +00:00
|
|
|
return bIOU;
|
2023-03-15 16:06:58 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
// block again because we couldn't decrement the counter
|
|
|
|
// broadcast still thinks we're asleep
|
|
|
|
// ...and we still owe NtReleaseKeyedEvent 1 therad
|
|
|
|
continue;
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2023-03-15 17:54:59 +00:00
|
|
|
// we good?
|
|
|
|
if (bIOU || CheckOut(bRet))
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 17:54:59 +00:00
|
|
|
return true;
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-15 00:35:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2023-05-30 11:46:38 +00:00
|
|
|
bool bIOU {};
|
2023-03-15 08:28:16 +00:00
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
auto uNow = this->wlist;
|
|
|
|
auto waiting = uNow >> 2u;
|
2023-05-30 11:46:38 +00:00
|
|
|
auto uNext = ((waiting + 1) << 2u) | (!bool(waiting)) | (uNow & 1);
|
2023-03-15 08:28:16 +00:00
|
|
|
|
|
|
|
if (AuAtomicCompareExchange(&this->wlist, uNext, uNow) == uNow)
|
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2023-05-31 04:34:36 +00:00
|
|
|
|
|
|
|
this->mutex_->Unlock();
|
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
pNtWaitForKeyedEvent(gKeyedEventHandle, &this->wlist, 0, nullptr);
|
|
|
|
|
2023-05-31 04:34:36 +00:00
|
|
|
this->mutex_->Lock();
|
|
|
|
|
2023-03-15 17:54:59 +00:00
|
|
|
if (bIOU || CheckOut(bRet))
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
return bRet;
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-03-16 18:25:23 +00:00
|
|
|
#else
|
|
|
|
AuInt64 uEndTime = qwTimeout ? AuTime::SteadyClockNS() + qwTimeout : 0;
|
|
|
|
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
AuInt64 uSecondTimeout = INFINITE;
|
|
|
|
|
|
|
|
if (qwTimeout)
|
|
|
|
{
|
|
|
|
uSecondTimeout = uEndTime - AuTime::SteadyClockNS();
|
|
|
|
|
|
|
|
if (uSecondTimeout <= 0)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
uSecondTimeout = AuNSToMS<AuUInt64>(uSecondTimeout);
|
|
|
|
if (!uSecondTimeout)
|
|
|
|
{
|
|
|
|
SMPPause();
|
|
|
|
AuThreading::ContextYield();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
auto bOk = ::SleepConditionVariableSRW(&this->winCond_, reinterpret_cast<PSRWLOCK>(mutex_->GetOSHandle()), uSecondTimeout, 0);
|
|
|
|
|
|
|
|
if (bOk)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
SysAssert(GetLastError() == ERROR_TIMEOUT, "SleepConditionVariable failure");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
bool ConditionVariableImpl::CheckOut(bool &bRet)
|
2022-11-17 07:46:07 +00:00
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if defined(AURORA_FORCE_SRW_LOCKS)
|
|
|
|
return false;
|
|
|
|
#else
|
2023-03-15 16:06:58 +00:00
|
|
|
return DoTryIf([&]()
|
2023-03-15 08:43:46 +00:00
|
|
|
{
|
|
|
|
auto uSignalNow = this->signalCount;
|
2023-03-15 00:35:29 +00:00
|
|
|
|
2023-03-15 16:06:58 +00:00
|
|
|
if (uSignalNow == 0)
|
2023-03-15 08:28:16 +00:00
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
return false;
|
|
|
|
}
|
2023-03-15 08:28:16 +00:00
|
|
|
|
2023-05-30 11:46:38 +00:00
|
|
|
auto uSignalNext = uSignalNow - 1;
|
|
|
|
|
|
|
|
if (AuAtomicCompareExchange(&this->signalCount, uSignalNext, uSignalNow) != uSignalNow)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (uSignalNext == 0)
|
|
|
|
{
|
|
|
|
InterlockedOr((volatile LONG*)&this->wlist, 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
2023-03-15 16:06:58 +00:00
|
|
|
});
|
2023-03-16 18:25:23 +00:00
|
|
|
#endif
|
2023-03-15 08:28:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void ConditionVariableImpl::Signal()
|
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if !defined(AURORA_FORCE_SRW_LOCKS)
|
2023-03-15 08:28:16 +00:00
|
|
|
auto original = this->wlist;
|
|
|
|
auto expected = original;
|
|
|
|
expected = expected >> 2;
|
2023-03-15 00:35:29 +00:00
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
if (expected)
|
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
AuAtomicAdd(&this->signalCount, 1u);
|
|
|
|
|
2023-03-15 00:35:29 +00:00
|
|
|
while (expected)
|
|
|
|
{
|
2023-03-15 16:06:58 +00:00
|
|
|
if (AuAtomicCompareExchange(&this->wlist, ((expected - 1) << 2) /*intentional clear*/, original) == original)
|
2023-03-15 00:35:29 +00:00
|
|
|
{
|
|
|
|
pNtReleaseKeyedEvent(gKeyedEventHandle, &this->wlist, FALSE, nullptr);
|
2023-03-15 08:28:16 +00:00
|
|
|
return;
|
2023-03-15 00:35:29 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 08:28:16 +00:00
|
|
|
original = this->wlist;
|
|
|
|
expected = original >> 2;
|
2023-03-15 00:35:29 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-16 18:25:23 +00:00
|
|
|
#else
|
|
|
|
::WakeConditionVariable(&this->winCond_);
|
|
|
|
#endif
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void ConditionVariableImpl::Broadcast()
|
|
|
|
{
|
2023-03-16 18:25:23 +00:00
|
|
|
#if !defined(AURORA_FORCE_SRW_LOCKS)
|
2023-03-15 08:28:16 +00:00
|
|
|
auto original = this->wlist;
|
|
|
|
auto expected = original;
|
|
|
|
expected = expected >> 2;
|
2023-03-15 00:35:29 +00:00
|
|
|
|
2023-03-15 16:06:58 +00:00
|
|
|
auto uBroadcastIterations = expected;
|
|
|
|
|
|
|
|
while (expected && uBroadcastIterations)
|
2023-03-15 00:35:29 +00:00
|
|
|
{
|
2023-05-30 11:46:38 +00:00
|
|
|
AuAtomicAdd(&this->signalCount, 1u);
|
|
|
|
|
|
|
|
while (expected && uBroadcastIterations)
|
2023-03-15 00:35:29 +00:00
|
|
|
{
|
2023-05-31 04:21:05 +00:00
|
|
|
bool bBreak {};
|
2023-05-30 11:46:38 +00:00
|
|
|
if (AuAtomicCompareExchange(&this->wlist, ((expected - 1) << 2) /*intentional clear*/, original) == original)
|
|
|
|
{
|
|
|
|
pNtReleaseKeyedEvent(gKeyedEventHandle, &this->wlist, FALSE, nullptr);
|
2023-03-15 00:35:29 +00:00
|
|
|
|
2023-05-30 11:46:38 +00:00
|
|
|
uBroadcastIterations--;
|
2023-05-31 04:21:05 +00:00
|
|
|
bBreak = true;
|
2023-05-30 11:46:38 +00:00
|
|
|
}
|
2023-03-15 16:06:58 +00:00
|
|
|
|
2023-05-30 11:46:38 +00:00
|
|
|
original = this->wlist;
|
|
|
|
expected = original >> 2;
|
2023-05-31 04:21:05 +00:00
|
|
|
|
|
|
|
if (bBreak)
|
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
2023-03-15 00:35:29 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-16 18:25:23 +00:00
|
|
|
#else
|
|
|
|
::WakeAllConditionVariable(&this->winCond_);
|
|
|
|
#endif
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
AUKN_SYM IConditionVariable *ConditionVariableNew(const AuSPtr<IConditionMutex> &pMutex)
|
|
|
|
{
|
|
|
|
return _new ConditionVariableImpl(pMutex);
|
|
|
|
}
|
|
|
|
|
|
|
|
AUKN_SYM void ConditionVariableRelease(IConditionVariable *pCV)
|
|
|
|
{
|
|
|
|
AuSafeDelete<ConditionVariableImpl *>(pCV);
|
|
|
|
}
|
2023-03-21 03:18:09 +00:00
|
|
|
|
|
|
|
AUROXTL_INTERFACE_SOO_SRC(ConditionVariable, ConditionVariableImpl, (const AuSPtr<IConditionMutex> &, pMutex))
|
2022-11-17 07:46:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|