[*] [Pre-Win8.1] Optimize for modern nt instead of windows vista synch in legacy path; yes, this is how windows 7 and vista synch is somewhat implemented.

...on apis that predate those kernel revisions. so, technically this might be able to run on xp.
[*] GetThreadCookie optimization for all platforms
This commit is contained in:
Reece Wilson 2023-03-15 00:35:29 +00:00
parent 14c7d538e1
commit 046b70d7bc
18 changed files with 424 additions and 53 deletions

View File

@ -40,8 +40,17 @@ namespace Aurora
} \
}
if (pRtlGetVersion)
{
return;
}
ADD_GET_PROC(Nt, RtlGetVersion)
ADD_GET_PROC(Nt, NtDelayExecution)
ADD_GET_PROC(Nt, NtWaitForKeyedEvent)
ADD_GET_PROC(Nt, NtReleaseKeyedEvent)
ADD_GET_PROC(Nt, NtOpenKeyedEvent)
ADD_GET_PROC(Nt, NtCreateKeyedEvent)
ADD_GET_PROC_BI(Kernel32, KernelBase, VirtualAlloc2)
ADD_GET_PROC_BI(Kernel32, KernelBase, MapViewOfFile3)

View File

@ -64,6 +64,34 @@ namespace Aurora
ULONG UnmapFlags
);
inline NTSTATUS(__stdcall *pNtWaitForKeyedEvent)(
HANDLE Handle,
PVOID Key,
BOOLEAN Alertable,
PLARGE_INTEGER NTTimeout
);
inline NTSTATUS(__stdcall *pNtReleaseKeyedEvent)(
HANDLE Handle,
PVOID Key,
BOOLEAN Alertable,
PLARGE_INTEGER NTTimeout
);
inline NTSTATUS(__stdcall *pNtCreateKeyedEvent)(
HANDLE Handle,
ACCESS_MASK Access,
POBJECT_ATTRIBUTES Attr,
ULONG Flags
);
inline NTSTATUS(__stdcall *pNtOpenKeyedEvent)(
HANDLE Handle,
ACCESS_MASK Access,
POBJECT_ATTRIBUTES Attr,
ULONG Flags
);
#if defined(AURORA_PLATFORM_WIN32)
inline NTSTATUS(_stdcall *pRtlGetVersion)(
PRTL_OSVERSIONINFOW lpVersionInformation

View File

@ -7,14 +7,24 @@
***/
#include <Source/RuntimeInternal.hpp>
#include "AuConditionMutex.Generic.hpp"
#include "SMPYield.hpp"
#include "AuProcAddresses.NT.hpp"
#if !defined(_AURUNTIME_GENERICCM)
namespace Aurora::Threading::Primitives
{
Win32ConditionMutex::Win32ConditionMutex()
{
::InitializeSRWLock(&this->lock_);
if (gKeyedEventHandle == INVALID_HANDLE_VALUE)
{
if (!pNtCreateKeyedEvent)
{
InitNTAddresses();
}
pNtCreateKeyedEvent(&gKeyedEventHandle, -1, NULL, 0);
}
}
Win32ConditionMutex::~Win32ConditionMutex()
@ -23,17 +33,66 @@ namespace Aurora::Threading::Primitives
bool Win32ConditionMutex::TryLock()
{
return ::TryAcquireSRWLockExclusive(&this->lock_);
return DoTryIf([=]()
{
return !AuAtomicTestAndSet(&this->lock_.uWaitCount, 0);
});
}
void Win32ConditionMutex::Lock()
{
::AcquireSRWLockExclusive(&this->lock_);
while (!TryLock())
{
auto &uValueRef = this->lock_.uWaitCount;
auto uValue = uValueRef | 1;
if (AuAtomicCompareExchange(&uValueRef, uValue + FAST_M_WAIT, uValue) == uValue)
{
pNtWaitForKeyedEvent(gKeyedEventHandle, &uValueRef, 0, NULL);
AuAtomicSub(&uValueRef, FAST_M_WAIT);
}
}
}
void Win32ConditionMutex::Unlock()
{
::ReleaseSRWLockExclusive(&this->lock_);
auto &uValueRef = this->lock_.uWaitCount;
auto uValue = uValueRef;
if (uValue == 1)
{
if (AuAtomicCompareExchange(&uValueRef, 0u, 1u) == 1u)
{
return;
}
}
while (true)
{
if (uValue < FAST_M_WAIT)
{
return;
}
if (uValue & 1)
{
return;
}
if (uValue & FAST_M_WAKE)
{
return;
}
if (AuAtomicCompareExchange(&uValueRef, uValue - FAST_M_WAIT + FAST_M_WAKE, uValue) == uValue)
{
pNtReleaseKeyedEvent(gKeyedEventHandle, &uValueRef, 0, NULL);
return;
}
SMPPause();
}
}
AuUInt Win32ConditionMutex::GetOSHandle()

View File

@ -1,7 +1,7 @@
/***
Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.
File: AuConditionMutex.Win32.hpp
File: AuConditionMutex.NT.hpp
Date: 2021-6-12
Author: Reece
***/
@ -12,6 +12,16 @@
#if !defined(_AURUNTIME_GENERICCM)
namespace Aurora::Threading::Primitives
{
#define FAST_M_WAKE 256u
#define FAST_M_WAIT 512u
inline HANDLE gKeyedEventHandle { INVALID_HANDLE_VALUE };
struct NT4Mutex
{
AuUInt32 uWaitCount {}; // yields while bits are high, dec to release one from the semaphore yield
};
struct Win32ConditionMutex : IConditionMutexEx
{
Win32ConditionMutex();
@ -22,8 +32,7 @@ namespace Aurora::Threading::Primitives
void Unlock() override;
AuUInt GetOSHandle() override;
private:
SRWLOCK lock_;
NT4Mutex lock_;
};
using ConditionMutexImpl = Win32ConditionMutex;

View File

@ -7,15 +7,20 @@
***/
#include <Source/RuntimeInternal.hpp>
#include "AuConditionVariable.Generic.hpp"
#include <Time/Time.hpp>
#include "SMPYield.hpp"
#if !defined(_AURUNTIME_GENERICCV)
#if !defined(NTSTATUS_TIMEOUT)
#define NTSTATUS_TIMEOUT 0x102
#endif
namespace Aurora::Threading::Primitives
{
ConditionVariableImpl::ConditionVariableImpl(const AuSPtr<IConditionMutex> &pMutex) :
mutex_(AuDynamicCast<IConditionMutexEx>(pMutex))
mutex_(AuStaticCast<Win32ConditionMutex>(pMutex))
{
InitializeConditionVariable(&this->winCond_);
}
AuSPtr<IConditionMutex> ConditionVariableImpl::GetMutex()
@ -25,27 +30,113 @@ namespace Aurora::Threading::Primitives
bool ConditionVariableImpl::WaitForSignal(AuUInt32 uTimeout)
{
auto ok = SleepConditionVariableSRW(&this->winCond_,
reinterpret_cast<PSRWLOCK>(this->mutex_->GetOSHandle()),
uTimeout ? uTimeout : INFINITE, 0);
if (!ok)
{
SysAssert(GetLastError() == ERROR_TIMEOUT, "SleepConditionVariable failure");
return false;
return WaitForSignalNS(AuMSToNS<AuUInt64>(uTimeout));
}
return true;
bool ConditionVariableImpl::WaitForSignalNS(AuUInt64 qwTimeout)
{
bool bRet { true };
auto pThatMutex = reinterpret_cast<NT4Mutex *>(&this->mutex_->lock_);
this->mutex_->Unlock();
bool bStatus {};
if (qwTimeout)
{
auto uEndTimeSteady = AuTime::SteadyClockNS() + qwTimeout;
auto uEndTimeWall = AuTime::CurrentClockNS() + qwTimeout;
auto uTargetTimeNt = AuTime::ConvertTimestampNs(uEndTimeWall);
LARGE_INTEGER word;
word.QuadPart = uTargetTimeNt;
AuAtomicAdd(&this->wlist, 1u);
bRet = pNtWaitForKeyedEvent(gKeyedEventHandle, &this->wlist, 0, &word) != NTSTATUS_TIMEOUT;
if (!bRet)
{
AuAtomicAdd(&this->expander, 1u);
}
}
else
{
AuAtomicAdd(&this->wlist, 1u);
pNtWaitForKeyedEvent(gKeyedEventHandle, &this->wlist, 0, nullptr);
}
this->mutex_->Lock();
return bRet;
}
void ConditionVariableImpl::Signal()
{
WakeConditionVariable(&this->winCond_);
auto expected = this->wlist;
AuUInt32 uExpander { this->expander };
if (expected || uExpander)
{
AuUInt32 uOffset { 0 };
while (uExpander && AuAtomicCompareExchange(&this->expander, 0u, uExpander) != uExpander)
{
SMPPause();
uExpander = this->expander;
}
expected = this->wlist;
while (expected && AuAtomicCompareExchange(&this->wlist, expected - uExpander, expected) != expected)
{
SMPPause();
expected = this->wlist;
}
expected = this->wlist;
while (expected)
{
if (AuAtomicCompareExchange(&this->wlist, expected - 1, expected) == expected)
{
pNtReleaseKeyedEvent(gKeyedEventHandle, &this->wlist, FALSE, nullptr);
}
expected = this->wlist;
}
}
}
void ConditionVariableImpl::Broadcast()
{
WakeAllConditionVariable(&this->winCond_);
auto expected = this->wlist;
AuUInt32 uExpander { this->expander };
while (expected || uExpander)
{
AuUInt32 uOffset { 0 };
if (uExpander && AuAtomicCompareExchange(&this->expander, 0u, uExpander) != uExpander)
{
SMPPause();
uExpander = this->expander;
expected = this->wlist;
continue;
}
expected = this->wlist;
while (AuAtomicCompareExchange(&this->wlist, expected - uExpander, expected) != expected)
{
SMPPause();
expected = this->wlist;
}
expected = this->wlist;
while (expected)
{
if (AuAtomicCompareExchange(&this->wlist, expected - 1, expected) == expected)
{
pNtReleaseKeyedEvent(gKeyedEventHandle, &this->wlist, FALSE, nullptr);
}
expected = this->wlist;
}
}
}
AUKN_SYM IConditionVariable *ConditionVariableNew(const AuSPtr<IConditionMutex> &pMutex)

View File

@ -8,6 +8,8 @@
#pragma once
#if !defined(_AURUNTIME_GENERICCV)
#include "AuConditionMutex.NT.hpp"
namespace Aurora::Threading::Primitives
{
struct ConditionVariableImpl : IConditionVariable
@ -16,12 +18,15 @@ namespace Aurora::Threading::Primitives
AuSPtr<IConditionMutex> GetMutex() override;
bool WaitForSignal(AuUInt32 timeout) override;
bool WaitForSignalNS(AuUInt64 qwTimeout);
void Signal() override;
void Broadcast() override;
private:
CONDITION_VARIABLE winCond_;
AuSPtr<IConditionMutexEx> mutex_;
AuUInt32 wlist {};
AuUInt32 expander {};
Win32ConditionMutex selfLock_;
AuSPtr<Win32ConditionMutex> mutex_;
};
}
#endif

View File

@ -34,7 +34,12 @@ namespace Aurora::Threading::Primitives
return this->mutex_;
}
bool ConditionVariableImpl::WaitForSignal(AuUInt32 timeout)
bool ConditionVariableImpl::WaitForSignal(AuUInt32 uTimeout)
{
return WaitForSignalNS(AuMSToNS<AuUInt64>(uTimeout));
}
bool ConditionVariableImpl::WaitForSignalNS(AuUInt64 qwTimeout)
{
auto mutex = reinterpret_cast<pthread_mutex_t*>(this->mutex_->GetOSHandle());
@ -60,7 +65,7 @@ namespace Aurora::Threading::Primitives
else
{
struct timespec tspec;
Time::ms2tsabs(&tspec, timeout);
Time::auabsns2ts(&tspec, AuTime::CurrentClockNS() + qwTimeout);
int ret {};

View File

@ -17,6 +17,7 @@ namespace Aurora::Threading::Primitives
AuSPtr<IConditionMutex> GetMutex() override;
bool WaitForSignal(AuUInt32 timeout) override;
bool WaitForSignalNS(AuUInt64 timeout) override;
void Signal() override;
void Broadcast() override;

View File

@ -12,7 +12,7 @@ namespace Aurora::Threading::Primitives
{
CriticalSection::CriticalSection()
{
this->owner_ = nullptr;
this->owner_ = {};
this->count_ = 0;
}
@ -28,7 +28,7 @@ namespace Aurora::Threading::Primitives
bool CriticalSection::TryLock()
{
auto cur = Threads::GetThread();
auto cur = GetThreadCookie();
if (this->owner_ == cur)
{
@ -55,7 +55,7 @@ namespace Aurora::Threading::Primitives
bool CriticalSection::Lock(AuUInt64 timeout)
{
auto cur = Threads::GetThread();
auto cur = GetThreadCookie();
if (this->owner_ == cur)
{
@ -76,7 +76,7 @@ namespace Aurora::Threading::Primitives
bool CriticalSection::LockNS(AuUInt64 timeout)
{
auto cur = Threads::GetThread();
auto cur = GetThreadCookie();
if (this->owner_ == cur)
{
@ -99,7 +99,7 @@ namespace Aurora::Threading::Primitives
{
if (--this->count_ == 0)
{
this->owner_ = nullptr;
this->owner_ = {};
this->mutex_.Unlock();
}
}

View File

@ -8,6 +8,7 @@
#pragma once
#include "AuMutex.Generic.hpp"
#include "ThreadCookie.hpp"
namespace Aurora::Threading::Primitives
{
@ -26,7 +27,7 @@ namespace Aurora::Threading::Primitives
private:
Mutex mutex_;
Threads::IAuroraThread *owner_;
ThreadCookie_t owner_;
std::atomic<int> count_;
};
}

View File

@ -39,7 +39,7 @@ namespace Aurora::Threading::Primitives
while (!AtomicIsEventSetLogic())
{
AuUInt32 uTimeoutMs {};
AuUInt32 uTimeoutNS {};
if (uTimeout)
{
@ -49,18 +49,10 @@ namespace Aurora::Threading::Primitives
return false;
}
uTimeoutMs = AuNSToMS<AuUInt64>(uEndTime - uStartTime);
if (!uTimeoutMs)
{
this->mutex_.Unlock();
SMPPause();
AuThreading::ContextYield();
this->mutex_.Lock();
continue;
}
uTimeoutNS = uEndTime - uStartTime;
}
if (!this->condition_.WaitForSignal(uTimeoutMs))
if (!this->condition_.WaitForSignalNS(uTimeoutNS))
{
continue;
}

View File

@ -11,6 +11,12 @@
#if !defined(_AURUNTIME_GENERICMUTEX)
#include "AuMutex.NT.hpp"
#include "AuConditionMutex.NT.hpp"
#include <Time/Time.hpp>
#if !defined(NTSTATUS_TIMEOUT)
#define NTSTATUS_TIMEOUT 0x102
#endif
namespace Aurora::Threading::Primitives
{
@ -18,8 +24,10 @@ namespace Aurora::Threading::Primitives
{
if (!pWaitOnAddress)
{
#if 0
::InitializeSRWLock(&this->atomicHolder_);
::InitializeConditionVariable(&this->wakeup_);
#endif
}
this->state_ = 0;
}
@ -38,7 +46,7 @@ namespace Aurora::Threading::Primitives
{
return DoTryIf([=]()
{
return ::_interlockedbittestandset(&this->state_, 0) == 0;
return ::_interlockedbittestandset((volatile LONG *)&this->state_, 0) == 0;
});
}
@ -62,7 +70,7 @@ namespace Aurora::Threading::Primitives
{
bool returnValue = false;
if (this->TryLock())
if (TryLock())
{
return true;
}
@ -73,7 +81,7 @@ namespace Aurora::Threading::Primitives
if (pWaitOnAddress)
{
auto state = this->state_;
while (::_interlockedbittestandset(&this->state_, 0) != 0)
while (::_interlockedbittestandset((volatile LONG *)&this->state_, 0) != 0)
{
AuUInt32 uTimeoutMS = INFINITE;
@ -105,6 +113,7 @@ namespace Aurora::Threading::Primitives
}
else
{
#if 0
::AcquireSRWLockShared(&this->atomicHolder_);
BOOL status = false;
@ -140,6 +149,67 @@ namespace Aurora::Threading::Primitives
exitWin32:
::ReleaseSRWLockShared(&this->atomicHolder_);
#else
if (!uTimeout)
{
while (!TryLock())
{
auto &uValueRef = this->state_;
auto uValue = uValueRef | 1;
if (AuAtomicCompareExchange(&uValueRef, uValue + FAST_M_WAIT, uValue) == uValue)
{
pNtWaitForKeyedEvent(gKeyedEventHandle, (void *)&uValueRef, 0, NULL);
AuAtomicSub(&uValueRef, FAST_M_WAIT);
}
}
return true;
}
else
{
auto &uValueRef = this->state_;
returnValue = true;
auto uEndTimeSteady = AuTime::SteadyClockNS() + uTimeout;
auto uEndTimeWall = AuTime::CurrentClockNS() + uTimeout;
while (!TryLock())
{
auto uValue = uValueRef | 1;
if (AuTime::SteadyClockNS() >= uEndTimeSteady)
{
returnValue = TryLock();
break;
}
if (AuAtomicCompareExchange(&uValueRef, uValue + FAST_M_WAIT, uValue) == uValue)
{
auto uTargetTimeNt = AuTime::ConvertTimestampNs(uEndTimeWall);
LARGE_INTEGER word;
word.QuadPart = uTargetTimeNt;
auto uStatus = pNtWaitForKeyedEvent(gKeyedEventHandle, (void *)&this->state_, 0, &word);
AuAtomicSub(&uValueRef, FAST_M_WAIT);
if (uStatus == NTSTATUS_TIMEOUT)
{
// i suspect we're bailing out too fast
//returnValue = false;
}
else
{
SysAssertDbg(uStatus == 0);
}
}
}
}
#endif
return returnValue;
}
}
@ -148,10 +218,51 @@ namespace Aurora::Threading::Primitives
{
if (!pWaitOnAddress)
{
#if 0
::AcquireSRWLockExclusive(&this->atomicHolder_);
this->state_ = 0;
::ReleaseSRWLockExclusive(&this->atomicHolder_);
::WakeAllConditionVariable(&this->wakeup_);
#else
auto &uValueRef = this->state_;
auto uValue = uValueRef;
if (uValue == 1)
{
if (AuAtomicCompareExchange(&uValueRef, 0u, 1u) == 1u)
{
return;
}
}
while (true)
{
if (uValue < FAST_M_WAIT)
{
return;
}
if (uValue & 1)
{
return;
}
if (uValue & FAST_M_WAKE)
{
return;
}
if (AuAtomicCompareExchange(&uValueRef, uValue - FAST_M_WAIT + FAST_M_WAKE, uValue) == uValue)
{
pNtReleaseKeyedEvent(gKeyedEventHandle, (void *)&uValueRef, 0, NULL);
return;
}
SMPPause();
}
#endif
}
else
{

View File

@ -23,8 +23,10 @@ namespace Aurora::Threading::Primitives
void Unlock() override;
private:
#if 0
SRWLOCK atomicHolder_;
CONDITION_VARIABLE wakeup_;
volatile AuAtomicInt state_{};
#endif
volatile AuUInt32 state_{};
};
}

View File

@ -87,7 +87,7 @@ namespace Aurora::Threading::Primitives
bool RWLockImpl<bIsWriteRecursionAllowed>::LockReadNS(AuUInt64 uTimeout)
{
if (this->state_ < 0 &&
this->reentrantWriteLockHandle_ == AuThreads::GetThreadId())
this->reentrantWriteLockHandle_ == GetThreadCookie())
{
return true;
}
@ -158,7 +158,7 @@ namespace Aurora::Threading::Primitives
auto uOld = this->state_;
if (uOld < 0)
{
if (this->reentrantWriteLockHandle_ == AuThreads::GetThreadId())
if (this->reentrantWriteLockHandle_ == GetThreadCookie())
{
AuAtomicSub(&this->state_, 1);
return true;
@ -168,7 +168,7 @@ namespace Aurora::Threading::Primitives
{
if (AuAtomicCompareExchange(&this->state_, -1, uOld) == uOld)
{
this->reentrantWriteLockHandle_ = AuThreads::GetThreadId();
this->reentrantWriteLockHandle_ = GetThreadCookie();
return true;
}
}
@ -212,7 +212,7 @@ namespace Aurora::Threading::Primitives
if (AuAtomicCompareExchange(&this->state_, -1, 0) == 0)
{
this->reentrantWriteLockHandle_ = AuThreads::GetThreadId();
this->reentrantWriteLockHandle_ = GetThreadCookie();
this->writersPending_--;
return true;
}
@ -228,7 +228,7 @@ namespace Aurora::Threading::Primitives
if (iCurState == -1)
{
return this->reentrantWriteLockHandle_ == AuThreads::GetThreadId();
return this->reentrantWriteLockHandle_ == GetThreadCookie();
}
return AuAtomicCompareExchange(&this->state_, iCurState + 1, iCurState) == iCurState;
@ -250,7 +250,7 @@ namespace Aurora::Threading::Primitives
}
else
{
if (this->reentrantWriteLockHandle_ == AuThreads::GetThreadId())
if (this->reentrantWriteLockHandle_ == GetThreadCookie())
{
AuAtomicSub(&this->state_, 1);
return true;
@ -270,7 +270,7 @@ namespace Aurora::Threading::Primitives
if (AuAtomicCompareExchange(&this->state_, -1, curVal) == curVal)
{
this->reentrantWriteLockHandle_ = AuThreads::GetThreadId();
this->reentrantWriteLockHandle_ = GetThreadCookie();
return true;
}
}
@ -283,7 +283,7 @@ namespace Aurora::Threading::Primitives
{
if (this->state_ < 0)
{
SysAssertDbg(this->reentrantWriteLockHandle_ == AuThreads::GetThreadId());
SysAssertDbg(this->reentrantWriteLockHandle_ == GetThreadCookie());
return;
}
@ -339,7 +339,7 @@ namespace Aurora::Threading::Primitives
}
this->bElevaterPending_ = false;
this->reentrantWriteLockHandle_ = AuThreads::GetThreadId();
this->reentrantWriteLockHandle_ = GetThreadCookie();
this->state_ = -1;
return true;
}

View File

@ -9,6 +9,7 @@
#include "AuConditionVariable.Generic.hpp"
#include "AuConditionMutex.Generic.hpp"
#include "ThreadCookie.hpp"
namespace Aurora::Threading::Primitives
{
@ -73,7 +74,7 @@ namespace Aurora::Threading::Primitives
private:
//bool reentrantWriteLock_ {true};
AuUInt reentrantWriteLockHandle_ {};
ThreadCookie_t reentrantWriteLockHandle_ {};
RWLockAccessView<true, RWLockImpl> read_;
RWLockAccessView<false, RWLockImpl> write_;

View File

@ -0,0 +1,41 @@
/***
Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved.
File: ThreadCookie.hpp
Date: 2023-3-14
Author: Reece
***/
#pragma once
#if defined(AURORA_IS_MODERNNT_DERIVED) && defined(AURORA_IS_64BIT)
extern "C"
{
AuUInt32 GetCurrentThreadIDFast();
}
#endif
namespace Aurora::Threading::Primitives
{
using ThreadCookie_t = AuUInt;
static auline ThreadCookie_t GetThreadCookie()
{
#if defined(AURORA_IS_MODERNNT_DERIVED)
#if defined(AURORA_IS_32BIT)
AuUInt32 uRet;
__asm {
mov eax, fs:[0x18]
mov [uRet], [eax + 0x24]
}
return uRet;
#else
return ::GetCurrentThreadIDFast();
#endif
#elif defined(AURORA_IS_POSIX_DERIVED)
return (ThreadCookie_t)pthread_self();
#else
return (ThreadCookie_t)Threads::GetThread();
#endif
}
}

View File

@ -0,0 +1,16 @@
;; Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved.
;;
;; File:
;; Date: 2023-3-14
;; Author: Reece
public GetCurrentThreadIDFast
.code
GetCurrentThreadIDFast PROC
mov rax, gs:[30h]
mov eax, [rax+48h]
ret
GetCurrentThreadIDFast ENDP
END