/*** Copyright (C) 2021-2024 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: AuRWLock.cpp Date: 2021-6-12 Author: Reece ***/ #include //#define RWLOCK_VIEW_HAS_PARENT #include "AuRWLock.hpp" #include "SMTYield.hpp" #include "../AuWakeInternal.hpp" #if defined(AURORA_RWLOCK_IS_CONDLESS) #define gUseFutexRWLock 1 #endif namespace Aurora::Threading::Primitives { /* static const AuUInt8 ... = ...; */ #define kOffsetOfRead AuOffsetOf(&RWLockImpl::read_) /* static const AuUInt8 ... = ...; */ #define kOffsetOfWrite AuOffsetOf(&RWLockImpl::write_) #if defined(RWLOCK_VIEW_HAS_PARENT) #define ViewParent (&this->parent_) #else #define ViewParent ((T *)(((char *)this) - (bIsReadView ? kOffsetOfRead : kOffsetOfWrite))) #endif #define RWLockAcquire AU_LOCK_GUARD(this->mutex_); #define RWLockBarrier(expMutex) { (expMutex).Lock(); (expMutex).Unlock(); } #define RWLockBarrierSelf() RWLockBarrier(this->mutex_) static const auto kRWThreadWriterHardContextSwitchBias = 15; template void RWLockAccessView::Unlock() { if constexpr (bIsReadView) { ViewParent->UnlockRead(); } else { ViewParent->UnlockWrite(); } } template bool RWLockAccessView::LockAbsMS(AuUInt64 timeout) { if constexpr (bIsReadView) { return ViewParent->LockReadNSAbs(AuMSToNS(timeout)); } else { return ViewParent->LockWriteNSAbs(AuMSToNS(timeout)); } } template bool RWLockAccessView::LockAbsNS(AuUInt64 timeout) { if constexpr (bIsReadView) { return ViewParent->LockReadNSAbs(timeout); } else { return ViewParent->LockWriteNSAbs(timeout); } } template bool RWLockAccessView::LockMS(AuUInt64 timeout) { if constexpr (bIsReadView) { return ViewParent->LockReadNS(AuMSToNS(timeout)); } else { return ViewParent->LockWriteNS(AuMSToNS(timeout)); } } template bool RWLockAccessView::LockNS(AuUInt64 timeout) { if constexpr (bIsReadView) { return ViewParent->LockReadNS(timeout); } else { return ViewParent->LockWriteNS(timeout); } } template bool RWLockAccessView::TryLock() { if constexpr (bIsReadView) { return ViewParent->TryLockRead(); } else { return ViewParent->TryLockWriteMaybeSpin(); } } template RWLockImpl::RWLockImpl() #if defined(RWLOCK_VIEW_HAS_PARENT) : read_(*this), write_(*this) #endif #if 0 , condition_(AuUnsafeRaiiToShared(&this->mutex_)), conditionWriter_(AuUnsafeRaiiToShared(&this->mutex_)) #endif { } template RWLockImpl::~RWLockImpl() { } #if !defined(AURORA_RWLOCK_IS_CONDLESS) template ConditionVariableInternal &RWLockImpl::GetCondition() { #if !defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) return *(ConditionVariableInternal *)this->conditionVariable_; #else return this->condition_; #endif } template ConditionVariableInternal &RWLockImpl::GetConditionWriter() { #if !defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) return *(ConditionVariableInternal *)this->conditionVariableWriter_; #else return this->conditionWriter_; #endif } #endif template AuUInt32 *RWLockImpl::GetFutexCondition() { return (AuUInt32 *)&this->iState_; } template AuUInt32 *RWLockImpl::GetFutexConditionWriter() { #if defined(AURORA_RWLOCK_IS_CONDLESS) return (AuUInt32 *)&this->uSemaphore_; #else return (AuUInt32 *)this->conditionVariableWriter_; #endif } template AuUInt32 *RWLockImpl::GetReadSleepCounter() { #if defined(AURORA_RWLOCK_IS_CONDLESS) return (AuUInt32 *)&this->uRDCounter_; #else return (AuUInt32 *)this->conditionVariable_; #endif } template bool RWLockImpl::LockReadNSAbs(AuUInt64 uTimeout) { AuInt32 iCurState {}; bool bRet {}; if (this->TryLockRead()) { return true; } auto pCounter = this->GetReadSleepCounter(); do { iCurState = this->iState_; if (iCurState < 0) { if (gUseFutexRWLock) { AuAtomicAdd(pCounter, 1u); bRet = InternalLTSWaitOnAddressHighRes((const void *)&this->iState_, &iCurState, sizeof(iCurState), uTimeout); AuAtomicSub(pCounter, 1u); if (!bRet) { return false; } } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockAcquire; iCurState = this->iState_; if (iCurState < 0) { AuInt64 iSecondTimeout {}; if (uTimeout) { iSecondTimeout = AuInt64(uTimeout) - AuTime::SteadyClockNS(); if (iSecondTimeout <= 0) { return false; } } #if defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) if (!this->GetCondition().WaitForSignalNS(iSecondTimeout)) #else if (!this->GetCondition().WaitForSignalNsEx(&this->mutex_, iSecondTimeout)) #endif { return false; } } } #endif } } while (iCurState < 0 || AuAtomicCompareExchange(&this->iState_, iCurState + 1, iCurState) != iCurState); return true; } template bool RWLockImpl::LockReadNS(AuUInt64 uTimeout) { AuInt32 iCurState {}; bool bRet {}; if (this->TryLockReadNoSpin()) { return true; } AuInt64 uEndTime = uTimeout ? AuTime::SteadyClockNS() + uTimeout : 0; if (this->TryLockRead()) { return true; } auto pCounter = this->GetReadSleepCounter(); do { iCurState = this->iState_; if (iCurState < 0) { if (gUseFutexRWLock) { AuAtomicAdd(pCounter, 1u); bRet = InternalLTSWaitOnAddressHighRes((const void *)&this->iState_, &iCurState, sizeof(iCurState), uEndTime); AuAtomicSub(pCounter, 1u); if (!bRet) { return false; } } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockAcquire; iCurState = this->iState_; if (iCurState < 0) { AuInt64 iSecondTimeout {}; if (uTimeout) { iSecondTimeout = uEndTime - AuTime::SteadyClockNS(); if (iSecondTimeout <= 0) { return false; } } #if defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) if (!this->GetCondition().WaitForSignalNS(iSecondTimeout)) #else if (!this->GetCondition().WaitForSignalNsEx(&this->mutex_, iSecondTimeout)) #endif { return false; } } } #endif } } while (iCurState < 0 || AuAtomicCompareExchange(&this->iState_, iCurState + 1, iCurState) != iCurState); return true; } template bool RWLockImpl::TryLockWriteMaybeSpin() { if (this->TryLockWriteNoSpin()) { return true; } if (gUseFutexRWLock) { if (DoTryIfAlderLake([=]() { return this->TryLockWriteNoSpin(); }, &this->iState_)) { return true; } } return false; } template bool RWLockImpl::LockWriteNSAbs(AuUInt64 uTimeout) { bool bRet {}; if (this->TryLockWriteMaybeSpin()) { return true; } AuAtomicAdd(&this->dwWritersPending_, 1); if (gUseFutexRWLock) { bRet = this->LockWriteNSAbsUnlocked(uTimeout); } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockAcquire; bRet = this->LockWriteNSAbsUnlocked(uTimeout); } #endif AuAtomicSub(&this->dwWritersPending_, 1); return bRet; } template bool RWLockImpl::TryLockWriteNoSpin() { auto uOld = this->iState_; if constexpr (bIsWriteRecursionAllowed) { if (uOld < 0) { if (this->reentrantWriteLockHandle_ == GetThreadCookie()) { AuAtomicSub(&this->iState_, 1); return true; } } } if (uOld == 0) { if (AuAtomicCompareExchange(&this->iState_, -1, uOld) == uOld) { this->reentrantWriteLockHandle_ = GetThreadCookie(); return true; } } return false; } template bool RWLockImpl::LockWriteNS(AuUInt64 uTimeout) { bool bRet {}; AuInt64 uEndTime {}; if (this->TryLockWriteMaybeSpin()) { return true; } AuAtomicAdd(&this->dwWritersPending_, 1); if (gUseFutexRWLock) { uEndTime = uTimeout ? AuTime::SteadyClockNS() + uTimeout : 0; bRet = this->LockWriteNSAbsUnlocked(uEndTime); } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { uEndTime = uTimeout ? AuTime::SteadyClockNS() + uTimeout : 0; RWLockAcquire; bRet = this->LockWriteNSAbsUnlocked(uEndTime); } #endif AuAtomicSub(&this->dwWritersPending_, 1); return bRet; } template bool RWLockImpl::LockWriteNSAbsUnlocked(AuUInt64 qwTimeoutNS) { while (true) { if (this->WriterSleep(qwTimeoutNS)) { if (this->WriterTryLock()) { return true; } } else { return this->WriterTryLock(); } } } template void RWLockImpl::FutexWriterWake() { auto pSemaphore = this->GetFutexConditionWriter(); while (true) { auto uState = *pSemaphore; if (uState == 0) { break; } if (AuAtomicCompareExchange(pSemaphore, uState - 1, uState) == uState) { break; } } } template bool RWLockImpl::WriterTryLock() { if (AuAtomicCompareExchange(&this->iState_, -1, 0) == 0) { this->reentrantWriteLockHandle_ = GetThreadCookie(); return true; } else { return false; } } template bool RWLockImpl::WriterSleep(AuUInt64 qwTimeoutNS) { bool bStatus { true }; AuInt32 iCurState; auto pSemaphore = this->GetFutexConditionWriter(); while ((iCurState = AuAtomicLoad(&this->iState_)) != 0) { AuInt64 uSecondTimeout = 0; if (gUseFutexRWLock) { static const AuUInt32 kExpect { 0 }; bStatus = InternalLTSWaitOnAddressHighRes(pSemaphore, &kExpect, sizeof(kExpect), qwTimeoutNS); this->WriterWake(); if (!bStatus) { return false; } this->FutexWriterWake(); } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { if (qwTimeoutNS) { uSecondTimeout = qwTimeoutNS - AuTime::SteadyClockNS(); if (uSecondTimeout <= 0) { return false; } } #if defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) bStatus = this->GetConditionWriter().WaitForSignalNS(uSecondTimeout); #else bStatus = this->GetConditionWriter().WaitForSignalNsEx(&this->mutex_, uSecondTimeout); #endif this->WriterWake(); if (!bStatus) { continue; } } #endif } return true; } template void RWLockImpl::WriterWake() { if (AuAtomicLoad(&this->iState_) == 1 && AuAtomicLoad(&this->dwWritersPending_) > 1) { this->SignalManyWriter(-1); } } template void RWLockImpl::SignalOneReader() { // Unused method! if (gUseFutexRWLock) { InternalLTSWakeOne((const void *)&this->iState_); } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockBarrierSelf(); this->GetCondition().Signal(); } #endif } template void RWLockImpl::SignalOneWriter() { if (gUseFutexRWLock) { auto pThat = this->GetFutexConditionWriter(); AuAtomicAdd(pThat, 1u); InternalLTSWakeOne(pThat); // Note: all paths check for a waiter ahead of time! } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockBarrierSelf(); this->GetConditionWriter().Signal(); } #endif } template void RWLockImpl::SignalManyReader() { if (gUseFutexRWLock) { if (AuAtomicLoad(this->GetReadSleepCounter())) { InternalLTSWakeAll((const void *)&this->iState_); } } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockBarrierSelf(); this->GetCondition().Broadcast(); } #endif } template void RWLockImpl::SignalManyWriter(int iBias) { if (gUseFutexRWLock) { auto pThat = this->GetFutexConditionWriter(); if (auto uCount = AuUInt32(AuAtomicLoad(&this->dwWritersPending_) + iBias)) { AuAtomicAdd(pThat, uCount); InternalLTSWakeCount(pThat, uCount); } } #if !defined(AURORA_RWLOCK_IS_CONDLESS) else { RWLockBarrierSelf(); this->GetConditionWriter().Broadcast(); } #endif } template bool RWLockImpl::TryLockRead() { if (this->TryLockReadNoSpin()) { return true; } if (ThrdCfg::gPreferRWLockReadLockSpin && AuAtomicLoad(&this->dwWritersPending_) == 0) { return DoTryIfAlderLake([=]() { return this->TryLockReadNoSpin(); }, &this->iState_); } return false; } template template bool RWLockImpl::TryLockReadNoSpin() { auto iCurState = this->iState_; if (iCurState < 0) { return this->reentrantWriteLockHandle_ == GetThreadCookie(); } if constexpr (bCheckWrite) { if ((AuAtomicLoad(&this->dwWritersPending_)) && (iCurState > 0 || ThrdCfg::gAlwaysRWLockWriteBiasOnReadLock) && (ThrdCfg::gEnableRWLockWriteBiasOnReadLock)) { return false; } } return AuAtomicCompareExchange(&this->iState_, iCurState + 1, iCurState) == iCurState; } template void RWLockImpl::UnlockRead() { AuInt32 uVal {}; if (this->iState_ < 0) { SysAssertDbg(this->reentrantWriteLockHandle_ == GetThreadCookie()); return; } uVal = AuAtomicSub(&this->iState_, 1); bool bAlt {}; if constexpr (bIsWriteRecursionAllowed) { bAlt = uVal == 1; } if (uVal == 0 || bAlt) { if (AuAtomicLoad(&this->dwWritersPending_) > 0) { this->SignalOneWriter(); } } } template void RWLockImpl::UnlockWrite() { if constexpr (!bIsWriteRecursionAllowed) { this->reentrantWriteLockHandle_ = 0; AuAtomicStore(&this->iState_, 0); if (AuAtomicLoad(&this->dwWritersPending_) > 0) { this->SignalOneWriter(); } else { this->SignalManyReader(); } } else { AuInt32 val {}; // love me cas { AuInt32 curVal {}; do { curVal = this->iState_; if (curVal != -1) { continue; } this->reentrantWriteLockHandle_ = 0; } while (AuAtomicCompareExchange(&this->iState_, val = (curVal + 1), curVal) != curVal); } if (val == 0) { if (AuAtomicLoad(&this->dwWritersPending_) > 0) { this->SignalOneWriter(); } else { this->SignalManyReader(); } } } } template bool RWLockImpl::UpgradeReadToWrite(AuUInt64 uTimeout) { if (this->iState_ == 1) { if (this->UpgradeReadToWriteDoUpgrade()) { return true; } } auto uEndTime = uTimeout ? AuTime::SteadyClockNS() + uTimeout : 0; #if !defined(AURORA_RWLOCK_IS_CONDLESS) if (!gUseFutexRWLock) { AuAtomicAdd(&this->dwWritersPending_, 1); RWLockAcquire; while (this->iState_ != 1) { AuInt64 iSecondTimeout {}; if (uTimeout) { iSecondTimeout = AuInt64(uEndTime) - AuTime::SteadyClockNS(); if (iSecondTimeout <= 0) { AuAtomicSub(&this->dwWritersPending_, 1); return false; } } #if defined(AURWLOCK_NO_SIZE_OPTIMIZED_CONDVAR) if (!this->GetConditionWriter().WaitForSignalNS(iSecondTimeout)) #else if (!this->GetConditionWriter().WaitForSignalNsEx(&this->mutex_, iSecondTimeout)) #endif { AuAtomicSub(&this->dwWritersPending_, 1); return false; } } AuAtomicSub(&this->dwWritersPending_, 1); return this->UpgradeReadToWriteDoUpgrade(); } else #endif { while (true) { auto pSemaphore = this->GetFutexConditionWriter(); AuInt32 iCurState; while ((iCurState = AuAtomicLoad(&this->iState_)) != 1) { bool bStatusTwo {}; bool bStatus {}; AuAtomicAdd(&this->dwWritersPending_, 1); static const AuUInt32 kExpect { 0 }; if ((iCurState = AuAtomicLoad(&this->iState_)) == 1) { bStatus = true; bStatusTwo = true; } else { bStatus = InternalLTSWaitOnAddressHighRes(pSemaphore, &kExpect, sizeof(kExpect), uEndTime); } AuAtomicSub(&this->dwWritersPending_, 1); if (!bStatus) { return false; } if (!bStatusTwo) { while (true) { auto uState = *pSemaphore; if (uState == 0) { break; } if (AuAtomicCompareExchange(pSemaphore, uState - 1, uState) == uState) { break; } } } } if (this->UpgradeReadToWriteDoUpgrade()) { return true; } } } /* unreachable */ return false; } template bool RWLockImpl::UpgradeReadToWriteDoUpgrade() { if (AuAtomicCompareExchange(&this->iState_, -1, 1) == 1) { this->reentrantWriteLockHandle_ = GetThreadCookie(); return true; } else { return false; } } template bool RWLockImpl::DowngradeWriteToRead() { if (AuAtomicCompareExchange(&this->iState_, 1, -1) == -1) { this->SignalManyReader(); return true; } else { return false; } } template IWaitable *RWLockImpl::AsReadable() { return &this->read_; } template IWaitable *RWLockImpl::AsWritable() { return &this->write_; } template bool RWLockImpl::CheckSelfThreadIsWriter() { return this->reentrantWriteLockHandle_ == GetThreadCookie(); } AUKN_SYM IRWLock *RWLockNew() { return _new RWLockImpl(); } AUKN_SYM void RWLockRelease(IRWLock *pRwLock) { AuSafeDelete *>(pRwLock); } AUKN_SYM IRWLock *RWRenterableLockNew() { return _new RWLockImpl(); } AUKN_SYM void RWRenterableLockRelease(IRWLock *pRwLock) { AuSafeDelete *>(pRwLock); } AUROXTL_INTERFACE_SOO_SRC_EX(AURORA_SYMBOL_EXPORT, RWRenterableLock, RWLockImpl) AUROXTL_INTERFACE_SOO_SRC_EX(AURORA_SYMBOL_EXPORT, RWLock, RWLockImpl) }