2023-03-12 15:27:28 +00:00
/***
Copyright ( C ) 2023 J Reece Wilson ( a / k / a " Reece " ) . All rights reserved .
File : AuWakeOnAddress . cpp
Date : 2023 - 3 - 10
Author : Reece
* * */
# include <Source/RuntimeInternal.hpp>
# include "AuWakeOnAddress.hpp"
2023-03-21 03:18:09 +00:00
# include "Primitives/SMTYield.hpp"
2023-03-12 15:27:28 +00:00
2023-03-21 03:18:09 +00:00
namespace Aurora : : Threading
2023-03-12 15:27:28 +00:00
{
# if defined(AURORA_IS_LINUX_DERIVED)
static int futex_wait ( uint32_t * addr , uint32_t expected , const struct timespec * timeout )
{
if ( timeout )
{
return futex ( addr , FUTEX_WAIT_BITSET , expected , timeout , 0 , FUTEX_BITSET_MATCH_ANY ) ;
}
else
{
return futex ( addr , FUTEX_WAIT , expected , timeout , 0 , 0 ) ;
}
}
static int futex_wake ( uint32_t * addr , uint32_t nthreads )
{
return futex ( addr , FUTEX_WAKE , nthreads , 0 , 0 , 0 ) ;
}
# endif
static ProcessWaitContainer gProcessWaitables ;
bool WaitEntry : : TryAcquire ( const void * pAddress , AuUInt8 uSize )
{
while ( AuAtomicTestAndSet ( & this - > uAtomic , 0 ) )
{
while ( this - > uAtomic )
{
AuThreading : : ContextYield ( ) ;
}
}
//AU_LOCK_GUARD(this->mutex);
if ( this - > pAddress )
{
this - > uAtomic = 0 ;
return false ;
}
this - > pAddress = pAddress ;
this - > uSize = uSize ;
this - > uAtomic = 0 ;
return true ;
}
void WaitEntry : : Release ( )
{
if ( this - > bOverflow )
{
gProcessWaitables . Remove ( this ) ;
this - > bOverflow = false ;
}
AuResetMember ( this - > uSize ) ;
AuResetMember ( this - > pAddress ) ;
}
WaitEntry : : WaitEntry ( ) :
variable ( AuUnsafeRaiiToShared ( & this - > mutex ) )
{
}
WaitEntry : : ~ WaitEntry ( )
{
this - > Release ( ) ;
}
bool WaitEntry : : SleepOn ( WaitState & state )
{
AU_LOCK_GUARD ( this - > mutex ) ;
if ( state . qwNanoseconds )
{
if ( ! WaitBuffer : : From ( this - > pAddress , this - > uSize ) . Compare ( state ) )
{
return true ;
}
auto uNow = AuTime : : SteadyClockNS ( ) ;
auto uEndTime = uNow + state . qwNanoseconds . value ( ) ;
# if defined(AURORA_IS_POSIX_DERIVED)
struct timespec tspec ;
Time : : auabsns2ts ( & tspec , uEndTime ) ;
# endif
while ( uNow < uEndTime )
{
if ( ! WaitBuffer : : From ( this - > pAddress , this - > uSize ) . Compare ( state ) )
{
return true ;
}
auto uTimeRemNS = uEndTime - uNow ;
# if defined(AURORA_IS_POSIX_DERIVED)
auto pCond = reinterpret_cast < pthread_cond_t * > ( & this - > variable . pthreadCv_ ) ;
auto mutex = reinterpret_cast < pthread_mutex_t * > ( this - > mutex - > GetOSHandle ( ) ) ;
int ret { } ;
do
{
ret = : : pthread_cond_timedwait ( pCond , mutex , & tspec ) ;
if ( ret = = 0 )
{
break ;
}
if ( ret = = ETIMEDOUT )
{
return ! WaitBuffer : : From ( this - > pAddress , this - > uSize ) . Compare ( state ) ;
}
}
while ( ret = = EINTR ) ;
# else
2023-03-17 15:40:15 +00:00
this - > variable . WaitForSignalNS ( uTimeRemNS ) ;
2023-03-12 15:27:28 +00:00
# endif
uNow = AuTime : : SteadyClockNS ( ) ;
}
return ! WaitBuffer : : From ( this - > pAddress , this - > uSize ) . Compare ( state ) ;
}
else
{
while ( WaitBuffer : : From ( this - > pAddress , this - > uSize ) . Compare ( state ) )
{
this - > variable . WaitForSignal ( 0 ) ;
}
return true ;
}
return false ;
}
bool WaitEntry : : TryWake ( const void * pAddress )
{
while ( AuAtomicTestAndSet ( & this - > uAtomic , 0 ) )
{
while ( this - > uAtomic )
{
AuThreading : : ContextYield ( ) ;
}
}
auto bRet = TryWakeNoLock ( pAddress ) ;
if ( ! bRet )
{
this - > uAtomic = 0 ;
}
return bRet ;
}
bool WaitEntry : : TryWakeNoLockNoReallyNoLock ( const void * pAddress )
{
if ( AuReinterpretCast < const char * > ( this - > pAddress ) > AuReinterpretCast < const char * > ( pAddress ) | |
AuReinterpretCast < const char * > ( this - > pAddress ) + this - > uSize < = AuReinterpretCast < const char * > ( pAddress ) )
{
return false ;
}
this - > uAtomic = 0 ;
this - > variable . Signal ( ) ;
return true ;
}
bool WaitEntry : : TryWakeNoLock ( const void * pAddress )
{
if ( AuReinterpretCast < const char * > ( this - > pAddress ) > AuReinterpretCast < const char * > ( pAddress ) | |
AuReinterpretCast < const char * > ( this - > pAddress ) + this - > uSize < = AuReinterpretCast < const char * > ( pAddress ) )
{
return false ;
}
AU_LOCK_GUARD ( this - > mutex ) ;
this - > uAtomic = 0 ;
this - > variable . Signal ( ) ;
return true ;
}
WaitBuffer WaitBuffer : : From ( const void * pBuf , AuUInt8 uSize )
{
WaitBuffer wait ;
AuMemcpy ( wait . buffer , pBuf , uSize ) ;
wait . uSize = uSize ;
return AuMove ( wait ) ;
}
bool WaitBuffer : : Compare ( const void * pBuf )
{
return AuMemcmp ( this - > buffer , pBuf , this - > uSize ) = = 0 ;
}
bool WaitBuffer : : Compare ( WaitState & state )
{
if ( ! state . uDownsizeMask )
{
return AuMemcmp ( this - > buffer , state . compare . buffer , AuMin ( this - > uSize , state . compare . uSize ) ) = = 0 ;
}
else
{
auto uMask = state . uDownsizeMask . value ( ) ;
auto & uSrcWord = * AuReinterpretCast < AuUInt32 * > ( this - > buffer ) ;
auto & uCmpWord = * AuReinterpretCast < AuUInt32 * > ( state . compare . buffer ) ;
return ( uSrcWord & uMask ) = = ( uCmpWord & uMask ) ;
}
}
AuSPtr < WaitEntry > ProcessWaitContainer : : WaitBufferFrom ( void * pAddress , AuUInt8 uSize )
{
for ( AU_ITERATE_N ( i , kDefaultWaitPerProcess ) )
{
if ( this - > entries [ i ] . TryAcquire ( pAddress , uSize ) )
{
return AuUnsafeRaiiToShared ( & this - > entries [ i ] ) ;
}
}
AuSPtr < WaitEntry > pNew ;
{
AuDebug : : AddMemoryCrunch ( ) ;
pNew = AuMakeSharedPanic < WaitEntry > ( ) ;
AuDebug : : DecMemoryCrunch ( ) ;
}
{
Lock ( ) ;
this - > overflow . push_back ( pNew ) ;
Unlock ( ) ;
}
return pNew ;
}
template < typename T >
bool ProcessWaitContainer : : IterateAll ( T callback )
{
for ( AU_ITERATE_N ( i , kDefaultWaitPerProcess ) )
{
auto & entry = this - > entries [ i ] ;
{
while ( AuAtomicTestAndSet ( & entry . uAtomic , 0 ) )
{
while ( entry . uAtomic )
{
AuThreading : : ContextYield ( ) ;
}
}
if ( ! entry . pAddress )
{
entry . uAtomic = 0 ;
}
else
{
AU_LOCK_GUARD ( entry . mutex ) ;
entry . uAtomic = 0 ;
if ( ! callback ( entry ) )
{
return false ;
}
}
}
}
Lock ( ) ;
for ( auto & overflow : this - > overflow )
{
AU_LOCK_GUARD ( overflow - > mutex ) ;
if ( ! callback ( * overflow . get ( ) ) )
{
return false ;
}
}
Unlock ( ) ;
return true ;
}
template < typename T >
bool ProcessWaitContainer : : IterateForceNoCreateDuringOp ( T callback )
{
bool bRetStatus { true } ;
for ( AU_ITERATE_N ( i , kDefaultWaitPerProcess ) )
{
auto & entry = this - > entries [ i ] ;
{
while ( AuAtomicTestAndSet ( & entry . uAtomic , 0 ) )
{
while ( entry . uAtomic )
{
AuThreading : : ContextYield ( ) ;
}
}
if ( entry . pAddress )
{
AU_LOCK_GUARD ( entry . mutex ) ;
if ( ! callback ( entry ) )
{
for ( AU_ITERATE_N ( z , i + 1 ) )
{
this - > entries [ z ] . uAtomic = 0 ;
}
return false ;
}
}
}
}
Lock ( ) ;
for ( auto & overflow : this - > overflow )
{
AU_LOCK_GUARD ( overflow - > mutex ) ;
if ( ! callback ( * overflow . get ( ) ) )
{
bRetStatus = false ;
break ;
}
}
Unlock ( ) ;
for ( AU_ITERATE_N ( i , kDefaultWaitPerProcess ) )
{
auto & entry = this - > entries [ i ] ;
{
entry . uAtomic = 0 ;
}
}
return bRetStatus ;
}
void ProcessWaitContainer : : Lock ( )
{
while ( AuAtomicTestAndSet ( & this - > uAtomic , 0 ) )
{
while ( this - > uAtomic )
{
AuThreading : : ContextYield ( ) ;
}
}
}
void ProcessWaitContainer : : Unlock ( )
{
this - > uAtomic = 0 ;
}
void ProcessWaitContainer : : Remove ( WaitEntry * pParent )
{
Lock ( ) ;
for ( auto itr = this - > overflow . begin ( ) ;
itr ! = this - > overflow . end ( ) ;
)
{
if ( ( * itr ) . get ( ) = = pParent )
{
itr = this - > overflow . erase ( itr ) ;
}
else
{
itr + + ;
}
}
Unlock ( ) ;
}
AUKN_SYM bool IsWaitOnRecommended ( )
{
# if defined(AURORA_IS_MODERNNT_DERIVED)
return pWaitOnAddress & &
AuSwInfo : : IsWindows8Point1OrGreater ( ) ;
# elif defined(AURORA_PLATFORM_LINUX)
return true ;
# endif
return false ;
}
/// @deprecated
AUKN_SYM const AuList < AuUInt8 > & GetValidWordSizes ( )
{
static const AuList < AuUInt8 > kArray =
# if defined(AURORA_IS_MODERNNT_DERIVED)
{ 1 , 2 , 4 , 8 } ;
# else
{ 4 } ;
# endif
return kArray ;
}
bool WaitOnAddressWide ( void * pTargetAddress ,
void * pCompareAddress ,
AuUInt8 uWordSize ,
AuOptional < AuUInt64 > qwNanoseconds ,
bool bOSSupportsWait
)
{
WaitState state ;
2023-04-06 21:05:45 +00:00
SysAssertDbg ( uWordSize < = 8 ) ;
2023-03-12 15:27:28 +00:00
auto pWaitEntry = gProcessWaitables . WaitBufferFrom ( pTargetAddress , uWordSize ) ;
state . compare = WaitBuffer : : From ( pCompareAddress , uWordSize ) ;
state . qwNanoseconds = qwNanoseconds ? AuOptionalEx < AuUInt64 > { qwNanoseconds . value ( ) } : AuOptionalEx < AuUInt64 > { } ; // from default/zeroable optional, to boolean suffix
auto bResult = pWaitEntry - > SleepOn ( state ) ;
pWaitEntry - > Release ( ) ;
return bResult ;
}
2023-03-13 23:57:32 +00:00
AuTuple < const void * , AuUInt8 , AuOptionalEx < AuUInt32 > > DecodeAddress ( const void * pAddress ,
2023-04-16 22:58:27 +00:00
AuUInt32 uWordSize )
2023-03-12 15:27:28 +00:00
{
# if defined(AURORA_IS_MODERNNT_DERIVED)
2023-03-13 23:57:32 +00:00
return AuMakeTuple ( pAddress , 0 , AuOptionalEx < AuUInt32 > { } ) ;
2023-03-12 15:27:28 +00:00
# endif
2023-04-16 22:58:27 +00:00
if ( uWordSize = = 8 )
{
return AuMakeTuple ( pAddress , 0xFFFFFFFF , 0xFFFFFFFF ) ;
}
2023-03-12 15:27:28 +00:00
auto pRounded = AuPageRound ( AuUInt ( pAddress ) , AuUInt ( 4 ) ) ;
auto uDelta = ( AuUInt ) ( pAddress ) - ( AuUInt ) ( pRounded ) ;
2023-04-16 22:58:27 +00:00
AuUInt32 uSizeMask = std : : pow ( AuUInt64 ( 2 ) , AuUInt64 ( uWordSize * 8 ) ) - 1ull ;
2023-03-12 15:27:28 +00:00
switch ( uDelta )
{
case 0 :
return AuMakeTuple ( pAddress , 0 , 0xFFFFFFFF & ( uSizeMask < < 0 ) ) ;
case 1 :
return AuMakeTuple ( pAddress , 1 , 0xFFFFFF00 & ( uSizeMask < < 8 ) ) ;
case 2 :
return AuMakeTuple ( pAddress , 2 , 0xFFFF0000 & ( uSizeMask < < 16 ) ) ;
case 3 :
return AuMakeTuple ( pAddress , 3 , 0xFF000000 & ( uSizeMask < < 24 ) ) ;
default :
SysPanic ( " Invalid Branch " ) ;
}
}
static bool RunOSWaitOnAddressNoTimed ( const void * pTargetAddress ,
const void * pCompareAddress ,
AuUInt8 dwWordSize )
{
# if defined(AURORA_IS_MODERNNT_DERIVED)
return pWaitOnAddress ( ( void * ) pTargetAddress , ( void * ) pCompareAddress , dwWordSize , 0 ) ;
# endif
# if defined(AURORA_IS_LINUX_DERIVED)
int ret { } ;
2023-03-22 15:19:32 +00:00
# if defined(AU_CPU_ENDIAN_BIG)
2023-03-12 15:27:28 +00:00
if ( dwWordSize = = 8 )
{
pTargetAddress = AuReinterpretCast < const char * > ( pTargetAddress ) + 4 ;
pCompareAddress = AuReinterpretCast < const char * > ( pCompareAddress ) + 4 ;
}
# endif
auto uCurrent = * ( AuUInt32 * ) pCompareAddress ;
do
{
ret = futex_wait ( ( AuUInt32 * ) pTargetAddress , uCurrent , nullptr ) ;
if ( ret = = 0 )
{
continue ;
}
if ( ret = = EAGAIN | | errno = = EAGAIN )
{
continue ;
}
if ( ret = = ETIMEDOUT | | errno = = ETIMEDOUT )
{
return false ;
}
}
while ( ret = = EINTR ) ;
return true ;
# endif
return false ;
}
static bool RunOSWaitOnAddressTimed ( const void * pTargetAddress ,
const void * pCompareAddress ,
AuUInt8 uWordSize ,
AuUInt64 uAbsTime ,
AuUInt32 uNanoseconds )
{
# if defined(AURORA_IS_MODERNNT_DERIVED)
auto uMS = AuNSToMS < AuUInt32 > ( uNanoseconds ) ;
if ( ! uMS )
{
auto expect = WaitBuffer : : From ( pCompareAddress , uWordSize ) ;
do
{
if ( expect . Compare ( pTargetAddress ) )
{
AuThreading : : ContextYield ( ) ;
}
}
while ( uAbsTime > AuTime : : SteadyClockNS ( ) ) ;
}
else
{
2023-03-13 23:57:32 +00:00
uMS = AuMin < AuUInt32 > ( uMS , 2000 ) ;
( void ) pWaitOnAddress ( ( void * ) pTargetAddress , ( void * ) pCompareAddress , uWordSize , uMS ) ;
if ( ! WaitBuffer : : From ( pCompareAddress , uWordSize ) . Compare ( pTargetAddress ) )
{
return true ;
}
AuUInt64 uNow { } ;
while ( uAbsTime > ( uNow = AuTime : : SteadyClockNS ( ) ) )
{
uMS = AuNSToMS < AuUInt32 > ( uAbsTime - uNow ) ;
if ( Primitives : : DoTryIf ( [ = ] ( )
{
return ! WaitBuffer : : From ( pCompareAddress , uWordSize ) . Compare ( pTargetAddress ) ;
} ) )
{
return true ;
}
if ( ! uMS )
{
AuThreading : : ContextYield ( ) ;
}
else
{
( void ) pWaitOnAddress ( ( void * ) pTargetAddress , ( void * ) pCompareAddress , uWordSize , uMS ) ;
}
}
2023-03-12 15:27:28 +00:00
}
# endif
# if defined(AURORA_IS_LINUX_DERIVED)
int ret { } ;
2023-03-22 15:19:32 +00:00
# if defined(AU_CPU_ENDIAN_BIG)
2023-03-12 15:27:28 +00:00
if ( uWordSize = = 8 )
{
pTargetAddress = AuReinterpretCast < const char * > ( pTargetAddress ) + 4 ;
pCompareAddress = AuReinterpretCast < const char * > ( pCompareAddress ) + 4 ;
}
# endif
auto uCurrent = * ( AuUInt32 * ) pCompareAddress ;
struct timespec tspec ;
Time : : auabsns2ts ( & tspec , uAbsTime ) ;
do
{
ret = futex_wait ( ( AuUInt32 * ) pTargetAddress , uCurrent , & tspec ) ;
if ( ret = = 0 )
{
continue ;
}
if ( ret = = EAGAIN | | errno = = EAGAIN )
{
continue ;
}
if ( ret = = ETIMEDOUT | | errno = = ETIMEDOUT )
{
return false ;
}
}
while ( ret = = EINTR ) ;
# endif
2023-03-13 23:57:32 +00:00
return ! WaitBuffer : : From ( pCompareAddress , uWordSize ) . Compare ( pTargetAddress ) ;
2023-03-12 15:27:28 +00:00
}
static void RunOSWaitOnAddressNoTimedNoErrors ( const void * pTargetAddress ,
const void * pCompareAddress ,
2023-03-13 23:57:32 +00:00
WaitState & state )
2023-03-12 15:27:28 +00:00
{
2023-03-13 23:57:32 +00:00
while ( WaitBuffer : : From ( pTargetAddress , state . uWordSize ) . Compare ( state ) )
2023-03-12 15:27:28 +00:00
{
2023-03-13 23:57:32 +00:00
if ( ! RunOSWaitOnAddressNoTimed ( pTargetAddress , pCompareAddress , state . uWordSize ) )
2023-03-12 15:27:28 +00:00
{
AuThreading : : ContextYield ( ) ;
}
}
}
static bool RunOSWaitOnAddressTimedNoErrors ( const void * pTargetAddress ,
const void * pCompareAddress ,
2023-03-13 23:57:32 +00:00
WaitState & state )
2023-03-12 15:27:28 +00:00
{
2023-03-13 23:57:32 +00:00
if ( ! WaitBuffer : : From ( pTargetAddress , state . uWordSize ) . Compare ( state ) )
2023-03-12 15:27:28 +00:00
{
return true ;
}
2023-03-13 23:57:32 +00:00
( void ) RunOSWaitOnAddressTimed ( pTargetAddress , pCompareAddress , state . uWordSize , AuTime : : SteadyClockNS ( ) + state . qwNanoseconds . value ( ) , state . qwNanoseconds . value ( ) ) ;
return ! WaitBuffer : : From ( pTargetAddress , state . uWordSize ) . Compare ( state ) ;
2023-03-12 15:27:28 +00:00
}
static void RunOSWakeNOnAddress ( const void * pAddress ,
AuUInt32 dwCount )
{
# if defined(AURORA_IS_LINUX_DERIVED)
futex_wake ( ( AuUInt32 * ) pAddress , dwCount ) ;
# endif
# if defined(AURORA_IS_MODERNNT_DERIVED)
for ( AuUInt i = 0 ; i < dwCount ; i + + )
{
pWakeByAddressSingle ( ( void * ) pAddress ) ;
}
# endif
}
static void RunOSWakeAllOnAddress ( const void * pAddress )
{
# if defined(AURORA_IS_LINUX_DERIVED)
futex_wake ( ( AuUInt32 * ) pAddress , INT_MAX ) ;
# endif
# if defined(AURORA_IS_MODERNNT_DERIVED)
pWakeByAddressAll ( ( void * ) pAddress ) ;
# endif
}
AUKN_SYM bool WaitOnAddress ( void * pTargetAddress ,
void * pCompareAddress ,
AuUInt8 uWordSize ,
AuUInt64 qwNanoseconds )
{
bool bWaitOnAddress = IsWaitOnRecommended ( ) ;
if ( bWaitOnAddress )
{
auto [ pWaitAddress , uDelta , uMask ] = DecodeAddress ( pTargetAddress , uWordSize ) ;
auto pCompareAddress2 = AuReinterpretCast < char * > ( pCompareAddress ) - uDelta ;
2023-03-13 23:57:32 +00:00
WaitState state ;
state . uDownsizeMask = uMask ;
state . compare = uMask ?
WaitBuffer : : From ( pCompareAddress2 , 4 ) :
WaitBuffer : : From ( pCompareAddress2 , uWordSize ) ;
state . uWordSize = uMask ? 4 : uWordSize ;
2023-03-12 15:27:28 +00:00
if ( ! qwNanoseconds )
{
2023-03-13 23:57:32 +00:00
RunOSWaitOnAddressNoTimedNoErrors ( pWaitAddress , pCompareAddress2 , state ) ;
2023-03-12 15:27:28 +00:00
return true ;
}
else
{
2023-03-13 23:57:32 +00:00
state . qwNanoseconds = qwNanoseconds ;
return RunOSWaitOnAddressTimedNoErrors ( pWaitAddress , pCompareAddress2 , state ) ;
2023-03-12 15:27:28 +00:00
}
}
else
{
2023-04-05 08:42:48 +00:00
if ( TryWaitOnAddress ( pTargetAddress , pCompareAddress , uWordSize ) )
{
return true ;
}
2023-03-12 15:27:28 +00:00
return WaitOnAddressWide ( pTargetAddress , pCompareAddress , uWordSize , qwNanoseconds , false ) ;
}
return false ;
}
AUKN_SYM bool TryWaitOnAddress ( void * pTargetAddress ,
void * pCompareAddress ,
AuUInt8 uWordSize )
{
2023-03-13 23:57:32 +00:00
return Primitives : : DoTryIf ( [ = ] ( )
{
return ! WaitBuffer : : From ( pCompareAddress , uWordSize ) . Compare ( pTargetAddress ) ;
} ) ;
2023-03-12 15:27:28 +00:00
}
AUKN_SYM void WakeNOnAddress ( void * pTargetAddress ,
AuUInt8 uNMaximumThreads )
{
if ( IsWaitOnRecommended ( ) )
{
RunOSWakeNOnAddress ( pTargetAddress , uNMaximumThreads ) ;
}
else
{
( void ) gProcessWaitables . IterateForceNoCreateDuringOp ( [ & ] ( WaitEntry & entry ) - > bool
{
if ( ! uNMaximumThreads )
{
return false ;
}
if ( entry . TryWakeNoLockNoReallyNoLock ( pTargetAddress ) )
{
uNMaximumThreads - - ;
}
return uNMaximumThreads ! = 0 ;
} ) ;
}
}
AUKN_SYM void WakeOnAddress ( void * pTargetAddress )
{
WakeNOnAddress ( pTargetAddress , 1 ) ;
}
AUKN_SYM void WakeAllOnAddress ( void * pTargetAddress )
{
if ( IsWaitOnRecommended ( ) )
{
RunOSWakeAllOnAddress ( pTargetAddress ) ;
}
else
{
( void ) gProcessWaitables . IterateForceNoCreateDuringOp ( [ = ] ( WaitEntry & entry ) - > bool
{
entry . TryWakeNoLockNoReallyNoLock ( pTargetAddress ) ;
return true ;
} ) ;
}
}
}