AuroraRuntime/Source/Threading/AuWakeOnAddress.cpp

/***
    Copyright (C) 2023 J Reece Wilson (a/k/a "Reece"). All rights reserved.

    File: AuWakeOnAddress.cpp
    Date: 2023-3-10
    Author: Reece
***/
#include <Source/RuntimeInternal.hpp>
#include "AuWakeOnAddress.hpp"
#include "Primitives/SMTYield.hpp"

#if defined(AURORA_PLATFORM_WIN32)
    #include <timeapi.h>
#endif

#if defined(AURORA_IS_LINUX_DERIVED)
    #include <unistd.h>
#endif

#include <Time/Time.hpp>

#define HACK_NO_INVALID_ACCESS_LEAK_SHARED_REF_ON_DESTROYED_THREAD
//#define FALLBACK_WAKEONADDRESS_SUPPORTS_NONEXACT_MATCHING

namespace Aurora::Threading
{
#if defined(HACK_NO_INVALID_ACCESS_LEAK_SHARED_REF_ON_DESTROYED_THREAD)
    static thread_local AuSPtr<WaitEntry> tlsWaitEntry = AuMakeSharedPanic<WaitEntry>();
#else
    static thread_local WaitEntry tlsWaitEntry;
#endif

    static ProcessWaitContainer gProcessWaitables;
    static int gShouldSpinOnlyInCPU = 1; // TODO: havent decided
    
    template<typename T>
    static void DoSpinLockOnVar(T *uPointer)
    {
        if (gShouldSpinOnlyInCPU == 0)
        {
            while (!Primitives::DoTryIf([&]()
            {
                return AuAtomicTestAndSet(uPointer, 0) == 0;
            }))
            {

            }
        }
        else if (gShouldSpinOnlyInCPU == 1)
        {
            while (!Primitives::DoTryIf([&]()
            {
                return AuAtomicTestAndSet(uPointer, 0) == 0;
            }))
            {
                ContextYield();
            }
        }
        else if (gShouldSpinOnlyInCPU == 2)
        {
            while (AuAtomicTestAndSet(uPointer, 0))
            {
                while (*uPointer)
                {
                    ContextYield();
                }
            }
        }
        else
        {
            SysUnreachable();
        }
    }

    bool WaitEntry::TryAcquire(const void *pAddress, AuUInt8 uSize)
    {
        DoSpinLockOnVar(&this->uAtomic);

        //AU_LOCK_GUARD(this->mutex);

        if (this->pAddress)
        {
            this->uAtomic = 0;
            return false;
        }

        this->pAddress = pAddress;
        this->uSize = uSize;
        this->uAtomic = 0;
        return true;
    }

    void WaitEntry::Release()
    {
        AuResetMember(this->uSize);
        AuResetMember(this->pAddress);
    }
    
    WaitEntry::WaitEntry()
    {

    }

    WaitEntry::~WaitEntry()
    {
        this->Release();
    }

    bool WaitEntry::SleepOn(WaitState &state)
    {
        AU_LOCK_GUARD(this->mutex);
        
        if (state.qwNanosecondsAbs)
        {
            if (!WaitBuffer::From(this->pAddress, this->uSize).Compare(state))
            {
                return true;
            }

            auto uNow = AuTime::SteadyClockNS();
            auto uEndTime = state.qwNanosecondsAbs.value();

            while (uNow < uEndTime)
            {
                if (!WaitBuffer::From(this->pAddress, this->uSize).Compare(state))
                {
                    return true;
                }

                auto uTimeRemNS = uEndTime - uNow;

            #if defined(AURORA_PLATFORM_WIN32)
                Win32DropSchedulerResolution();
            #endif
                
                this->variable.WaitForSignalNsEx(&this->mutex, uTimeRemNS);

                uNow = AuTime::SteadyClockNS();
            }

            return !WaitBuffer::From(this->pAddress, this->uSize).Compare(state);
        }
        else
        {
            while (WaitBuffer::From(this->pAddress, this->uSize).Compare(state))
            {
                this->variable.WaitForSignalNsEx(&this->mutex, 0);
            }

            return true;
        }

        return false;
    }

    bool WaitEntry::TryWake(const void *pAddress)
    {
        DoSpinLockOnVar(&this->uAtomic);

        auto bRet = TryWakeNoLock(pAddress);
        if (!bRet)
        {
            this->uAtomic = 0;
        }

        return bRet;
    }

    bool WaitEntry::TryWakeNoLockNoReallyNoLock(const void *pAddress)
    {
    #if defined(FALLBACK_WAKEONADDRESS_SUPPORTS_NONEXACT_MATCHING)
        if (AuReinterpretCast<const char *>(this->pAddress) > AuReinterpretCast<const char *>(pAddress) ||
            AuReinterpretCast<const char *>(this->pAddress) + this->uSize <= AuReinterpretCast<const char *>(pAddress))
        {
            return false;
        }
    #else
        if (this->pAddress != pAddress)
        {
            return false;
        }
    #endif

        this->uAtomic = 0;
        this->variable.Signal();
        return true;
    }

    bool WaitEntry::TryWakeNoLock(const void *pAddress)
    {
    #if defined(FALLBACK_WAKEONADDRESS_SUPPORTS_NONEXACT_MATCHING)
        if (AuReinterpretCast<const char *>(this->pAddress) > AuReinterpretCast<const char *>(pAddress) ||
          AuReinterpretCast<const char *>(this->pAddress) + this->uSize <= AuReinterpretCast<const char *>(pAddress))
        {
            return false;
        }
    #else
        if (this->pAddress != pAddress)
        {
            return false;
        }
    #endif

        AU_LOCK_GUARD(this->mutex);
        this->uAtomic = 0;
        this->variable.Signal();
        return true;
    }

    WaitBuffer WaitBuffer::From(const void *pBuf, AuUInt8 uSize)
    {
        WaitBuffer wait;
        AuMemcpy(wait.buffer, pBuf, uSize);
        wait.uSize = uSize;
        return AuMove(wait);
    }

    bool WaitBuffer::Compare(const void *pBuf)
    {
        return AuMemcmp(this->buffer, pBuf, this->uSize) == 0;
    }

    bool WaitBuffer::Compare(WaitState &state)
    {
        if (!state.uDownsizeMask)
        {
            return AuMemcmp(this->buffer, state.compare.buffer, AuMin(this->uSize, state.compare.uSize)) == 0;
        }
        else
        {
            auto uMask     = state.uDownsizeMask.value();

            auto &uSrcWord = *AuReinterpretCast<AuUInt32 *>(this->buffer);
            auto &uCmpWord = *AuReinterpretCast<AuUInt32 *>(state.compare.buffer);

            return (uSrcWord & uMask) == (uCmpWord & uMask);
        }
    }

    WaitEntry *ProcessWaitNodeContainer::WaitBufferFrom(const void *pAddress, AuUInt8 uSize)
    {
    #if defined(HACK_NO_INVALID_ACCESS_LEAK_SHARED_REF_ON_DESTROYED_THREAD)
        auto pReturn = tlsWaitEntry.get();
    #else
        auto pReturn = &tlsWaitEntry;
    #endif

        pReturn->bReleaseOnWake = true;

        pReturn->pAddress = pAddress;
        pReturn->uSize    = uSize;
        pReturn->uAtomic  = 0;

        {
            Lock();
            if (auto pLoadFromMemory = this->waitList.pHead)
            {
                pReturn->pNext = pLoadFromMemory;
                pLoadFromMemory->pBefore = pReturn;
            }
            else
            {
                this->waitList.pTail = pReturn;
            }
            this->waitList.pHead = pReturn;
            Unlock();
        }

        return pReturn;
    }

    template <typename T>
    bool ProcessWaitNodeContainer::IterateWake(T callback)
    {
        bool bRetStatus { true };

        Lock();
        {
            // FIFO
            auto pCurrentHead = this->waitList.pTail;
            decltype(pCurrentHead) pLast {};
            while (pCurrentHead)
            {
                AU_LOCK_GUARD(pCurrentHead->mutex);

                auto [bCont, bRemove] = callback(*pCurrentHead);

                if (bRemove)
                {
                    if (pLast)
                    {
                        pLast->pNext = pCurrentHead->pNext;
                    }
                    
                    if (this->waitList.pHead == pCurrentHead)
                    {
                        this->waitList.pHead = pCurrentHead->pNext;
                    }

                    if (pCurrentHead->pNext)
                    {
                        pCurrentHead->pNext->pBefore = pCurrentHead->pBefore;
                    }

                    if (this->waitList.pTail == pCurrentHead)
                    {
                        this->waitList.pTail = pCurrentHead->pBefore;
                    }
                }

                if (!bCont)
                {
                    bRetStatus = false;
                    break;
                }

                pLast = pCurrentHead;
                pCurrentHead = pCurrentHead->pBefore;
            }
        }
        Unlock();

        return bRetStatus;
    }

    void ProcessWaitNodeContainer::Lock()
    {
        DoSpinLockOnVar(&this->uAtomic);
    }

    void ProcessWaitNodeContainer::Unlock()
    {
        this->uAtomic = 0;
    }

    WaitEntry *ProcessWaitContainer::WaitBufferFrom(const void *pAddress, AuUInt8 uSize)
    {
    #if defined(FALLBACK_WAKEONADDRESS_SUPPORTS_NONEXACT_MATCHING)
        auto pAddressIDC = (void *)(AuUInt(pAddress) & ~(8 - 1));
    #else
        auto pAddressIDC = pAddress;
    #endif
        return this->list[AuHashCode(pAddressIDC) % AuArraySize(this->list)].WaitBufferFrom(pAddress, uSize);
    }

    template <typename T>
    bool ProcessWaitContainer::IterateWake(const void *pAddress, T callback)
    {
    #if defined(FALLBACK_WAKEONADDRESS_SUPPORTS_NONEXACT_MATCHING)
        auto pAddressIDC = (void *)(AuUInt(pAddress) & ~(8 - 1));
    #else
        auto pAddressIDC = pAddress;
    #endif
        return this->list[AuHashCode(pAddressIDC) % AuArraySize(this->list)].IterateWake(callback);
    }

    bool IsNativeWaitOnSupported()
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        return pWaitOnAddress &&
            AuSwInfo::IsWindows8Point1OrGreater();
    #elif defined(AURORA_PLATFORM_LINUX)
        return true;
    #else
        return false;
    #endif
    }

    AUKN_SYM bool IsWaitOnRecommended()
    {
        static AuOptionalEx<bool> gIsWaitOnRecommendedCache {};

        if (gIsWaitOnRecommendedCache)
        {
            return gIsWaitOnRecommendedCache.value();
        }

        if (Primitives::ThrdCfg::gPreferEmulatedWakeOnAddress)
        {
            return false;
        }

        bool bState = IsNativeWaitOnSupported();
        gIsWaitOnRecommendedCache = bState;
        return bState;
    }

    /// @deprecated
    AUKN_SYM const AuList<AuUInt8> &GetValidWordSizes()
    {
        static const AuList<AuUInt8> kArray =
    #if defined(AURORA_IS_MODERNNT_DERIVED)
            { 1, 2, 4, 8 };
    #else
            { 4 };
    #endif
        return kArray;
    }

    bool WaitOnAddressWide(const void *pTargetAddress,
                           const void *pCompareAddress,
                           AuUInt8 uWordSize,
                           AuOptional<AuUInt64> qwNanoseconds,
                           AuOptional<AuUInt64> qwNanosecondsAbs,
                           bool bOSSupportsWait
    )
    {
        WaitState state;
        SysAssertDbg(uWordSize <= 8);
        auto pWaitEntry = gProcessWaitables.WaitBufferFrom(pTargetAddress, uWordSize);
        state.compare = WaitBuffer::From(pCompareAddress, uWordSize);

        if (qwNanoseconds)
        {
            state.qwNanosecondsAbs = AuTime::SteadyClockNS() + qwNanoseconds.value();
        }
        else if (qwNanosecondsAbs)
        {
            state.qwNanosecondsAbs = qwNanosecondsAbs.value();
        }

    #if defined(HACK_NO_INVALID_ACCESS_LEAK_SHARED_REF_ON_DESTROYED_THREAD)
        auto pTempHoldMe = tlsWaitEntry;
    #endif

        auto bResult = pWaitEntry->SleepOn(state);

    #if defined(HACK_NO_INVALID_ACCESS_LEAK_SHARED_REF_ON_DESTROYED_THREAD)
        pTempHoldMe.reset();
    #endif
        return bResult;
    }

    AuTuple<const void *, AuUInt8, AuOptionalEx<AuUInt32>> DecodeAddress(const void *pAddress,
                                                                         AuUInt32 uWordSize)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        return AuMakeTuple(pAddress, 0, AuOptionalEx<AuUInt32> {});
    #endif

        auto pRounded = AuPageRound(AuUInt(pAddress), AuUInt(4));
        auto uDelta   = (AuUInt)pAddress - pRounded;

        if (uWordSize == 8)
        {
            return AuMakeTuple((const void *)pRounded, uDelta, 0xFFFFFFFF);
        }

        AuUInt32 uSizeMask = (1ull << (uWordSize * 8)) - 1ull;

        switch (uDelta)
        {
        case 0:
            return AuMakeTuple(pAddress, 0, 0xFFFFFFFF & (uSizeMask << 0));
        case 1:
            return AuMakeTuple(pAddress, 1, 0xFFFFFF00 & (uSizeMask << 8));
        case 2:
            return AuMakeTuple(pAddress, 2, 0xFFFF0000 & (uSizeMask << 16));
        case 3:
            return AuMakeTuple(pAddress, 3, 0xFF000000 & (uSizeMask << 24));
        default:
            SysPanic("Invalid Branch");
        }
    }

    static bool RunOSWaitOnAddressNoTimed(const void *pTargetAddress,
                                          const void *pCompareAddress,
                                          AuUInt8 uWordSize)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        if (pRtlWaitOnAddress)
        {
            return pRtlWaitOnAddress((void *)pTargetAddress, (void *)pCompareAddress, uWordSize, nullptr);
        }
        else
        {
            return pWaitOnAddress((void *)pTargetAddress, (void *)pCompareAddress, uWordSize, INFINITE);
        }
    #endif

    #if defined(AURORA_IS_LINUX_DERIVED)
        int ret {};

    #if defined(AU_CPU_ENDIAN_BIG)
        if (uWordSize == 8)
        {
            pTargetAddress = AuReinterpretCast<const char *>(pTargetAddress) + 4;
            pCompareAddress = AuReinterpretCast<const char *>(pCompareAddress) + 4;
        }
    #endif

        auto uCurrent = *(AuUInt32 *)pCompareAddress;
        auto expect = WaitBuffer::From(pCompareAddress, uWordSize);

        do
        {
            ret = futex_wait((AuUInt32 *)pTargetAddress, uCurrent, nullptr);

            if (ret == 0)
            {
                continue;
            }

            if (ret == -EAGAIN)
            {
                continue;
            }

            if (ret == -ETIMEDOUT)
            {
                return false;
            }
        }
        while (ret == -EINTR);

        return !expect.Compare(pTargetAddress);
    #endif

        return false;
    }

    static bool RunOSWaitOnAddressTimed(const void *pTargetAddress,
                                        const void *pCompareAddress,
                                        AuUInt8 uWordSize,
                                        AuUInt64 uAbsTimeSteadyClock,
                                        AuUInt64 uRelativeNanoseconds,
                                        AuOptional<AuUInt64> uAbsTimeAltClock /* hint */)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        
        if (pRtlWaitOnAddress)
        {
            auto expect = WaitBuffer::From(pCompareAddress, uWordSize);

            AuUInt64 uNow {};
            while (uAbsTimeSteadyClock ?
                   (uAbsTimeSteadyClock > (uNow = AuTime::SteadyClockNS())) :
                   true)
            {
                LARGE_INTEGER word {};

                if (uAbsTimeAltClock)
                {
                    word.QuadPart = AuTime::ConvertTimestampNs(uAbsTimeAltClock.value());
                }
                else if (uAbsTimeSteadyClock)
                {
                    if (uAbsTimeSteadyClock <= uNow)
                    {
                        return !WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress);
                    }

                    word.QuadPart = -(AuInt64(uAbsTimeSteadyClock - uNow) / 100ull);

                    if (!word.QuadPart)
                    {
                        word.QuadPart = 1;
                    }
                }
                
                if (expect.Compare(pTargetAddress))
                {
                    pRtlWaitOnAddress(pTargetAddress, pCompareAddress, uWordSize, &word);
                    if (!expect.Compare(pTargetAddress))
                    {
                        return true;
                    }
                    else if (!uAbsTimeSteadyClock)
                    {
                        return false;
                    }
                }
                else
                {
                    return true;
                }
            }

            return false;
        }
        else
        {
            // ~~some paths might miss the uRelativeNanoseconds, like cas loops.~~
            // most paths will now skimp on the relative values
            if (uAbsTimeSteadyClock && !uRelativeNanoseconds)
            {
                AuInt64 iDelta = uAbsTimeSteadyClock;
                iDelta -= AuTime::SteadyClockNS();

                if (iDelta <= 0)
                {
                    return !WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress);
                }

                uRelativeNanoseconds = iDelta;
            }

            auto uMaxSwitches = gRuntimeConfig.threadingConfig.uUWPNanosecondEmulationMaxYields;
            auto bUWPNanosecondEmulationCheckFirst = Primitives::ThrdCfg::gUWPNanosecondEmulationCheckFirst;

            // LockN(<1MS) on a platform without that resolution of yielding... damn
            auto uMS = AuNSToMS<AuUInt32>(uRelativeNanoseconds);
            if (!uMS)
            {
                // take a copy
                auto expect = WaitBuffer::From(pCompareAddress, uWordSize);

                // first: cpu spin to avoid the kernel all together
                if (TryWaitOnAddress((void *)pTargetAddress, (void *)pCompareAddress, uWordSize))
                {
                    return true;
                }

                // second: yield
                unsigned uLimit {};
                do
                {
                    if (!expect.Compare(pTargetAddress))
                    {
                        break;
                    }
                 
                    AuThreading::ContextYield();

                    if (bUWPNanosecondEmulationCheckFirst)
                    {
                        if (uLimit++ > uMaxSwitches)
                        {
                            break;
                        }
                    }
                }
                while (uAbsTimeSteadyClock > AuTime::SteadyClockNS()); // ...until times up
            }
            else // high level lock function was called with ms scale resolution
            {
                // first: wait on the address with an ms scale timeout
                (void)pWaitOnAddress((void *)pTargetAddress, (void *)pCompareAddress, uWordSize, uMS);

                // never trust the error value/status provided by wait addresses - instead, do a quick compare
                if (!WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress))
                {
                    // best case: we woke up during the ms-res waitonaddress
                    return true;
                }

                // attempt to yield again, potentially context switching a few times to hit any NS remainder 
                AuUInt64 uNow {};
                unsigned uLimit {};
                while (uAbsTimeSteadyClock > (uNow = AuTime::SteadyClockNS()))
                {
                    uMS = AuNSToMS<AuUInt32>(uAbsTimeSteadyClock - uNow);

                    if (Primitives::DoTryIf([=]()
                    {
                        return !WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress);
                    }))
                    {
                        // hit it within the span of 1 << SpinLoopPowerA SMT stalls
                        return true;
                    }

                    if (!uMS)
                    {
                        // burn off any remainder cycles by switching contexts (this isnt a very long time usually)
                        if (uLimit++ < uMaxSwitches)
                        {
                            AuThreading::ContextYield();
                        }
                        else
                        {
                            // do not burn the cpu to meet the timeout. we'll just undershoot.
                            return false;
                        }
                    }
                    else
                    {
                        (void)pWaitOnAddress((void *)pTargetAddress, (void *)pCompareAddress, uWordSize, uMS);
                    }
                }
            }
        }

    #endif

    #if defined(AURORA_IS_LINUX_DERIVED)
        int ret {};

    #if defined(AU_CPU_ENDIAN_BIG)
        if (uWordSize == 8)
        {
            pTargetAddress = AuReinterpretCast<const char *>(pTargetAddress) + 4;
            pCompareAddress = AuReinterpretCast<const char *>(pCompareAddress) + 4;
        }
    #endif

        auto uCurrent = *(AuUInt32 *)pCompareAddress;

        struct timespec tspec;
        Time::monoabsns2ts(&tspec, uAbsTimeAltClock ? uAbsTimeAltClock.value() : uAbsTimeSteadyClock);

        do
        {
            ret = futex_wait((AuUInt32 *)pTargetAddress, uCurrent, &tspec);

            if (ret == 0)
            {
                continue;
            }

            if (ret == -EAGAIN)
            {
                continue;
            }

            if (ret == -ETIMEDOUT)
            {
                return false;
            }
        }
        while (ret == -EINTR);

    #endif

        return !WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress);
    }

    static void RunOSWaitOnAddressNoTimedNoErrors(const void *pTargetAddress,
                                                  const void *pCompareAddress,
                                                  WaitState &state)
    {
        while (WaitBuffer::From(pTargetAddress, state.uWordSize).Compare(state))
        {
            if (!RunOSWaitOnAddressNoTimed(pTargetAddress, pCompareAddress, state.uWordSize))
            {
                AuThreading::ContextYield();
            }
        }
    }

    static bool RunOSWaitOnAddressTimedSteady(const void *pTargetAddress,
                                             const void *pCompareAddress,
                                             WaitState &state)
    {
        if (!WaitBuffer::From(pTargetAddress, state.uWordSize).Compare(state))
        {
            return true;
        }

        (void)RunOSWaitOnAddressTimed(pTargetAddress, pCompareAddress, state.uWordSize, state.qwNanosecondsAbs.value(), { }, { });
        return !WaitBuffer::From(pTargetAddress, state.uWordSize).Compare(state);
    }

    static void RunOSWakeNOnAddress(const void *pAddress,
                                    AuUInt32 dwCount)
    {
    #if defined(AURORA_IS_LINUX_DERIVED)
        futex_wake((AuUInt32 *)pAddress, dwCount);
    #endif

    #if defined(AURORA_IS_MODERNNT_DERIVED)
        if (pRtlWakeAddressSingle)
        {
            if (dwCount < 6)
            {
                for (AuUInt i = 0; i < dwCount; i++)
                {
                    pRtlWakeAddressSingle((void *)pAddress);
                }
            }
            else
            {
                pRtlWakeByAddressAll((void *)pAddress);
            }
        }
        else
        {
            for (AuUInt i = 0; i < dwCount; i++)
            {
                pWakeByAddressSingle((void *)pAddress);
            }
        }
    #endif
    }

    static void RunOSWakeAllOnAddress(const void *pAddress)
    {
    #if defined(AURORA_IS_LINUX_DERIVED)
        futex_wake((AuUInt32 *)pAddress, INT_MAX);
    #endif

    #if defined(AURORA_IS_MODERNNT_DERIVED)
        if (pRtlWakeByAddressAll)
        {
            pRtlWakeByAddressAll((void *)pAddress);
        }
        else
        {
            pWakeByAddressAll((void *)pAddress);
        }
    #endif
    }
    
    // Windows 8+ thread primitives might use me instead of the public API
    // it does work on Linux and Windows 8+
    // it does not, however, work on emulated platforms
    // this is intentional
    bool InternalLTSWaitOnAddressHighRes(const void *pTargetAddress,
                                         const void *pCompareAddress,
                                         AuUInt8 uWordSize,
                                         AuUInt64 qwNanosecondsAbs)
    {
        auto [pWaitAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, uWordSize);
        auto pCompareAddress2 = AuReinterpretCast<const char *>(pCompareAddress) - uDelta;

        WaitState state;
        state.uDownsizeMask = uMask;
        state.compare = uMask ?
            WaitBuffer::From(pCompareAddress2, 4) :
            WaitBuffer::From(pCompareAddress2, uWordSize);
        state.uWordSize = uMask ? 4 : uWordSize;

        if (!qwNanosecondsAbs)
        {
            RunOSWaitOnAddressNoTimedNoErrors(pWaitAddress, pCompareAddress2, state);
            return true;
        }
        else
        {
            state.qwNanosecondsAbs = qwNanosecondsAbs;
            return RunOSWaitOnAddressTimedSteady(pWaitAddress, pCompareAddress2, state);
        }
    }

    void InternalLTSWakeAll(const void *pTargetAddress)
    {
        auto [pWakeAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, 1);
        RunOSWakeAllOnAddress(pWakeAddress);
    }

    void InternalLTSWakeOne(const void *pTargetAddress)
    {
        auto [pWakeAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, 1);
        RunOSWakeNOnAddress(pWakeAddress, uDelta ? INT_MAX : 1);
    }

    void InternalLTSWakeCount(const void *pTargetAddress, AuUInt32 uCount)
    {
        auto [pWakeAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, 1);
        RunOSWakeNOnAddress(pWakeAddress, uDelta ? INT_MAX : uCount);
    }

    AUKN_SYM bool WaitOnAddress(const void *pTargetAddress,
                                const void *pCompareAddress,
                                AuUInt8 uWordSize,
                                AuUInt64 qwNanoseconds)
    {
        bool bWaitOnAddress = IsWaitOnRecommended();
        if (bWaitOnAddress)
        {
            auto [pWaitAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, uWordSize);
            auto pCompareAddress2 = AuReinterpretCast<const char *>(pCompareAddress) - uDelta;
            
            WaitState state;
            state.uDownsizeMask = uMask;
            state.compare = uMask ?
                WaitBuffer::From(pCompareAddress2, 4) :
                WaitBuffer::From(pCompareAddress2, uWordSize);
            state.uWordSize = uMask ? 4 : uWordSize;

            if (!qwNanoseconds)
            {
                RunOSWaitOnAddressNoTimedNoErrors(pWaitAddress, pCompareAddress2, state);
                return true;
            }
            else
            {
                state.qwNanosecondsAbs = qwNanoseconds + AuTime::SteadyClockNS();
                return RunOSWaitOnAddressTimedSteady(pWaitAddress, pCompareAddress2, state);
            }
        }
        else
        {
            if (Primitives::ThrdCfg::gPreferWaitOnAddressAlwaysSpin)
            {
                if (TryWaitOnAddress(pTargetAddress, pCompareAddress, uWordSize))
                {
                    return true;
                }
            }

            return WaitOnAddressWide(pTargetAddress, pCompareAddress, uWordSize, qwNanoseconds, {}, false);
        }

        return false;
    }

    AUKN_SYM bool TryWaitOnAddress(const void *pTargetAddress,
                                   const void *pCompareAddress,
                                   AuUInt8 uWordSize)
    {
        return Primitives::DoTryIf([=]()
        {
            return !WaitBuffer::From(pCompareAddress, uWordSize).Compare(pTargetAddress);
        });
    }

    AUKN_SYM void WakeNOnAddress(const void *pTargetAddress,
                                 AuUInt8 uNMaximumThreads)
    {
        if (IsWaitOnRecommended())
        {
            auto [pWakeAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, 1);
            RunOSWakeNOnAddress(pWakeAddress, uDelta ? INT_MAX : uNMaximumThreads);
        }
        else
        {
            (void)gProcessWaitables.IterateWake(pTargetAddress, [&](WaitEntry &entry) -> AuPair<bool, bool>
            {
                if (!uNMaximumThreads)
                {
                    return AuMakePair(false, false);
                }

                bool bWake {};
                if (entry.TryWakeNoLockNoReallyNoLock(pTargetAddress))
                {
                    bWake = true;
                    uNMaximumThreads--;
                }

                bool bCont = uNMaximumThreads != 0;
                return AuMakePair(bCont, bWake);
            });
        }
    }

    AUKN_SYM void WakeOnAddress(const void *pTargetAddress)
    {
        WakeNOnAddress(pTargetAddress, 1);
    }

    AUKN_SYM void WakeAllOnAddress(const void *pTargetAddress)
    {
        if (IsWaitOnRecommended())
        {
            auto [pWakeAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, 1);
            RunOSWakeAllOnAddress(pWakeAddress);
        }
        else
        {
            (void)gProcessWaitables.IterateWake(pTargetAddress, [&](WaitEntry &entry) -> AuPair<bool, bool>
            {
                return AuMakePair(true, entry.TryWakeNoLockNoReallyNoLock(pTargetAddress));
            });
        }
    }

    AUKN_SYM bool WaitOnAddressSteady(const void *pTargetAddress,
                                      const void *pCompareAddress,
                                      AuUInt8 uWordSize,
                                      AuUInt64 qwNanoseconds)
    {
        bool bWaitOnAddress = IsWaitOnRecommended();
        if (bWaitOnAddress)
        {
            auto [pWaitAddress, uDelta, uMask] = DecodeAddress(pTargetAddress, uWordSize);
            auto pCompareAddress2 = AuReinterpretCast<const char *>(pCompareAddress) - uDelta;

            WaitState state;
            state.uDownsizeMask = uMask;
            state.compare = uMask ?
                WaitBuffer::From(pCompareAddress2, 4) :
                WaitBuffer::From(pCompareAddress2, uWordSize);
            state.uWordSize = uMask ? 4 : uWordSize;

            if (!qwNanoseconds)
            {
                RunOSWaitOnAddressNoTimedNoErrors(pWaitAddress, pCompareAddress2, state);
                return true;
            }
            else
            {
                state.qwNanosecondsAbs = qwNanoseconds;
                return RunOSWaitOnAddressTimedSteady(pWaitAddress, pCompareAddress2, state);
            }
        }
        else
        {
            if (Primitives::ThrdCfg::gPreferWaitOnAddressAlwaysSpin)
            {
                if (TryWaitOnAddress(pTargetAddress, pCompareAddress, uWordSize))
                {
                    return true;
                }
            }

            return WaitOnAddressWide(pTargetAddress, pCompareAddress, uWordSize, {}, qwNanoseconds, false);
        }

        return false;
    }
}