AuroraRuntime/Source/Time/AuClock.cpp

/***
    Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.

    File: AuClock.cpp
    Date: 2021-6-13
    Author: Reece
***/
#include <Source/RuntimeInternal.hpp>
#include "AuClock.hpp"
#include "Time.hpp"

#if defined(AURORA_IS_POSIX_DERIVED)

    #if defined(AURORA_IS_XNU_DERIVED)
        #include <mach/mach_time.h>

        AUKN_SYM unsigned long long _NTLikeQueryFrequency()
        {
            static unsigned long long uFrequency {};
            static AuInitOnce gInitOnce;
            gInitOnce.Call([]
            {
                mach_timebase_info_data_t timebase;
                mach_timebase_info(&timebase);
                uFrequency = (double(timebase.denom) * 1'000'000'000.0) / double(timebase.numer);
            });
            return uFrequency;
        }

        AUKN_SYM unsigned long long _NTLikeQueryCounter()
        {
            return mach_continuous_time();
        }

        #define GIB__GetSteadyTimeNS
    #endif

    #include <sys/resource.h>

#elif defined(AURORA_IS_MODERNNT_DERIVED)

    #include <SWInfo/AuSWInfo.hpp>

    static AuUInt8 _gNTClockMode {};
    static long long(__cdecl *_gNTQuery)();
    static long long(__cdecl *_gNTFreq)();

    struct NTSystemTime
    {
        NTSystemTime(volatile NTSystemTime &dumb)
        {
            this->LowPart = dumb.LowPart;
            this->High1Time = dumb.High1Time;
            this->High2Time = dumb.High2Time;
        }

        unsigned long LowPart;
        long High1Time;
        long High2Time;
    };

    struct NTQPCoefficients
    {
        union
        {
            UCHAR TscQpcData;
            struct
            {
                UCHAR TscQpcEnabled : 1;
                UCHAR TscQpcSpareFlag : 1;
                UCHAR TscQpcShift : 6;
            };
        };
    };
    
    static unsigned long long  _NT3_Query_Frequency();
    static void _NTSetFallbackClock();

    static void *      kKUSERShardDataOffset = AuReinterpretCast<void *>(0x7ffe0000);
    static auto *      kKInterruptTime = AuReinterpretCast<volatile NTSystemTime *>(AuReinterpretCast<AuUInt8 *>(kKUSERShardDataOffset) + 8);
    static auto *      kKQPCData      = AuReinterpretCast<NTQPCoefficients *>(AuReinterpretCast<AuUInt8 *>(kKUSERShardDataOffset) + 0x2ED);
    static AuUInt64    _gNTTimeShift {};
    static AuUInt64    _gNTTimeBias {};
    static AuUInt64    _gNTTimeFreq { _NT3_Query_Frequency() };

    static unsigned long long  _NT3_Query_Frequency()
    {
        return 1'0000'000ull;
    }

    static void _NT6_1_Init()
    {
        NTQPCoefficients now;
        now.TscQpcData = AuAtomicLoad(&kKQPCData->TscQpcData);

        if (!now.TscQpcEnabled)
        {
            _NTSetFallbackClock();
            return;
        }

        _gNTTimeShift = now.TscQpcShift;
        _gNTTimeBias = *AuReinterpretCast<ULONGLONG *>(AuReinterpretCast<AuUInt8 *>(kKUSERShardDataOffset) + 0x3B8);
        long long uvalue {};
        SysAssert(Aurora::pQueryPerformanceFrequency &&
                  Aurora::pQueryPerformanceFrequency(&uvalue), "no perf frequency");
        _gNTTimeFreq = uvalue;
        _gNTClockMode = 1;
    }

    static long long _NT6_1_Query_Frequency()
    {
        return _gNTTimeFreq;
    }

    static unsigned long long _NT6_1_Query_Counter()
    {

    #if defined(AURORA_ARCH_X64) || defined(AURORA_ARCH_X86)
        AuUInt64 uTimeNow = __rdtsc();
        uTimeNow += _gNTTimeBias;
        uTimeNow >>= _gNTTimeShift;
        return uTimeNow;
    #endif

        return 0;
    }

    static long long _NT3_Query_Counter()
    {
        auto interruptTime = *kKInterruptTime;
        return (AuUInt64(kKInterruptTime->High1Time) << 32ull) | interruptTime.LowPart;
    }

    static void _NTSetFallbackClock()
    {
        _gNTFreq = []()
        {
            long long uvalue {};
            SysAssert(Aurora::pQueryPerformanceFrequency &&
                      Aurora::pQueryPerformanceFrequency(&uvalue), "no perf frequency");
            return uvalue;
        };
        _gNTQuery = []()
        {
            long long uvalue {};
            SysAssert(Aurora::pQueryPerformanceCounter &&
                      Aurora::pQueryPerformanceCounter(&uvalue), "no perf counter");
            return uvalue;
        };
        _gNTClockMode = 2;
        _gNTTimeFreq = _gNTFreq();
    }

    static void _NTDoClockInit()
    {
        static AuInitOnce gInitOnce;
        gInitOnce.Call([]
        {
            Aurora::InitNTAddressesForClock();
            AuSwInfo::InitSwInfoEarly();
         
            {
                // Might be useful for real-time applications under busted motherboards and hypervisors.
                // Should the QPC fastpath be invalidated, and the kernel hit each time because of NT HAL retardation,
                //  this can be used to enforce a userland-only clock query, when Windows would otherwise resort to a 
                //  system call.
                // Pretty much every Win32 environment and platform should support this
                wchar_t buffer[64];
                if (GetEnvironmentVariableW(L"AURORA_STEADY_TIME_SHID_CPU_XP_MODE", buffer, AuArraySize(buffer)))
                {
                    _gNTClockMode = 0;
                    return;
                }
            }

            if constexpr (AuBuild::kCurrentPlatform == AuBuild::EPlatform::ePlatformWin32)
            {
                if (!AuSwInfo::IsWindows7OrGreater())
                {
                    _gNTClockMode = 0;
                }
                else if (AuSwInfo::IsWindows7Any())
                {
                    _NT6_1_Init();
                }
                else //if (AuSwInfo::IsWindows8OrGreater())
                {
                    _NTSetFallbackClock();
                }
            }
            else
            {
                _gNTClockMode = 2;
                _gNTFreq = _Query_perf_frequency;
                _gNTQuery = _Query_perf_counter;
            }
        });
    }

    AUKN_SYM unsigned long long _NTLikeQueryFrequency()
    {
        _NTDoClockInit();

        switch (_gNTClockMode)
        {
        case 0:
            return _NT3_Query_Frequency();
        case 1:
            return _NT6_1_Query_Frequency();
        case 2:
        #if 0
            return _gNTFreq();
        #else
            return _gNTTimeFreq;
        #endif
        default:
            return 0;
        }
    }

    AUKN_SYM unsigned long long _NTLikeQueryCounter()
    {
        switch (_gNTClockMode)
        {
        case 0:
            return _NT3_Query_Counter();
        case 1:
            return _NT6_1_Query_Counter();
        case 2:
            return _gNTQuery();
        default:
            return 0;
        }
    }

    // benchmarking: https://github.com/microsoft/STL/issues/2085
    #define GIB__GetSteadyTimeNS
    // moved: _GetSteadyTimeNS

    // ~3.0741 seconds

    // using high_res_clock = std::chrono::high_resolution_clock;
    // ~6.07 seconds

    // holy fuck, we're keeping this
    // ~2x improvement

#else

    using steady_clock = std::chrono::steady_clock;

#endif

#if defined(GIB__GetSteadyTimeNS)
    static AuUInt64 _GetSteadyTimeNS()
    {
        static const long long gFreq = _NTLikeQueryFrequency();
        const long long uCounter = _NTLikeQueryCounter();
    
        if (gFreq == 10000000)
        {
            return uCounter * 100;
        }
        else if (gFreq == 1000000)
        {
            return uCounter * 1000;
        }
        else if (gFreq == 100000)
        {
            return uCounter * 10000;
        }
        else if (gFreq == 100000000)
        {
            return uCounter * 10;
        }
        else if (gFreq == 1000000000)
        {
            return uCounter;
        }
        else
        {
            // 6 branches: the default threshold for most jit and language compiler backends to decide to pick a jump table, if the values were in a close range
            // otherwise, back to a tree of paths. either way, im sure 6 if elses are faster than grug math with large numbers, modulus, division, and multiplication
            const long long uWhole = (uCounter / gFreq) * 1'000'000'000ull;
            const long long uPart  = (uCounter % gFreq) * 1'000'000'000ull / gFreq;
            return uWhole + uPart;
        }
    }
#endif

using sys_clock = std::chrono::system_clock; // more stds to remove

sys_clock::duration __NormalizeEpoch(sys_clock::duration sysEpoch);

static AuInt64 _CurrentClock()
{
    return __NormalizeEpoch(sys_clock::now().time_since_epoch()).count();
}

static AuInt64 _CurrentClockMS()
{
    return std::chrono::duration_cast<std::chrono::milliseconds>(__NormalizeEpoch(sys_clock::now().time_since_epoch())).count();
}

static AuInt64 _CurrentClockNS()
{
    return std::chrono::duration_cast<std::chrono::nanoseconds>(__NormalizeEpoch(sys_clock::now().time_since_epoch())).count();
}

namespace Aurora::Time
{
    AUKN_SYM AuInt64 CurrentClock()
    {
        return _CurrentClock();
    }

    AUKN_SYM AuInt64 CurrentClockMS()
    {
        return _CurrentClockMS();
    }

    AUKN_SYM AuInt64 CurrentClockNS()
    {
        return _CurrentClockNS();
    }

    AUKN_SYM AuUInt64 SteadyClock()
    {
    #if defined(GIB__GetSteadyTimeNS)
        return _NTLikeQueryCounter();
    #else
        return SteadyClockNS() / (1000000000ull / SteadyClockFrequency());
    #endif
    }

    AUKN_SYM AuUInt64 SteadyClockMS()
    {
    #if defined(GIB__GetSteadyTimeNS)
        return AuNSToMS<AuUInt64>(_GetSteadyTimeNS());
    #elif defined(AURORA_IS_POSIX_DERIVED)
        ::timespec spec {};
        if (::clock_gettime(CLOCK_MONOTONIC, &spec) == 0)
        {
            return AuSToMS<AuUInt64>(spec.tv_sec) + AuNSToMS<AuUInt64>(spec.tv_nsec);
        }
        else
        {
            return 0;
        }
    #else
        return std::chrono::duration_cast<std::chrono::milliseconds>(steady_clock::now().time_since_epoch()).count();
    #endif
    }

    AUKN_SYM AuUInt64 SteadyClockNS()
    {
    #if defined(GIB__GetSteadyTimeNS)
        return _GetSteadyTimeNS();
    #elif defined(AURORA_IS_POSIX_DERIVED)
        ::timespec spec {};
        if (::clock_gettime(CLOCK_MONOTONIC, &spec) == 0)
        {
            return AuMSToNS<AuUInt64>(AuSToMS<AuUInt64>(spec.tv_sec)) + (AuUInt64)spec.tv_nsec;
        }
        else
        {
            return 0;
        }
    #else
        return std::chrono::duration_cast<std::chrono::nanoseconds>(steady_clock::now().time_since_epoch()).count();
    #endif
    }
    
    AUKN_SYM AuUInt64 SteadyClockFrequency()
    {
        static AuUInt64 gFrequency = 0;
        if (gFrequency != 0)
        {
            return gFrequency;
        }

    #if defined(GIB__GetSteadyTimeNS)
        return gFrequency = _NTLikeQueryFrequency();
    #elif defined(AURORA_IS_POSIX_DERIVED)
        ::timespec spec {};
        if (::clock_getres(CLOCK_MONOTONIC, &spec) == 0)
        {
            if (spec.tv_nsec && !spec.tv_sec)
            {
                return gFrequency = 1000000000ull / spec.tv_nsec;
            }
        }
        
        return gFrequency = (1000000000ull / 100ull);
    #else
        return gFrequency = static_cast<double>(steady_clock::period::den) / static_cast<double>(steady_clock::period::num);
    #endif
    }

    AUKN_SYM AuInt64 CTimeToMS(time_t time)
    {
        return std::chrono::duration_cast<std::chrono::milliseconds>(__NormalizeEpoch(sys_clock::from_time_t(time).time_since_epoch())).count();
    }

#if defined(AURORA_IS_POSIX_DERIVED)

    enum class EPseudoPosixClock
    {
        eUser,
        eKernel,
        eAll
    };

    static AuUInt64 GetPOSIXTimeEx(struct rusage *usage, EPseudoPosixClock e)
    {
        struct timeval *tv {};

        switch (e)
        {
        case EPseudoPosixClock::eAll:
        {
            return GetPOSIXTimeEx(usage, EPseudoPosixClock::eKernel) +
                GetPOSIXTimeEx(usage, EPseudoPosixClock::eUser);
        }
        case EPseudoPosixClock::eUser:
        {
            tv = &usage->ru_utime;
            break;
        }
        case EPseudoPosixClock::eKernel:
        {
            tv = &usage->ru_stime;
            break;
        }
        };

        auto uMS = AuSToMS<AuUInt64>(tv->tv_sec);
        auto uNS = AuMSToNS<AuUInt64>(uMS) +
            tv->tv_usec * 1'000ull;

        return uNS;
    }

    static AuUInt64 GetPOSIXTime(bool bThread, EPseudoPosixClock e)
    {
        struct rusage usage;

        getrusage(bThread ? RUSAGE_THREAD : RUSAGE_SELF,
                  &usage);

        return GetPOSIXTimeEx(&usage, e);
    }

    static AuPair<AuUInt64, AuUInt64> GetPOSIXTimePair(bool bThread)
    {
        struct rusage usage;

        getrusage(bThread ? RUSAGE_THREAD : RUSAGE_SELF,
                  &usage);

        return {
            GetPOSIXTimeEx(&usage, EPseudoPosixClock::eKernel),
            GetPOSIXTimeEx(&usage, EPseudoPosixClock::eUser)
        };
    }
    
    #if !defined(CLOCK_THREAD_CPUTIME_ID)
        #define CLOCK_THREAD_CPUTIME_ID 0
    #endif

    #if !defined(CLOCK_PROCESS_CPUTIME_ID)
        #define CLOCK_PROCESS_CPUTIME_ID 0
    #endif

#endif

#if defined(AURORA_IS_MODERNNT_DERIVED)    

#define ADD_CLOCK_FAMILY(fn, type, expr, posixId, posixCall)                                    \
    AUKN_SYM  AuUInt64 fn ## ClockFrequency();                                                  \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockMS()                                                           \
    {                                                                                           \
        return AuNSToMS<AuUInt64>(fn ## ClockNS());                                             \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockNS()                                                           \
    {                                                                                           \
        FILETIME creation, exit, kernel, user;                                                  \
        if (::Get ## type ## Times(GetCurrent ## type(), &creation, &exit, &kernel, &user))     \
        {                                                                                       \
            ULARGE_INTEGER ullUser;                                                             \
            {                                                                                   \
                ullUser.LowPart = user.dwLowDateTime;                                           \
                ullUser.HighPart = user.dwHighDateTime;                                         \
            }                                                                                   \
                                                                                                \
            ULARGE_INTEGER ullKernel;                                                           \
            {                                                                                   \
                ullKernel.LowPart = kernel.dwLowDateTime;                                       \
                ullKernel.HighPart = kernel.dwHighDateTime;                                     \
            }                                                                                   \
            return (expr) * 100ull;                                                             \
        }                                                                                       \
        return 0;                                                                               \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## Clock()                                                             \
    {                                                                                           \
        FILETIME creation, exit, kernel, user;                                                  \
                                                                                                \
        if (::Get ## type ## Times(GetCurrent ## type(), &creation, &exit, &kernel, &user))     \
        {                                                                                       \
            ULARGE_INTEGER ullUser;                                                             \
            {                                                                                   \
                ullUser.LowPart = user.dwLowDateTime;                                           \
                ullUser.HighPart = user.dwHighDateTime;                                         \
            }                                                                                   \
                                                                                                \
            ULARGE_INTEGER ullKernel;                                                           \
            {                                                                                   \
                ullKernel.LowPart = kernel.dwLowDateTime;                                       \
                ullKernel.HighPart = kernel.dwHighDateTime;                                     \
            }                                                                                   \
            return expr;                                                                        \
        }                                                                                       \
                                                                                                \
        return fn ##ClockNS() / (1000000000ull / fn ## ClockFrequency());                       \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockFrequency()                                                    \
    {                                                                                           \
        return 1000000000ull / 100u;                                                            \
    }

#elif defined(AURORA_IS_POSIX_DERIVED)

#define ADD_CLOCK_FAMILY(fn, type, expr, posixId, posixCall)                                    \
    AUKN_SYM  AuUInt64 fn ## ClockFrequency();                                                  \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockMS()                                                           \
    {                                                                                           \
        if (!posixId)                                                                           \
        {                                                                                       \
            return AuNSToMS<AuUInt64>(GetPOSIXTime AU_WHAT(posixCall));                         \
        }                                                                                       \
        return AuNSToMS<AuUInt64>(fn ## ClockNS());                                             \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockNS()                                                           \
    {                                                                                           \
        if (!posixId)                                                                           \
        {                                                                                       \
            return GetPOSIXTime AU_WHAT(posixCall);                                             \
        }                                                                                       \
        ::timespec spec {};                                                                     \
        if (::clock_gettime(posixId, &spec) == 0)                                               \
        {                                                                                       \
            return AuMSToNS<AuUInt64>(AuSToMS<AuUInt64>(spec.tv_sec)) + (AuUInt64)spec.tv_nsec; \
        }                                                                                       \
        return 0;                                                                               \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## Clock()                                                             \
    {                                                                                           \
        if (!posixId)                                                                           \
        {                                                                                       \
            return fn ##ClockNS() / 1000ull;                                                    \
        }                                                                                       \
        return fn ##ClockNS() / (1000000000ull / fn ## ClockFrequency());                       \
    }                                                                                           \
                                                                                                \
    AUKN_SYM AuUInt64 fn ## ClockFrequency()                                                    \
    {                                                                                           \
        if (!posixId)                                                                           \
        {                                                                                       \
            return 1'000'000ull;                                                                \
        }                                                                                       \
        static AuUInt64 frequency = 0;                                                          \
        if (frequency != 0)                                                                     \
        {                                                                                       \
            return frequency;                                                                   \
        }                                                                                       \
                                                                                                \
        ::timespec spec {};                                                                     \
        if (::clock_getres(posixId, &spec) == 0)                                                \
        {                                                                                       \
            if (spec.tv_nsec && !spec.tv_sec)                                                   \
            {                                                                                   \
                return frequency = 1000000000ull / spec.tv_nsec;                                \
            }                                                                                   \
            else                                                                                \
            {                                                                                   \
                SysUnreachable();                                                               \
                return 0;                                                                       \
            }                                                                                   \
        }                                                                                       \
                                                                                                \
        return 0;                                                                               \
    }

#else

    AUKN_SYM AuUInt64 fn ## ClockMS()
    {
        return 0;
    }

    AUKN_SYM AuUInt64 fn ## ClockNS()
    {
        return 0;
    }

    AUKN_SYM AuUInt64 fn ## Clock()
    {
        return 0;
    }

    AUKN_SYM AuUInt64 fn ## ClockFrequency()
    {
        return 0;
    }

#endif
    
    ADD_CLOCK_FAMILY(Process, Process, (ullUser.QuadPart + ullKernel.QuadPart), /*CLOCK_PROCESS_CPUTIME_ID*/ 0, (false, EPseudoPosixClock::eAll));
    ADD_CLOCK_FAMILY(ProcessKernel, Process, (ullKernel.QuadPart), 0, (false, EPseudoPosixClock::eKernel));
    ADD_CLOCK_FAMILY(ProcessUser, Process, (ullUser.QuadPart), /*CLOCK_PROCESS_CPUTIME_ID*/0, (false, EPseudoPosixClock::eUser));
    ADD_CLOCK_FAMILY(Thread, Thread, (ullUser.QuadPart + ullKernel.QuadPart), /*CLOCK_THREAD_CPUTIME_ID*/0, (true, EPseudoPosixClock::eAll));
    ADD_CLOCK_FAMILY(ThreadKernel, Thread, (ullKernel.QuadPart), 0, (true, EPseudoPosixClock::eKernel));
    ADD_CLOCK_FAMILY(ThreadUser, Thread, (ullUser.QuadPart), /*CLOCK_THREAD_CPUTIME_ID*/0, (true, EPseudoPosixClock::eUser));

#if defined(AURORA_IS_MODERNNT_DERIVED)

    #define ADD_CLOCK_FAMILY_PAIR(type, bIsThread, exprA, exprB)                                    \
                                                                                                    \
        static AuPair<AuUInt64, AuUInt64> type ## ClockPairNS()                                     \
        {                                                                                           \
            FILETIME creation, exit, kernel, user;                                                  \
            if (::Get ## type ## Times(GetCurrent ## type(), &creation, &exit, &kernel, &user))     \
            {                                                                                       \
                ULARGE_INTEGER ullUser;                                                             \
                {                                                                                   \
                    ullUser.LowPart = user.dwLowDateTime;                                           \
                    ullUser.HighPart = user.dwHighDateTime;                                         \
                }                                                                                   \
                                                                                                    \
                ULARGE_INTEGER ullKernel;                                                           \
                {                                                                                   \
                    ullKernel.LowPart = kernel.dwLowDateTime;                                       \
                    ullKernel.HighPart = kernel.dwHighDateTime;                                     \
                }                                                                                   \
                return AuMakePair((exprA) * 100ull, (exprB) * 100ull);                              \
            }                                                                                       \
            return {};                                                                              \
        }                                                                                           \

#elif defined(AURORA_IS_POSIX_DERIVED)

    #define ADD_CLOCK_FAMILY_PAIR(type, bIsThread, exprA, exprB)                                    \
                                                                                                    \
        static AuPair<AuUInt64, AuUInt64> type ## ClockPairNS()                                     \
        {                                                                                           \
            return GetPOSIXTimePair(bIsThread);                                                     \
        }                                                                                           \

#else

    #define ADD_CLOCK_FAMILY_PAIR(type, bIsThread, exprA, exprB)                                    \
                                                                                                    \
        static AuPair<AuUInt64, AuUInt64> type ## ClockPairNS()                                     \
        {                                                                                           \
            return {};                                                                              \
        }                                                                                           \
        

#endif

    ADD_CLOCK_FAMILY_PAIR(Thread, true, (ullKernel.QuadPart),
                                         (ullUser.QuadPart));
        
    ADD_CLOCK_FAMILY_PAIR(Process, false, (ullKernel.QuadPart),
                                          (ullUser.QuadPart));

    AUKN_SYM AuPair<AuUInt64, AuUInt64> GetClockUserAndKernelTimeNS(EClock clock)
    {
        AuPair<AuUInt64, AuUInt64> swizzle;

        switch (clock)
        {
        case EClock::eWall:
        case EClock::eSteady:
            SysPushErrorArg("Invalid clock");
            return {};
        case EClock::eProcessTime:
        case EClock::eProcessUserTime:
        case EClock::eProcessKernelTime:
            swizzle = ProcessClockPairNS();
            break;
        case EClock::eThreadTime:
        case EClock::eThreadUserTime:
        case EClock::eThreadKernelTime:
            swizzle = ThreadClockPairNS();
            break;
        default:
            SysPushErrorArg("Invalid clock");
            return {};
        }

        if (swizzle == AuPair<AuUInt64, AuUInt64> {})
        {
            SysPushErrorGeneric();
            return {};
        }

        return { swizzle.second, swizzle.first };
    }
}