AuroraRuntime/Source/Threading/Threads/OSThread.cpp

/***
    Copyright (C) 2021 J Reece Wilson (a/k/a "Reece"). All rights reserved.

    File: OSThread.cpp
    Date: 2021-6-12
    Author: Reece
***/
#include <Source/RuntimeInternal.hpp>
#include "Threads.hpp"
#include "OSThread.hpp"
#include "../WaitFor.hpp"
#include "ThreadHandles.hpp"
#include "TLSView.hpp"

#if defined(AURORA_IS_LINUX_DERIVED)
#include <sys/resource.h>
#endif

#if defined(AURORA_HAS_PTHREADS)
#include <sched.h>
#endif

#if defined(AURORA_PLATFORM_WIN32)
#include <process.h>
#endif


namespace Aurora::Threading::Threads
{
    static AbstractThreadVectors gDummyVectors {};
    static ThreadInfo gDummyThreadInfo(gDummyVectors);

    OSThread::OSThread(const ThreadInfo &info) : info_(vecs)
    {
        vecs = info.vectors;
        info_.stackSize = info.stackSize;
        info_.name = info.name;
        name_ = info.name.value_or("Aurora Thread");
        terminated_ = Primitives::EventShared(true, false);
        exitOnlyOnce_ = Primitives::CriticalSectionUnique();
        SysAssert(terminated_ ? true : false, "out of memory");
    }

    OSThread::OSThread() : info_(gDummyThreadInfo)
    {
        name_ = "Main Thread";
        terminated_ = Primitives::EventShared(true, false);
        exitOnlyOnce_ = Primitives::CriticalSectionUnique();
    }

    OSThread::OSThread(AuUInt64 id) : info_(gDummyThreadInfo)
    {
        name_ = "System Thread";
        handle_ = reinterpret_cast<decltype(handle_)>(id);
    }

    OSThread::~OSThread()
    {
        if (contextUsed_)
        {
            Exit();
            WaitFor(terminated_.get());
        }

        terminated_.reset();
        FreeOSContext();
        HookReleaseThreadResources();
    }

    // Called whenever an aurora thread is exiting
    void OSThread::HookOnExit()
    {

    }

    // Called whenever a thread object is released, or on thread termination
    // Whichever is soonest
    // It is possible a user may wish to keep a thread handle open
    // Is is expected for a system thread to release on dtor unlike our aurora threads 
    void OSThread::HookReleaseThreadResources()
    {
        tls_.reset();

        for (const auto &feature : threadFeatures_)
        {
            feature->Cleanup();
        }
    }

    void OSThread::AddLastHopeTlsHook(const AuSPtr<Threading::Threads::IThreadFeature> &feature)
    {
        // TODO: mutex
        AuTryInsert(threadFeatures_, feature);
    }

    AuSPtr<IWaitable> OSThread::AsWaitable()
    {
        return terminated_;
    }

    void OSThread::SendExitSignal()
    {
        exiting_ = true;
    }

    bool OSThread::Run()
    {
        if (!terminated_)
        {
            SysPanic("::Run called on system thread");
        }

        if (std::exchange(contextUsed_, true))
        {
            return false;
        }

        terminated_->Reset();

        return ExecuteNewOSContext([=]()
        {
            try
            {
                if (info_.vectors.DoRun)
                {
                    info_.vectors.DoRun(this);
                }
            }
            catch (...)
            {
                Debug::PrintError();
            }
        });
    }

    void OSThread::Exit()
    {
        while ((!terminated_) || (!terminated_->TryLock()))
        {
            if (Exit(false)) break;
        }
    }

    bool OSThread::Exit(bool willReturnToOS)
    {
        if (GetThread() == this)
        {
            if (!this->InternalKill(false))
            {
                return false;
            }

            // release handle + sigterm
            if (!willReturnToOS)
            {
                TeminateOSContext(true);
                while (true)
                {
                    YieldToOtherThread();
                }
            }
        }
        else
        {
            // exit signal 
            exiting_ = true;

            if (!terminated_)
            {
                return true;
            }

            // attempt to join with the thread once it has exited, or timeout 
            if (WaitFor(terminated_.get(), 15 * 1000))
            {
                return true;
            }

            // Do not force terminate if we're marked as dead and still running
            // The thread must've requested suicide and got stuck in a lengthy clean up effort
            if (!exitOnlyOnce_->TryLock())
            {
                LogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, but he should exiting now.");
                return false;
            }

            LogWarn("Watchdog error - OS thread context didn't finish in 15 seconds, forcefully terminating without a watchdog overlooking onExit");

            // Kill the current OS thread instance
            TeminateOSContext(false);

            // Dispatch kill callback from within an emulated thread context (no switch, just just change thread tls)
            ExecuteInDeadThread([=]()
            {
                this->InternalKill(true);
            });

            exitOnlyOnce_->Unlock();
        }
        return true;
    }

    bool OSThread::Exiting()
    {
        return exiting_;
    }

    void OSThread::SetName(const AuString &name)
    {
        name_ = name;
    }

    EThreadPrio OSThread::GetPrio()
    {
        return prio_;
    }

    AuUInt32 OSThread::GetMask()
    {
        return affinityProcessMask_;
    }

    AuString OSThread::GetName()
    {
        return name_;
    }

    void OSThread::SetAffinity(AuUInt32 mask)
    {
        affinityProcessMask_ = mask;
        UpdateAffinity(mask);
    }

    void OSThread::SetPrio(EThreadPrio prio)
    {
        prio_ = prio;
        UpdatePrio(prio);
    }

    AuSPtr<TLSView> OSThread::GetTlsView()
    {
        if (!tls_)
        {
            tls_ = AuMakeShared<TLSViewImpl>();
        }

        return tls_;
    }

    bool OSThread::ExecuteNewOSContext(std::function<void()> task)
    {
        task_ = task;

        // https://github.com/webrtc-uwp/chromium-base/blob/master/threading/platform_thread_win.cc#L129-L136
        // Do we care yet?
        // When targeting older CRTs, _beginthreadex
        // When targeting at least UWP, CreateThread but _beginthreadex is fine
        // https://web.archive.org/web/20110928122401/http://www.microsoft.com/msj/1099/win32/win321099.aspx
        // Even in 1999 it sounded like _tiddata was setup for all modules that may come across our thread
        // It wasn't until 2005, 6 years later, it became less of a requirement.
        // Does the modern CRT need it for anything estoeric or evil?
        // I think so long as we're targeting modern windows its fine to call _beginthreadex for all CRT users
        // Userland icds, plugins, software not in our build chain, it makes sense for them to have a _fully_ initialized crt
        // I think I switched it from CreateThread to _beginthreadex at somepoint and i don't remember why
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        unsigned(*OSEP_f)(void *) = [](void *that) -> unsigned
        {
            auto thiz = reinterpret_cast<OSThread *>(that);
            thiz->_ThreadEP();
            return 0;
        };

        auto ok = _beginthreadex(nullptr, info_.stackSize.value_or(0), OSEP_f, this, 0, 0);
        if (ok == -1L)
        {
            SysPushErrorGen("Couldn't initialize thread: {}", GetName());
            return false;
        }

        handle_ = reinterpret_cast<HANDLE>(ok);

    #elif defined(AURORA_HAS_PTHREADS)
        pthread_attr_t tattr;

        void *(*OSEP_f)(void *) = [](void *that) -> void *
        {
            auto thiz = reinterpret_cast<OSThread *>(that);
            thiz->_ThreadEP();
            return nullptr;
        };

        auto ret = pthread_attr_init(&tattr);
        if (ret != 0)
        {
            SysPushErrorGen("Couldn't create thread: {}", GetName());
            return false;
        }

        if (info_.stackSize.has_value())
        {
            ret = pthread_attr_setstacksize(&tattr, std::min(AuUInt32(PTHREAD_STACK_MIN), info_.stackSize.value()));
            if (ret != 0)
            {
                SysPushErrorGen("Couldn't create thread: {}", GetName());
                return false;
            }
        }

        ret = pthread_create(&handle_, &tattr, OSEP_f, this);
        if (ret != 0)
        {
            SysPushErrorGen("Couldn't create thread: {}", GetName());
            return false;
        }

    #else
    
        SysPanic("Not implemented");
    
    #endif

        return true;
    }

    void OSThread::_ThreadEP()
    {
        Attach();
        task_();
        Exit(true);
    }

    void OSThread::ExecuteInDeadThread(std::function<void()> callback)
    {
        auto old = HandleCurrent();

        if (this == old) [[unlikely]]
        {
            callback();
            return;
        }

        if (old)
        {
            static_cast<OSThread *>(old)->Deattach();
        }
        
        this->Attach();
        
        callback();

        if (old)
        {
            HandleRegister(old);
            static_cast<OSThread *>(old)->Attach();
        }
        else [[unlikely]]
        {
            HandleRemove();
        }
    }

    void OSThread::UpdateName()
    {
    #if defined(AURORA_PLATFORM_WIN32)
        static const DWORD kMSVCExceptionSetName = 0x406D1388;

    #pragma pack(push,8)
        typedef struct tagTHREADNAME_INFO
        {
            DWORD dwType;
            LPCSTR szName;
            DWORD dwThreadID;
            DWORD dwFlags;
        } THREADNAME_INFO;
    #pragma pack(pop)

        if (handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if (!IsDebuggerPresent())
        {
            return;
        }

        THREADNAME_INFO info;
        info.dwType = 0x1000;
        info.szName = name_.c_str();
        info.dwThreadID = ::GetThreadId(handle_);
        info.dwFlags = 0;

        __try
        {
            RaiseException(kMSVCExceptionSetName, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR *)&info);
        }
        __except (EXCEPTION_EXECUTE_HANDLER)
        {
        }
    
    #elif defined(AURORA_HAS_PTHREADS)

        pthread_setname_np(handle_, name_.c_str());
    
    #endif
    }


    void OSThread::Attach()
    {
        HandleRegister(this);
    #if defined(AURORA_IS_LINUX_DERIVED)
        this->unixThreadId_ = gettid();
    #elif defined(AURORA_IS_XNU_DERIVED)
        this->unixThreadId_ = pthread_getthreadid_np();
    #elif defined(AURORA_IS_BSD_DERIVED)
        #if __FreeBSD_version < 900031
            long lwpid;
            thr_self(&lwpid);
            this->unixThreadId_ = lwpid;
        #elif __FreeBSD_version > 900030
            this->unixThreadId_ = pthread_getthreadid_np();
        #else
            static_assert(false);
        #endif
    #elif defined(AURORA_HAS_PTHREADS)
        this->unixThreadId_ = 0; // !!!!
    #endif
        UpdatePrio(prio_);
        UpdateAffinity(affinityProcessMask_);
        UpdateName();
    }

    static AuHashMap<EThreadPrio, int> kNiceMap 
    {
        {
            EThreadPrio::ePrioRT, -19
        },
        {
            EThreadPrio::ePrioAboveHigh, -19
        },
        {
            EThreadPrio::ePrioHigh, -11
        },
        {
            EThreadPrio::ePrioNormal, -2
        },
        {
            EThreadPrio::ePrioSub, 5
        }
    };

#if defined(AURORA_IS_MODERNNT_DERIVED)
    static const AuHashMap<EThreadPrio, int> kWin32Map
    {
        {
            EThreadPrio::ePrioRT, THREAD_PRIORITY_TIME_CRITICAL
        },
        {
            EThreadPrio::ePrioAboveHigh, THREAD_PRIORITY_HIGHEST
        },
        {
            EThreadPrio::ePrioHigh, THREAD_PRIORITY_ABOVE_NORMAL
        },
        {
            EThreadPrio::ePrioNormal, THREAD_PRIORITY_NORMAL
        },
        {
            EThreadPrio::ePrioSub, THREAD_PRIORITY_LOWEST
        }
    };
#endif

    void OSThread::UpdatePrio(EThreadPrio prio)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        const int *val;
        if (!AuTryFind(kWin32Map, prio, val))
        {
            return;
        }

        SetThreadPriority(handle_, *val);

    #elif defined(AURORA_HAS_PTHREADS)

        if (!handle_)
        {
            return;
        }

        if (prio == EThreadPrio::ePrioRT)
        {
            sched_param param {};
            param.sched_priority = sched_get_priority_min(SCHED_RR);

            if (pthread_setschedparam(handle_, SCHED_RR, &param) == 0)
            {
                return;
            }

            // fall through on error
        }

        const int *val;
        if (!AuTryFind(kNiceMap, prio, val))
        {
            return;
        }

        if (auto tid = unixThreadId_)
        {
            setpriority(PRIO_PROCESS, tid, *val);
        }
    #endif 
    }

    void OSThread::UpdateAffinity(AuUInt32 mask)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        if (handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }
        SetThreadAffinityMask(handle_, static_cast<DWORD_PTR>(mask));
    #endif
    }

    void OSThread::Deattach()
    {

    }

    bool OSThread::InternalKill(bool locked)
    {
        if (terminated_)
        {
            if (!locked)
            {
                if (!exitOnlyOnce_->TryLock())
                {
                    return false;
                }
            }
        }

        HookOnExit();

        // dispatch kill callback
        if (info_.vectors.DoExit)
        {
            try
            {
                info_.vectors.DoExit(this);
            }
            catch (...)
            {
                Debug::PrintError();
                LogWarn("Couldn't deinitialize thread");
                LogWarn("The smart thing to do at this point would be to panic");
                LogWarn("...but we could continue");
                LogWarn("Carrying on despite the potential for data integrity loss and memory leaks");
                Telemetry::Mayday();
            }
        }

        HookReleaseThreadResources();

        if (terminated_)
        {
            exitOnlyOnce_->Unlock();
            terminated_->Set();
        }
        return true;
    }


    void OSThread::TeminateOSContext(bool calledFromThis)
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)

        if (handle_ == INVALID_HANDLE_VALUE)
        {
            return;
        }

        if (calledFromThis)
        {
            ExitThread(0);
        }
        else
        {
            TerminateThread(handle_, 0);
        }

    #elif defined(AURORA_HAS_PTHREADS)

        pthread_kill(handle_, SIGTERM);

    #else
    
        SysPanic("Not implemented");
    
    #endif
    }

    void OSThread::FreeOSContext()
    {
    #if defined(AURORA_IS_MODERNNT_DERIVED)
        if (auto handle = std::exchange(handle_, {}))
        {
            CloseHandle(handle_); 
        }
    #endif
    }
}