/*** Copyright (C) 2022 J Reece Wilson (a/k/a "Reece"). All rights reserved. File: Cache.cpp Date: 2022-3-21 Author: Reece ***/ #include #include "Memory.hpp" #include "Cache.hpp" #include #define LINUX_SUPPORTS_CACHE_CTL 0 #if defined(AURORA_IS_LINUX_DERIVED) #include #if LINUX_SUPPORTS_CACHE_CTL #include #endif #endif #if defined(AURORA_IS_XNU_DERIVED) void sys_icache_invalidate(void *start, size_t len); #endif namespace Aurora::Memory::Cache { #if defined(AURORA_IS_MODERNNT_DERIVED) struct WIN32_MEMORY_RANGE_ENTRY2 { PVOID VirtualAddress; SIZE_T NumberOfBytes; }; BOOL (__stdcall *PrefetchVirtualMemory_f)( HANDLE hProcess, ULONG_PTR NumberOfEntries, WIN32_MEMORY_RANGE_ENTRY2* VirtualAddresses, ULONG Flags ); #endif AUKN_SYM void OptimizeAddressRangeOnCore(const AuList> &addressRanges) { #if defined(AURORA_IS_MODERNNT_DERIVED) AuList arry; if (!PrefetchVirtualMemory_f) { return; } if (!AuTryResize(arry, addressRanges.size())) { return; } AuUInt index {}; #endif for (const auto &addressRange : addressRanges) { auto base = AuPageRound(addressRange.first, AuUInt(AuHwInfo::gPageSize)); auto length = AuPageRoundUp(addressRange.second + (addressRange.first - base), AuUInt(AuHwInfo::gPageSize)); #if defined(AURORA_IS_MODERNNT_DERIVED) arry[index++] = { AuReinterpretCast(base), length }; #endif #if defined(AURORA_IS_POSIX_DERIVED) // The posix_madvise() interface conforms to IEEE Std 1003.1-2001 ("POSIX.1"). // The madvise() system call first appeared in 4.4BSD. // https://www.freebsd.org/cgi/man.cgi?query=madvise&apropos=0&sektion=2&manpath=FreeBSD+9.0-RELEASE&arch=default&format=html // https://man7.org/linux/man-pages/man2/madvise.2.html if (madvise(AuReinterpretCast(base), length, MADV_WILLNEED) != 0) { SysPushErrorHAL("Couldn't advise memory range 0x{:x} {}", base, length); } #endif } #if defined(AURORA_IS_MODERNNT_DERIVED) if (!PrefetchVirtualMemory_f(GetCurrentProcess(), arry.size(), arry.data(), 0)) { SysPushErrorHAL("Couldn't poke memory physical memory into virtual address space (array)"); } #endif } AUKN_SYM void ClearInstructionCache(const AuPair &addressRange) { auto base = AuPageRound(addressRange.first, AuUInt(AuHwInfo::gPageSize)); auto length = AuPageRoundUp(addressRange.second + (addressRange.first - base), AuUInt(AuHwInfo::gPageSize)); #if defined(AURORA_IS_MODERNNT_DERIVED) FlushInstructionCache(GetCurrentProcess(), AuReinterpretCast(base), length); #elif defined(AURORA_IS_XNU_DERIVED) // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sys_icache_invalidate.3.html sys_icache_invalidate(AuReinterpretCast(base), length); #elif defined(AURORA_IS_LINUX_DERIVED) && defined(LINUX_SUPPORTS_CACHE_CTL) && LINUX_SUPPORTS_CACHE_CTL // https://man7.org/linux/man-pages/man2/cacheflush.2.html cacheflush(AuReinterpretCast(base), length, ICACHE); #elif (defined(AURORA_ARCH_X86) || defined(AURORA_ARCH_X64)) && (defined(AURORA_COMPILER_CLANG) || AURORA_COMPILER_GCC) auto dwCacheLine = AuHwInfo::GetCPUInfo().dwCacheLine; const auto kCacheLine = dwCacheLine ? dwCacheLine : 64; asm volatile("sfence\n\t" : : : "memory"); for (AuUInt i = 0; i < length; i += kCacheLine) { asm volatile("clflush (%0)\n\t" : : "r"(AuReinterpretCast(base) + i) : "memory"); } #elif defined(AURORA_COMPILER_GCC) __builtin___clear_cache(AuReinterpretCast(base), AuReinterpretCast(base) + length); #elif defined(AURORA_COMPILER_CLANG) && defined(AURORA_ARCH_ARM) // https://github.com/llvm/llvm-project/blob/d480f968ad8b56d3ee4a6b6df5532d485b0ad01e/clang/include/clang/Basic/BuiltinsARM.def#L25 // Does not exist anywhere else __clear_cache(AuReinterpretCast(base), AuReinterpretCast(base) + length); #endif } void InitCache() { #if defined(AURORA_IS_MODERNNT_DERIVED) PrefetchVirtualMemory_f = AuReinterpretCast(GetProcAddress(GetModuleHandleW(L"Kernel32.dll"), "PrefetchVirtualMemory")); #endif } }