diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 14c111c..fa14591 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -36,6 +36,7 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize); void _mi_os_free(void* p, size_t size, mi_stats_t* stats); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); +void _mi_os_init(void); // called from process init void* _mi_os_alloc_aligned(size_t size, size_t alignment, mi_os_tld_t* tld); size_t _mi_os_page_size(void); diff --git a/include/mimalloc.h b/include/mimalloc.h index 51f7ce1..efb1f38 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -216,6 +216,7 @@ typedef enum mi_option_e { mi_option_page_reset, mi_option_cache_reset, mi_option_pool_commit, + mi_option_large_os_pages, mi_option_secure, mi_option_show_stats, mi_option_show_errors, diff --git a/src/init.c b/src/init.c index 3e60fb0..6b05907 100644 --- a/src/init.c +++ b/src/init.c @@ -368,6 +368,7 @@ void mi_process_init(void) mi_attr_noexcept { atexit(&mi_process_done); mi_process_setup_auto_thread_done(); mi_stats_reset(); + _mi_os_init(); } static void mi_process_done(void) { diff --git a/src/options.c b/src/options.c index 96f0014..9de7aa8 100644 --- a/src/options.c +++ b/src/options.c @@ -31,6 +31,7 @@ static mi_option_desc_t options[_mi_option_last] = { { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, { 0, UNINIT, "pool_commit" }, + { 0, UNINIT, "large_os_pages" }, // use large OS pages #if MI_SECURE { MI_SECURE, INITIALIZED, "secure" }, // in secure build the environment setting is ignored #else diff --git a/src/os.c b/src/os.c index 370b489..386a00e 100644 --- a/src/os.c +++ b/src/os.c @@ -16,21 +16,86 @@ terms of the MIT license. A copy of the license can be found in the file #include /* ----------------------------------------------------------- - Raw allocation on Windows (VirtualAlloc) and Unix's (mmap). - Defines a portable `mmap`, `munmap` and `mmap_trim`. + Initialization. + On windows initializes support for aligned allocation and + large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). ----------------------------------------------------------- */ #if defined(_WIN32) - #include - #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) // rough check it VirtualAlloc2 is available (needs windows 10 or Windows server 2016) - #pragma comment(lib, "mincore.lib") // seems needed to resolve VirtualAlloc2 - #define USE_VIRTUALALLOC2 // allows aligned allocation - #endif + #include #else #include // mmap #include // sysconf #endif +// if non-zero, use large page allocation +static size_t large_os_page_size = 0; + +static bool use_large_os_page(size_t size, size_t alignment) { + // if we have access, check the size and alignment requirements + if (large_os_page_size == 0) return false; + return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); +} + + +#if defined(_WIN32) +// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. +// So, we need to look it up dynamically to run on older systems. +typedef PVOID (*VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG ); +static VirtualAlloc2Ptr pVirtualAlloc2 = NULL; + +void _mi_os_init(void) { + // Try to get the VirtualAlloc2 function (only supported on Windows 10 and Windows Server 2016) + HINSTANCE hDll; + hDll = LoadLibrary("kernelbase.dll"); + if (hDll!=NULL) { + pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2"); + FreeLibrary(hDll); + } + // Try to see if large OS pages are supported + unsigned long err = 0; + bool ok = mi_option_is_enabled(mi_option_large_os_pages); + if (ok) { + // To use large pages on Windows, we first need access permission + // Set "Lock pages in memory" permission in the group policy editor + // + HANDLE token = NULL; + ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + if (ok) { + TOKEN_PRIVILEGES tp; + ok = LookupPrivilegeValue(NULL, "SeLockMemoryPrivilege", &tp.Privileges[0].Luid); + if (ok) { + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + if (ok) { + err = GetLastError(); + ok = (err == ERROR_SUCCESS); + if (ok) { + large_os_page_size = GetLargePageMinimum(); + } + } + } + CloseHandle(token); + } + if (!ok) { + if (err==0) err = GetLastError(); + _mi_warning_message("cannot enable large OS page support, error %lu\n", err); + } + } +} +#else +void _mi_os_init() { + // nothing to do + use_large_os_page(0, 0); // dummy call to suppress warnings +} +#endif + + +/* ----------------------------------------------------------- + Raw allocation on Windows (VirtualAlloc) and Unix's (mmap). + Defines a portable `mmap`, `munmap` and `mmap_trim`. +----------------------------------------------------------- */ uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { uintptr_t x = (sz / alignment) * alignment; @@ -92,9 +157,14 @@ static bool mi_munmap(void* addr, size_t size) static void* mi_mmap(void* addr, size_t size, int extra_flags, mi_stats_t* stats) { UNUSED(stats); if (size == 0) return NULL; - void* p; + void* p = NULL; #if defined(_WIN32) - p = VirtualAlloc(addr, size, MEM_RESERVE | MEM_COMMIT | extra_flags, PAGE_READWRITE); + if (use_large_os_page(size, 0)) { + p = VirtualAlloc(addr, size, MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT | extra_flags, PAGE_READWRITE); + } + if (p == NULL) { + p = VirtualAlloc(addr, size, MEM_RESERVE | MEM_COMMIT | extra_flags, PAGE_READWRITE); + } #else #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON @@ -124,14 +194,18 @@ static void* mi_mmap(void* addr, size_t size, int extra_flags, mi_stats_t* stats static void* mi_mmap_aligned(size_t size, size_t alignment, mi_stats_t* stats) { if (alignment < _mi_os_page_size() || ((alignment & (~alignment + 1)) != alignment)) return NULL; void* p = NULL; - #if defined(_WIN32) && defined(USE_VIRTUALALLOC2) - // on modern Windows use VirtualAlloc2 - MEM_ADDRESS_REQUIREMENTS reqs = {0}; - reqs.Alignment = alignment; - MEM_EXTENDED_PARAMETER param = { 0 }; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &reqs; - p = VirtualAlloc2(NULL, NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE, ¶m, 1); + #if defined(_WIN32) && defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) + if (pVirtualAlloc2 != NULL) { + // on modern Windows try use VirtualAlloc2 + MEM_ADDRESS_REQUIREMENTS reqs = {0}; + reqs.Alignment = alignment; + MEM_EXTENDED_PARAMETER param = { 0 }; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &reqs; + DWORD extra_flags = 0; + if (use_large_os_page(size, alignment)) extra_flags |= MEM_LARGE_PAGES; + p = (*pVirtualAlloc2)(NULL, NULL, size, MEM_RESERVE | MEM_COMMIT | extra_flags, PAGE_READWRITE, ¶m, 1); + } #elif defined(MAP_ALIGNED) // on BSD, use the aligned mmap api size_t n = _mi_bsr(alignment); diff --git a/src/page.c b/src/page.c index 6b41625..73fc593 100644 --- a/src/page.c +++ b/src/page.c @@ -608,7 +608,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p else { mi_assert(pq->first == page); } - mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); return page; }