AuroraMiddleware/mimalloc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'dev' into dev-exp

Browse Source
This commit is contained in:
daan 2019-08-20 08:27:46 -07:00
commit 86d4ded831
13 changed files with 80 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ide/vs2017/mimalloc-override.vcxproj
View File

@ -150,7 +150,6 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
<PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@ -159,6 +158,7 @@
<WholeProgramOptimization>false</WholeProgramOptimization>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<CompileAs>Default</CompileAs>
<BufferSecurityCheck>false</BufferSecurityCheck>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
@ -182,7 +182,6 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
<PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@ -191,6 +190,7 @@
<WholeProgramOptimization>false</WholeProgramOptimization>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<CompileAs>Default</CompileAs>
<BufferSecurityCheck>false</BufferSecurityCheck>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>

16
ide/vs2017/mimalloc.vcxproj
View File

@ -141,8 +141,6 @@
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
<PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@ -150,11 +148,9 @@
<AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
<WholeProgramOptimization>false</WholeProgramOptimization>
<BufferSecurityCheck>false</BufferSecurityCheck>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
<OmitFramePointers>false</OmitFramePointers>
<EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
<InlineFunctionExpansion>Default</InlineFunctionExpansion>
<CompileAs>CompileAsCpp</CompileAs>
<IntrinsicFunctions>true</IntrinsicFunctions>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
@ -172,8 +168,6 @@
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
<PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@ -181,11 +175,9 @@
<AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
<WholeProgramOptimization>false</WholeProgramOptimization>
<BufferSecurityCheck>false</BufferSecurityCheck>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
<OmitFramePointers>false</OmitFramePointers>
<EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
<InlineFunctionExpansion>Default</InlineFunctionExpansion>
<CompileAs>CompileAsCpp</CompileAs>
<IntrinsicFunctions>true</IntrinsicFunctions>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>

8
include/mimalloc-internal.h
View File

@ -151,15 +151,15 @@ bool _mi_page_is_valid(mi_page_t* page);
// Overflow detecting multiply
#define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) {
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
#if (MI_INTPTR_SIZE == 4)
return __builtin_umul_overflow(size, count, total);
return __builtin_umul_overflow(count, size, total);
#else
return __builtin_umull_overflow(size, count, total);
return __builtin_umull_overflow(count, size, total);
#endif
#else /* __builtin_umul_overflow is unavailable */
*total = size * count;
*total = count * size;
return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
&& size > 0 && (SIZE_MAX / size) < count);
#endif

2
include/mimalloc-types.h
View File

@ -93,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4)
#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit
#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit
#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2Mb on 64-bit
#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT)
#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)

9
include/mimalloc.h
View File

@ -220,8 +220,8 @@ mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
typedef enum mi_option_e {
// stable options
mi_option_show_stats,
mi_option_show_errors,
mi_option_show_stats,
mi_option_verbose,
// the following options are experimental
mi_option_secure,
@ -231,14 +231,15 @@ typedef enum mi_option_e {
mi_option_page_reset,
mi_option_cache_reset,
mi_option_reset_decommits,
mi_option_reset_discards,
_mi_option_last
} mi_option_t;
mi_decl_export bool mi_option_is_enabled(mi_option_t option);
mi_decl_export void mi_option_enable(mi_option_t option, bool enable);
mi_decl_export void mi_option_enable_default(mi_option_t option, bool enable);
mi_decl_export void mi_option_enable(mi_option_t option);
mi_decl_export void mi_option_disable(mi_option_t option);
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
mi_decl_export long mi_option_get(mi_option_t option);
mi_decl_export void mi_option_set(mi_option_t option, long value);

2
src/alloc-override-win.c
View File

@ -666,7 +666,7 @@ static void mi_patches_at_quick_exit(void) {
mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op
}
__declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH) {
__security_init_cookie();
}

6
src/heap.c
View File

@ -303,14 +303,14 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
mi_assert_internal(heap!=NULL);
if (from==NULL || from->page_count == 0) return;
// unfull all full pages
mi_page_t* page = heap->pages[MI_BIN_FULL].first;
// unfull all full pages in the `from` heap
mi_page_t* page = from->pages[MI_BIN_FULL].first;
while (page != NULL) {
mi_page_t* next = page->next;
_mi_page_unfull(page);
page = next;
}
mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL);
mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL);
// free outstanding thread delayed free blocks
_mi_heap_delayed_free(from);

6
src/init.c
View File

@ -106,14 +106,14 @@ mi_heap_t _mi_heap_main = {
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY,
NULL,
0,
0,
0, // thread id
#if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
0xCDCDCDCDCDCDCDCDUL,
#else
0xCDCDCDCDUL,
#endif
0,
0, // random
0, // page count
false // can reclaim
};

21
src/memory.c
View File

@ -128,6 +128,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
size_t mask = mi_region_block_mask(blocks,bitidx);
mi_assert_internal(mask != 0);
mi_assert_internal((mask & mi_atomic_read(&region->map)) == mask);
mi_assert_internal(&regions[idx] == region);
// ensure the region is reserved
void* start = mi_atomic_read_ptr(&region->start);
@ -149,9 +150,23 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
mi_atomic_increment(&regions_count);
}
else {
// failed, another thread allocated just before us, free our allocated memory
// TODO: should we keep the allocated memory and assign it to some other region?
_mi_os_free(start, MI_REGION_SIZE, tld->stats);
// failed, another thread allocated just before us!
// we assign it to a later slot instead (up to 4 tries).
// note: we don't need to increment the region count, this will happen on another allocation
for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) {
void* s = mi_atomic_read_ptr(&regions[idx+i].start);
if (s == NULL) { // quick test
if (mi_atomic_compare_exchange_ptr(&regions[idx+i].start, start, s)) {
start = NULL;
break;
}
}
}
if (start != NULL) {
// free it if we didn't succeed to save it to some other region
_mi_os_free(start, MI_REGION_SIZE, tld->stats);
}
// and continue with the memory at our index
start = mi_atomic_read_ptr(&region->start);
}
}

44
src/options.c
View File

@ -34,34 +34,37 @@ typedef enum mi_init_e {
typedef struct mi_option_desc_s {
long value; // the value
mi_init_t init; // is it initialized yet? (from the environment)
mi_option_t option; // for debugging: the option index should match the option
const char* name; // option name without `mimalloc_` prefix
} mi_option_desc_t;
#define MI_OPTION(opt) mi_option_##opt, #opt
#define MI_OPTION_DESC(opt) {0, UNINIT, MI_OPTION(opt) }
static mi_option_desc_t options[_mi_option_last] =
{
// stable options
{ 0, UNINIT, "show_stats" },
{ MI_DEBUG, UNINIT, "show_errors" },
{ 0, UNINIT, "verbose" },
{ MI_DEBUG, UNINIT, MI_OPTION(show_errors) },
{ 0, UNINIT, MI_OPTION(show_stats) },
{ 0, UNINIT, MI_OPTION(verbose) },
#if MI_SECURE
{ MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored
{ MI_SECURE, INITIALIZED, MI_OPTION(secure) }, // in a secure build the environment setting is ignored
#else
{ 0, UNINIT, "secure" },
{ 0, UNINIT, MI_OPTION(secure) },
#endif
// the following options are experimental and not all combinations make sense.
{ 1, UNINIT, "eager_commit" }, // note: if eager_region_commit is on, this should be on too.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // note: if eager_region_commit is on, this should be on too.
#ifdef _WIN32 // and BSD?
{ 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...)
{ 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
#else
{ 1, UNINIT, "eager_region_commit" },
{ 1, UNINIT, MI_OPTION(eager_region_commit) },
#endif
{ 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, "page_reset" },
{ 0, UNINIT, "cache_reset" },
{ 0, UNINIT, "reset_decommits" }, // note: cannot enable this if secure is on
{ 0, UNINIT, "reset_discards" } // note: cannot enable this if secure is on
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(page_reset) },
{ 0, UNINIT, MI_OPTION(cache_reset) },
{ 0, UNINIT, MI_OPTION(reset_decommits) } // note: cannot enable this if secure is on
};
static void mi_option_init(mi_option_desc_t* desc);
@ -69,6 +72,7 @@ static void mi_option_init(mi_option_desc_t* desc);
long mi_option_get(mi_option_t option) {
mi_assert(option >= 0 && option < _mi_option_last);
mi_option_desc_t* desc = &options[option];
mi_assert(desc->option == option); // index should match the option
if (mi_unlikely(desc->init == UNINIT)) {
mi_option_init(desc);
if (option != mi_option_verbose) {
@ -81,6 +85,7 @@ long mi_option_get(mi_option_t option) {
void mi_option_set(mi_option_t option, long value) {
mi_assert(option >= 0 && option < _mi_option_last);
mi_option_desc_t* desc = &options[option];
mi_assert(desc->option == option); // index should match the option
desc->value = value;
desc->init = INITIALIZED;
}
@ -97,14 +102,23 @@ bool mi_option_is_enabled(mi_option_t option) {
return (mi_option_get(option) != 0);
}
void mi_option_enable(mi_option_t option, bool enable) {
void mi_option_set_enabled(mi_option_t option, bool enable) {
mi_option_set(option, (enable ? 1 : 0));
}
void mi_option_enable_default(mi_option_t option, bool enable) {
void mi_option_set_enabled_default(mi_option_t option, bool enable) {
mi_option_set_default(option, (enable ? 1 : 0));
}
void mi_option_enable(mi_option_t option) {
mi_option_set_enabled(option,true);
}
void mi_option_disable(mi_option_t option) {
mi_option_set_enabled(option,false);
}
// --------------------------------------------------------
// Messages
// --------------------------------------------------------

24
src/os.c
View File

@ -22,6 +22,9 @@ terms of the MIT license. A copy of the license can be found in the file
#else
#include <sys/mman.h> // mmap
#include <unistd.h> // sysconf
#if defined(__linux__)
#include <linux/mman.h> // linux mmap flags
#endif
#if defined(__APPLE__)
#include <mach/vm_statistics.h>
#endif
@ -67,7 +70,7 @@ size_t _mi_os_large_page_size() {
static bool use_large_os_page(size_t size, size_t alignment) {
// if we have access, check the size and alignment requirements
if (large_os_page_size == 0) return false;
if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false;
return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0);
}
@ -81,11 +84,9 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
#if defined(_WIN32)
// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
// Same for DiscardVirtualMemory. (hide MEM_EXTENDED_PARAMETER to compile with older SDK's)
// (hide MEM_EXTENDED_PARAMETER to compile with older SDK's)
typedef PVOID(__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
typedef DWORD(__stdcall *PDiscardVirtualMemory)(PVOID,SIZE_T);
static PVirtualAlloc2 pVirtualAlloc2 = NULL;
static PDiscardVirtualMemory pDiscardVirtualMemory = NULL;
void _mi_os_init(void) {
// get the page size
@ -100,7 +101,6 @@ void _mi_os_init(void) {
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps
pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2");
pDiscardVirtualMemory = (PDiscardVirtualMemory)GetProcAddress(hDll, "DiscardVirtualMemory");
FreeLibrary(hDll);
}
// Try to see if large OS pages are supported
@ -555,17 +555,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
#if defined(_WIN32)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
// (but this is for an access pattern that immediately reuses the memory)
if (mi_option_is_enabled(mi_option_reset_discards) && pDiscardVirtualMemory != NULL) {
DWORD ok = (*pDiscardVirtualMemory)(start, csize);
mi_assert_internal(ok == ERROR_SUCCESS);
if (ok != ERROR_SUCCESS) return false;
}
else {
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start);
if (p != start) return false;
}
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start);
if (p != start) return false;
#else
#if defined(MADV_FREE)
static int advice = MADV_FREE;

2
src/page.c
View File

@ -741,7 +741,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
// huge allocation?
mi_page_t* page;
if (mi_unlikely(size > MI_LARGE_OBJ_SIZE_MAX)) {
if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) {
if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
page = NULL;
}
else {

2
test/test-api.c
View File

@ -66,7 +66,7 @@ bool test_heap2();
// Main testing
// ---------------------------------------------------------------------------
int main() {
mi_option_enable(mi_option_verbose,false);
mi_option_disable(mi_option_verbose);
// ---------------------------------------------------
// Malloc