AuroraMiddleware/mimalloc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge medium pages

Browse Source
This commit is contained in:
daan 2019-07-12 20:27:56 -07:00
commit a927c07de9
4 changed files with 84 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/mimalloc-internal.h
View File

@ -218,7 +218,7 @@ static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const
mi_assert_internal(diff >= 0 && diff < MI_SEGMENT_SIZE);
uintptr_t idx = (uintptr_t)diff >> segment->page_shift;
mi_assert_internal(idx < segment->capacity);
mi_assert_internal(segment->page_kind == MI_PAGE_SMALL || idx == 0);
mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0);
return &((mi_segment_t*)segment)->pages[idx];
}

13
include/mimalloc-types.h
View File

@ -75,8 +75,9 @@ terms of the MIT license. A copy of the license can be found in the file
// Main tuning parameters for segment and page sizes
// Sizes for 64-bit, divide by two for 32-bit
#define MI_SMALL_PAGE_SHIFT (14 + MI_INTPTR_SHIFT) // 64kb
#define MI_LARGE_PAGE_SHIFT ( 5 + MI_SMALL_PAGE_SHIFT) // 4mb
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb
#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb
// Derived constants
@ -84,12 +85,16 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_SEGMENT_MASK ((uintptr_t)MI_SEGMENT_SIZE - 1)
#define MI_SMALL_PAGE_SIZE (1<<MI_SMALL_PAGE_SHIFT)
#define MI_MEDIUM_PAGE_SIZE (1<<MI_MEDIUM_PAGE_SHIFT)
#define MI_LARGE_PAGE_SIZE (1<<MI_LARGE_PAGE_SHIFT)
#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE)
#define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE)
#define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE)
#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512kb on 64-bit
#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/8) // 64kb on 64-bit
#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512kb on 64-bit
#define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT)
@ -200,6 +205,7 @@ typedef struct mi_page_s {
typedef enum mi_page_kind_e {
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
MI_PAGE_HUGE // huge blocks (>512kb) are put into a single page in a segment of the exact size (but still 2mb aligned)
} mi_page_kind_t;
@ -374,6 +380,7 @@ typedef struct mi_segment_queue_s {
// Segments thread local data
typedef struct mi_segments_tld_s {
mi_segment_queue_t small_free; // queue of segments with free small pages
mi_segment_queue_t medium_free; // queue of segments with free medium pages
size_t count; // current number of segments;
size_t peak_count; // peak number of segments
size_t current_size; // current size of all segments

2
src/init.c
View File

@ -91,7 +91,7 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
static mi_tld_t tld_main = {
0,
&_mi_heap_main,
{ { NULL, NULL }, 0, 0, 0, 0, 0, 0, {NULL,NULL}, tld_main_stats }, // segments
{ { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, 0, 0, {NULL,NULL}, tld_main_stats }, // segments
{ 0, NULL, NULL, 0, tld_main_stats }, // os
{ MI_STATS_NULL } // stats
};

128
src/segment.c
View File

@ -21,7 +21,8 @@ terms of the MIT license. A copy of the license can be found in the file
owns its own segments.
Currently we have:
- small pages (64kb), 32 in one segment
- small pages (64kb), 64 in one segment
- medium pages (512kb), 8 in one segment
- large pages (4mb), 1 in one segment
- huge blocks > MI_LARGE_SIZE_MAX (512kb) are directly allocated by the OS
@ -54,16 +55,6 @@ static bool mi_segment_queue_contains(const mi_segment_queue_t* queue, mi_segmen
}
#endif
// quick test to see if a segment is in the free pages queue
static bool mi_segment_is_in_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
bool in_queue = (segment->next != NULL || segment->prev != NULL || tld->small_free.first == segment);
if (in_queue) {
mi_assert(segment->page_kind == MI_PAGE_SMALL); // for now we only support small pages
mi_assert_expensive(mi_segment_queue_contains(&tld->small_free, segment));
}
return in_queue;
}
static bool mi_segment_queue_is_empty(const mi_segment_queue_t* queue) {
return (queue->first == NULL);
}
@ -104,6 +95,39 @@ static void mi_segment_queue_insert_before(mi_segment_queue_t* queue, mi_segment
else queue->last = segment;
}
static mi_segment_queue_t* mi_segment_free_queue_of_kind(mi_page_kind_t kind, mi_segments_tld_t* tld) {
if (kind == MI_PAGE_SMALL) return &tld->small_free;
else if (kind == MI_PAGE_MEDIUM) return &tld->medium_free;
else return NULL;
}
static mi_segment_queue_t* mi_segment_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
return mi_segment_free_queue_of_kind(segment->page_kind,tld);
}
// remove from free queue if it is in one
static void mi_segment_remove_from_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_segment_queue_t* queue = mi_segment_free_queue(segment,tld); // may be NULL
bool in_queue = (queue!=NULL && (segment->next != NULL || segment->prev != NULL || queue->first == segment));
if (in_queue) {
mi_segment_queue_remove(queue,segment);
}
}
static void mi_segment_insert_in_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_segment_enqueue(mi_segment_free_queue(segment, tld), segment);
}
#if MI_DEBUG > 1
static bool mi_segment_is_in_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_segment_queue_t* queue = mi_segment_free_queue(segment, tld);
bool in_queue = (queue!=NULL && (segment->next != NULL || segment->prev != NULL || queue->first == segment));
if (in_queue) {
mi_assert_expensive(mi_segment_queue_contains(queue, segment));
}
return in_queue;
}
#endif
#if (MI_DEBUG > 1)
static size_t mi_segment_pagesize(mi_segment_t* segment) {
@ -140,8 +164,8 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
// the first page starts after the segment info (and possible guard page)
p += segment->segment_info_size;
psize -= segment->segment_info_size;
// for small objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)
if (block_size > 0 && segment->page_kind == MI_PAGE_SMALL) {
// for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)
if (block_size > 0 && segment->page_kind <= MI_PAGE_MEDIUM) {
size_t adjust = block_size - ((uintptr_t)p % block_size);
if (adjust < block_size) {
p += adjust;
@ -348,8 +372,8 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
// Allocate the segment
mi_segment_t* segment = NULL;
// try to get it from our caches
bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind != MI_PAGE_SMALL);
// try to get it from our caches
bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM);
bool protection_still_good = false;
segment = mi_segment_cache_find(tld,segment_size);
mi_assert_internal(segment == NULL ||
@ -358,14 +382,14 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
if (segment != NULL) {
if (mi_option_is_enabled(mi_option_secure)) {
if (segment->page_kind != page_kind || segment->segment_size != segment_size) {
_mi_os_unprotect(segment, segment->segment_size);
_mi_os_unprotect(segment, segment->segment_size);
}
else {
protection_still_good = true; // otherwise, the guard pages are still in place
}
}
if (!mi_option_is_enabled(mi_option_eager_commit)) {
if (page_kind != MI_PAGE_SMALL) {
if (page_kind > MI_PAGE_MEDIUM) {
_mi_os_commit(segment, segment->segment_size, tld->stats);
}
else {
@ -427,18 +451,10 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) {
//fprintf(stderr,"mimalloc: free segment at %p\n", (void*)segment);
mi_assert(segment != NULL);
if (mi_segment_is_in_free_queue(segment,tld)) {
if (segment->page_kind != MI_PAGE_SMALL) {
fprintf(stderr, "mimalloc: expecting small segment: %i, %p, %p, %p\n", segment->page_kind, segment->prev, segment->next, tld->small_free.first);
fflush(stderr);
}
else {
mi_assert_internal(segment->page_kind == MI_PAGE_SMALL); // for now we only support small pages
mi_assert_expensive(mi_segment_queue_contains(&tld->small_free, segment));
mi_segment_queue_remove(&tld->small_free, segment);
}
}
mi_segment_remove_from_free_queue(segment,tld);
mi_assert_expensive(!mi_segment_queue_contains(&tld->small_free, segment));
mi_assert_expensive(!mi_segment_queue_contains(&tld->medium_free, segment));
mi_assert(segment->next == NULL);
mi_assert(segment->prev == NULL);
_mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size);
@ -493,7 +509,7 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats)
if (page->is_reset) {
page->is_reset = false;
_mi_os_unreset(start, psize, stats);
}
}
}
return page;
}
@ -557,9 +573,9 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
mi_segment_abandon(segment,tld);
}
else if (segment->used + 1 == segment->capacity) {
mi_assert_internal(segment->page_kind == MI_PAGE_SMALL); // for now we only support small pages
// move back to segments small pages free list
mi_segment_enqueue(&tld->small_free, segment);
mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM); // for now we only support small and medium pages
// move back to segments free list
mi_segment_insert_in_free_queue(segment,tld);
}
}
}
@ -582,11 +598,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(segment->abandoned_next == NULL);
mi_assert_expensive(mi_segment_is_valid(segment));
// remove the segment from the free page queue if needed
if (mi_segment_is_in_free_queue(segment,tld)) {
mi_assert(segment->page_kind == MI_PAGE_SMALL); // for now we only support small pages
mi_assert_expensive(mi_segment_queue_contains(&tld->small_free, segment));
mi_segment_queue_remove(&tld->small_free, segment);
}
mi_segment_remove_from_free_queue(segment,tld);
mi_assert_internal(segment->next == NULL && segment->prev == NULL);
// all pages in the segment are abandoned; add it to the abandoned list
segment->thread_id = 0;
@ -639,7 +651,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
mi_assert_internal(segment->next == NULL && segment->prev == NULL);
mi_assert_expensive(mi_segment_is_valid(segment));
_mi_stat_decrease(&tld->stats->segments_abandoned,1);
// add its abandoned pages to the current thread
mi_assert(segment->abandoned == segment->used);
for (size_t i = 0; i < segment->capacity; i++) {
@ -665,8 +677,8 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
else {
reclaimed++;
// add its free pages to the the current thread free small segment queue
if (segment->page_kind == MI_PAGE_SMALL && mi_segment_has_free(segment)) {
mi_segment_enqueue(&tld->small_free, segment);
if (segment->page_kind <= MI_PAGE_MEDIUM && mi_segment_has_free(segment)) {
mi_segment_insert_in_free_queue(segment,tld);
}
}
}
@ -680,7 +692,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
// Allocate a small page inside a segment.
// Requires that the page has free pages
static mi_page_t* mi_segment_small_page_alloc_in(mi_segment_t* segment, mi_segments_tld_t* tld) {
static mi_page_t* mi_segment_page_alloc_in(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(mi_segment_has_free(segment));
mi_page_t* page = mi_segment_find_free(segment, tld->stats);
page->segment_in_use = true;
@ -689,22 +701,29 @@ static mi_page_t* mi_segment_small_page_alloc_in(mi_segment_t* segment, mi_segme
if (segment->used == segment->capacity) {
// if no more free pages, remove from the queue
mi_assert_internal(!mi_segment_has_free(segment));
mi_assert_expensive(mi_segment_queue_contains(&tld->small_free, segment));
mi_segment_queue_remove(&tld->small_free, segment);
mi_segment_remove_from_free_queue(segment,tld);
}
return page;
}
static mi_page_t* mi_segment_small_page_alloc(mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
if (mi_segment_queue_is_empty(&tld->small_free)) {
mi_segment_t* segment = mi_segment_alloc(0,MI_PAGE_SMALL,MI_SMALL_PAGE_SHIFT,tld,os_tld);
static mi_page_t* mi_segment_page_alloc(mi_page_kind_t kind, size_t page_shift, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_segment_queue_t* free_queue = mi_segment_free_queue_of_kind(kind,tld);
if (mi_segment_queue_is_empty(free_queue)) {
mi_segment_t* segment = mi_segment_alloc(0,kind,page_shift,tld,os_tld);
if (segment == NULL) return NULL;
mi_segment_enqueue(&tld->small_free, segment);
mi_segment_enqueue(free_queue, segment);
}
mi_assert_internal(tld->small_free.first != NULL);
return mi_segment_small_page_alloc_in(tld->small_free.first,tld);
mi_assert_internal(free_queue->first != NULL);
return mi_segment_page_alloc_in(free_queue->first,tld);
}
static mi_page_t* mi_segment_small_page_alloc(mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
return mi_segment_page_alloc(MI_PAGE_SMALL,MI_SMALL_PAGE_SHIFT,tld,os_tld);
}
static mi_page_t* mi_segment_medium_page_alloc(mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
return mi_segment_page_alloc(MI_PAGE_MEDIUM, MI_MEDIUM_PAGE_SHIFT, tld, os_tld);
}
/* -----------------------------------------------------------
large page allocation
@ -736,17 +755,14 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld
mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_page_t* page;
if (block_size <= (MI_SMALL_PAGE_SIZE / 8)) {
// smaller blocks than 8kb (assuming MI_SMALL_PAGE_SIZE == 64kb)
if (block_size <= (MI_SMALL_PAGE_SIZE/16)*3) {
page = mi_segment_small_page_alloc(tld,os_tld);
}
else if (block_size <= (MI_SMALL_PAGE_SIZE/2) && (MI_SMALL_PAGE_SIZE % block_size) <= (MI_SMALL_PAGE_SIZE / 8)) {
// use small page too if it happens to fit well
page = mi_segment_small_page_alloc(tld, os_tld);
else if (block_size <= (MI_MEDIUM_PAGE_SIZE/16)*3) {
page = mi_segment_medium_page_alloc(tld, os_tld);
}
else if (block_size < (MI_LARGE_SIZE_MAX - sizeof(mi_segment_t))) {
// otherwise use a large page
page = mi_segment_large_page_alloc(tld, os_tld);
page = mi_segment_large_page_alloc(tld, os_tld);
}
else {
page = mi_segment_huge_page_alloc(block_size,tld,os_tld);