AuroraMiddleware/mimalloc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reduce memory order constraints for better efficiency on ARM etc

Browse Source
This commit is contained in:
daan 2020-07-26 11:58:02 -07:00
parent 116159cd40
commit a9f46dc86f
3 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
include/mimalloc-atomic.h
View File

@ -27,19 +27,23 @@ terms of the MIT license. A copy of the license can be found in the file
// Atomic operations specialized for mimalloc
// ------------------------------------------------------
// Atomically add a value; returns the previous value. Memory ordering is acquire-release.
// Atomically add a value; returns the previous value. Memory ordering is relaxed.
static inline uintptr_t mi_atomic_add(_Atomic(uintptr_t)* p, uintptr_t add);
// Atomically "and" a value; returns the previous value. Memory ordering is acquire-release.
// Atomically "and" a value; returns the previous value. Memory ordering is release.
static inline uintptr_t mi_atomic_and(_Atomic(uintptr_t)* p, uintptr_t x);
// Atomically "or" a value; returns the previous value. Memory ordering is acquire-release.
// Atomically "or" a value; returns the previous value. Memory ordering is release.
static inline uintptr_t mi_atomic_or(_Atomic(uintptr_t)* p, uintptr_t x);
// Atomically compare and exchange a value; returns `true` if successful.
// May fail spuriously. Memory ordering is acquire-release; with acquire on failure.
// May fail spuriously. Memory ordering is release; with relaxed on failure.
static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired);
// Atomically compare and exchange a value; returns `true` if successful.
// May fail spuriously. Memory ordering is acquire-release; with acquire on failure.
static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired);
// Atomically compare and exchange a value; returns `true` if successful.
// Memory ordering is acquire-release; with acquire on failure.
static inline bool mi_atomic_cas_strong(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired);
@ -180,6 +184,9 @@ static inline bool mi_atomic_cas_strong(_Atomic(uintptr_t)* p, uintptr_t* expect
static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired) {
return mi_atomic_cas_strong(p,expected,desired);
}
static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired) {
return mi_atomic_cas_strong(p, expected, desired);
}
static inline uintptr_t mi_atomic_exchange(_Atomic(uintptr_t)* p, uintptr_t exchange) {
return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
}
@ -225,17 +232,21 @@ static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t
#endif
static inline uintptr_t mi_atomic_add(_Atomic(uintptr_t)* p, uintptr_t add) {
MI_USING_STD
return atomic_fetch_add_explicit(p, add, memory_order_acq_rel);
return atomic_fetch_add_explicit(p, add, memory_order_relaxed);
}
static inline uintptr_t mi_atomic_and(_Atomic(uintptr_t)* p, uintptr_t x) {
MI_USING_STD
return atomic_fetch_and_explicit(p, x, memory_order_acq_rel);
return atomic_fetch_and_explicit(p, x, memory_order_release);
}
static inline uintptr_t mi_atomic_or(_Atomic(uintptr_t)* p, uintptr_t x) {
MI_USING_STD
return atomic_fetch_or_explicit(p, x, memory_order_acq_rel);
return atomic_fetch_or_explicit(p, x, memory_order_release);
}
static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired) {
MI_USING_STD
return atomic_compare_exchange_weak_explicit(p, expected, desired, memory_order_release, memory_order_relaxed);
}
static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired) {
MI_USING_STD
return atomic_compare_exchange_weak_explicit(p, expected, desired, memory_order_acq_rel, memory_order_acquire);
}
@ -266,7 +277,7 @@ static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add)
static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
MI_USING_STD
int64_t current = atomic_load_explicit((_Atomic(int64_t)*)p, memory_order_relaxed);
while (current < x && !atomic_compare_exchange_weak_explicit((_Atomic(int64_t)*)p, &current, x, memory_order_acq_rel, memory_order_acquire)) { /* nothing */ };
while (current < x && !atomic_compare_exchange_weak_explicit((_Atomic(int64_t)*)p, &current, x, memory_order_release, memory_order_relaxed)) { /* nothing */ };
}
#endif

2
src/page.c
View File

@ -159,7 +159,7 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
do {
head = mi_tf_block(tfree);
tfreex = mi_tf_set_block(tfree,NULL);
} while (!mi_atomic_cas_weak(&page->xthread_free, &tfree, tfreex));
} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));
// return if the list is empty
if (head == NULL) return;

2
src/segment.c
View File

@ -982,7 +982,7 @@ static mi_segment_t* mi_abandoned_pop(void) {
mi_segment_t* anext = mi_atomic_read_ptr_relaxed(mi_segment_t, &segment->abandoned_next);
next = mi_tagged_segment(anext, ts); // note: reads the segment's `abandoned_next` field so should not be decommitted
}
} while (segment != NULL && !mi_atomic_cas_weak(&abandoned, &ts, next));
} while (segment != NULL && !mi_atomic_cas_weak_acq_rel(&abandoned, &ts, next));
mi_atomic_decrement(&abandoned_readers); // release reader lock
if (segment != NULL) {
mi_atomic_write_ptr(mi_segment_t, &segment->abandoned_next, NULL);