mirror of
https://github.com/fluencelabs/redis
synced 2025-06-20 04:26:31 +00:00
Jemalloc upgraded to version 5.0.1.
This commit is contained in:
antirez
201
deps/jemalloc/src/mutex.c
vendored
201
deps/jemalloc/src/mutex.c
vendored
@ -1,12 +1,13 @@
|
||||
#define JEMALLOC_MUTEX_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
#define JEMALLOC_MUTEX_C_
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_includes.h"
|
||||
|
||||
#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
#include "jemalloc/internal/assert.h"
|
||||
#include "jemalloc/internal/malloc_io.h"
|
||||
#include "jemalloc/internal/spin.h"
|
||||
|
||||
#ifndef _CRT_SPINCOUNT
|
||||
#define _CRT_SPINCOUNT 4000
|
||||
#define _CRT_SPINCOUNT 4000
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
@ -20,10 +21,6 @@ static bool postpone_init = true;
|
||||
static malloc_mutex_t *postponed_mutexes = NULL;
|
||||
#endif
|
||||
|
||||
#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
|
||||
static void pthread_create_once(void);
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
/*
|
||||
* We intercept pthread_create() calls in order to toggle isthreaded if the
|
||||
@ -31,33 +28,11 @@ static void pthread_create_once(void);
|
||||
*/
|
||||
|
||||
#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
|
||||
static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
|
||||
void *(*)(void *), void *__restrict);
|
||||
|
||||
static void
|
||||
pthread_create_once(void)
|
||||
{
|
||||
|
||||
pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
|
||||
if (pthread_create_fptr == NULL) {
|
||||
malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
|
||||
"\"pthread_create\")\n");
|
||||
abort();
|
||||
}
|
||||
|
||||
isthreaded = true;
|
||||
}
|
||||
|
||||
JEMALLOC_EXPORT int
|
||||
pthread_create(pthread_t *__restrict thread,
|
||||
const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
|
||||
void *__restrict arg)
|
||||
{
|
||||
static pthread_once_t once_control = PTHREAD_ONCE_INIT;
|
||||
|
||||
pthread_once(&once_control, pthread_create_once);
|
||||
|
||||
return (pthread_create_fptr(thread, attr, start_routine, arg));
|
||||
void *__restrict arg) {
|
||||
return pthread_create_wrapper(thread, attr, start_routine, arg);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -68,18 +43,108 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
|
||||
void *(calloc_cb)(size_t, size_t));
|
||||
#endif
|
||||
|
||||
bool
|
||||
malloc_mutex_init(malloc_mutex_t *mutex)
|
||||
{
|
||||
void
|
||||
malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
|
||||
mutex_prof_data_t *data = &mutex->prof_data;
|
||||
UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER;
|
||||
|
||||
if (ncpus == 1) {
|
||||
goto label_spin_done;
|
||||
}
|
||||
|
||||
int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN;
|
||||
do {
|
||||
spin_cpu_spinwait();
|
||||
if (!malloc_mutex_trylock_final(mutex)) {
|
||||
data->n_spin_acquired++;
|
||||
return;
|
||||
}
|
||||
} while (cnt++ < max_cnt);
|
||||
|
||||
if (!config_stats) {
|
||||
/* Only spin is useful when stats is off. */
|
||||
malloc_mutex_lock_final(mutex);
|
||||
return;
|
||||
}
|
||||
label_spin_done:
|
||||
nstime_update(&before);
|
||||
/* Copy before to after to avoid clock skews. */
|
||||
nstime_t after;
|
||||
nstime_copy(&after, &before);
|
||||
uint32_t n_thds = atomic_fetch_add_u32(&data->n_waiting_thds, 1,
|
||||
ATOMIC_RELAXED) + 1;
|
||||
/* One last try as above two calls may take quite some cycles. */
|
||||
if (!malloc_mutex_trylock_final(mutex)) {
|
||||
atomic_fetch_sub_u32(&data->n_waiting_thds, 1, ATOMIC_RELAXED);
|
||||
data->n_spin_acquired++;
|
||||
return;
|
||||
}
|
||||
|
||||
/* True slow path. */
|
||||
malloc_mutex_lock_final(mutex);
|
||||
/* Update more slow-path only counters. */
|
||||
atomic_fetch_sub_u32(&data->n_waiting_thds, 1, ATOMIC_RELAXED);
|
||||
nstime_update(&after);
|
||||
|
||||
nstime_t delta;
|
||||
nstime_copy(&delta, &after);
|
||||
nstime_subtract(&delta, &before);
|
||||
|
||||
data->n_wait_times++;
|
||||
nstime_add(&data->tot_wait_time, &delta);
|
||||
if (nstime_compare(&data->max_wait_time, &delta) < 0) {
|
||||
nstime_copy(&data->max_wait_time, &delta);
|
||||
}
|
||||
if (n_thds > data->max_n_thds) {
|
||||
data->max_n_thds = n_thds;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
mutex_prof_data_init(mutex_prof_data_t *data) {
|
||||
memset(data, 0, sizeof(mutex_prof_data_t));
|
||||
nstime_init(&data->max_wait_time, 0);
|
||||
nstime_init(&data->tot_wait_time, 0);
|
||||
data->prev_owner = NULL;
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex) {
|
||||
malloc_mutex_assert_owner(tsdn, mutex);
|
||||
mutex_prof_data_init(&mutex->prof_data);
|
||||
}
|
||||
|
||||
static int
|
||||
mutex_addr_comp(const witness_t *witness1, void *mutex1,
|
||||
const witness_t *witness2, void *mutex2) {
|
||||
assert(mutex1 != NULL);
|
||||
assert(mutex2 != NULL);
|
||||
uintptr_t mu1int = (uintptr_t)mutex1;
|
||||
uintptr_t mu2int = (uintptr_t)mutex2;
|
||||
if (mu1int < mu2int) {
|
||||
return -1;
|
||||
} else if (mu1int == mu2int) {
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
|
||||
witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {
|
||||
mutex_prof_data_init(&mutex->prof_data);
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
InitializeSRWLock(&mutex->lock);
|
||||
# else
|
||||
if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
|
||||
_CRT_SPINCOUNT))
|
||||
return (true);
|
||||
_CRT_SPINCOUNT)) {
|
||||
return true;
|
||||
}
|
||||
# endif
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
mutex->lock = OS_UNFAIR_LOCK_INIT;
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
mutex->lock = 0;
|
||||
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
|
||||
@ -88,66 +153,72 @@ malloc_mutex_init(malloc_mutex_t *mutex)
|
||||
postponed_mutexes = mutex;
|
||||
} else {
|
||||
if (_pthread_mutex_init_calloc_cb(&mutex->lock,
|
||||
bootstrap_calloc) != 0)
|
||||
return (true);
|
||||
bootstrap_calloc) != 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
#else
|
||||
pthread_mutexattr_t attr;
|
||||
|
||||
if (pthread_mutexattr_init(&attr) != 0)
|
||||
return (true);
|
||||
if (pthread_mutexattr_init(&attr) != 0) {
|
||||
return true;
|
||||
}
|
||||
pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
|
||||
if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
|
||||
pthread_mutexattr_destroy(&attr);
|
||||
return (true);
|
||||
return true;
|
||||
}
|
||||
pthread_mutexattr_destroy(&attr);
|
||||
#endif
|
||||
return (false);
|
||||
if (config_debug) {
|
||||
mutex->lock_order = lock_order;
|
||||
if (lock_order == malloc_mutex_address_ordered) {
|
||||
witness_init(&mutex->witness, name, rank,
|
||||
mutex_addr_comp, mutex);
|
||||
} else {
|
||||
witness_init(&mutex->witness, name, rank, NULL, NULL);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_prefork(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(mutex);
|
||||
malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex) {
|
||||
malloc_mutex_lock(tsdn, mutex);
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
malloc_mutex_unlock(mutex);
|
||||
malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex) {
|
||||
malloc_mutex_unlock(tsdn, mutex);
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_postfork_child(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) {
|
||||
#ifdef JEMALLOC_MUTEX_INIT_CB
|
||||
malloc_mutex_unlock(mutex);
|
||||
malloc_mutex_unlock(tsdn, mutex);
|
||||
#else
|
||||
if (malloc_mutex_init(mutex)) {
|
||||
if (malloc_mutex_init(mutex, mutex->witness.name,
|
||||
mutex->witness.rank, mutex->lock_order)) {
|
||||
malloc_printf("<jemalloc>: Error re-initializing mutex in "
|
||||
"child\n");
|
||||
if (opt_abort)
|
||||
if (opt_abort) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
mutex_boot(void)
|
||||
{
|
||||
|
||||
malloc_mutex_boot(void) {
|
||||
#ifdef JEMALLOC_MUTEX_INIT_CB
|
||||
postpone_init = false;
|
||||
while (postponed_mutexes != NULL) {
|
||||
if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
|
||||
bootstrap_calloc) != 0)
|
||||
return (true);
|
||||
bootstrap_calloc) != 0) {
|
||||
return true;
|
||||
}
|
||||
postponed_mutexes = postponed_mutexes->postponed_next;
|
||||
}
|
||||
#endif
|
||||
return (false);
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user