File Manager

003 File Manager

Current Path: /usr/src/contrib/jemalloc/include/jemalloc/internal

usr / src / contrib / jemalloc / include / jemalloc / internal /

📁 ..
📄 arena_externs.h(4.83 KB)
📄 arena_inlines_a.h(1.44 KB)
📄 arena_inlines_b.h(11.54 KB)
📄 arena_stats.h(7.92 KB)
📄 arena_structs_a.h(293 B)
📄 arena_structs_b.h(6.63 KB)
📄 arena_types.h(1.67 KB)
📄 assert.h(1.3 KB)
📄 atomic.h(2.66 KB)
📄 atomic_c11.h(3.46 KB)
📄 atomic_gcc_atomic.h(4.06 KB)
📄 atomic_gcc_sync.h(6.21 KB)
📄 background_thread_externs.h(1.29 KB)
📄 background_thread_inlines.h(2.09 KB)
📄 background_thread_structs.h(1.74 KB)
📄 base_externs.h(946 B)
📄 base_inlines.h(312 B)
📄 base_structs.h(1.52 KB)
📄 base_types.h(1.08 KB)
📄 bin.h(3.6 KB)
📄 bin_stats.h(1.35 KB)
📄 bin_types.h(474 B)
📄 bit_util.h(5.22 KB)
📄 bitmap.h(10.97 KB)
📄 cache_bin.h(3.69 KB)
📄 ckh.h(3.19 KB)
📄 ctl.h(3.62 KB)
📄 div.h(1.14 KB)
📄 emitter.h(13.06 KB)
📄 extent_dss.h(728 B)
📄 extent_externs.h(3.75 KB)
📄 extent_inlines.h(13.4 KB)
📄 extent_mmap.h(328 B)
📄 extent_structs.h(8.46 KB)
📄 extent_types.h(700 B)
📄 hash.h(7.88 KB)
📄 hook.h(5.41 KB)
📄 jemalloc_internal_decls.h(2.19 KB)
📄 jemalloc_internal_defs.h(10.93 KB)
📄 jemalloc_internal_defs_FreeBSD.h(256 B)
📄 jemalloc_internal_externs.h(1.75 KB)
📄 jemalloc_internal_includes.h(4.19 KB)
📄 jemalloc_internal_inlines_a.h(4.39 KB)
📄 jemalloc_internal_inlines_b.h(2.24 KB)
📄 jemalloc_internal_inlines_c.h(7.08 KB)
📄 jemalloc_internal_macros.h(3.99 KB)
📄 jemalloc_internal_types.h(3.42 KB)
📄 jemalloc_preamble.h(3.9 KB)
📄 large_externs.h(1.4 KB)
📄 log.h(3.67 KB)
📄 malloc_io.h(2.9 KB)
📄 mutex.h(9.34 KB)
📄 mutex_pool.h(2.81 KB)
📄 mutex_prof.h(3.55 KB)
📄 nstime.h(1.27 KB)
📄 pages.h(3.05 KB)
📄 ph.h(12.26 KB)
📄 private_namespace.h(22.72 KB)
📄 prng.h(4.69 KB)
📄 prof_externs.h(3.79 KB)
📄 prof_inlines_a.h(2.44 KB)
📄 prof_inlines_b.h(6.42 KB)
📄 prof_structs.h(5.02 KB)
📄 prof_types.h(1.74 KB)
📄 public_namespace.h(1.03 KB)
📄 ql.h(2.45 KB)
📄 qr.h(2.29 KB)
📄 quantum.h(1.75 KB)
📄 rb.h(37.41 KB)
📄 rtree.h(17.01 KB)
📄 rtree_tsd.h(1.82 KB)
📄 safety_check.h(797 B)
📄 sc.h(12.98 KB)
📄 seq.h(1.81 KB)
📄 smoothstep.h(15.26 KB)
📄 spin.h(613 B)
📄 stats.h(998 B)
📄 sz.h(7.96 KB)
📄 tcache_externs.h(2.09 KB)
📄 tcache_inlines.h(5.73 KB)
📄 tcache_structs.h(2.05 KB)
📄 tcache_types.h(2.01 KB)
📄 test_hooks.h(459 B)
📄 ticker.h(2.21 KB)
📄 tsd.h(12.49 KB)
📄 tsd_generic.h(3.76 KB)
📄 tsd_malloc_thread_cleanup.h(1.19 KB)
📄 tsd_tls.h(1.12 KB)
📄 tsd_types.h(258 B)
📄 util.h(1.43 KB)
📄 witness.h(9.32 KB)

Editing: mutex.h

#ifndef JEMALLOC_INTERNAL_MUTEX_H
#define JEMALLOC_INTERNAL_MUTEX_H

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/witness.h"

typedef enum {
	/* Can only acquire one mutex of a given witness rank at a time. */
	malloc_mutex_rank_exclusive,
	/*
	 * Can acquire multiple mutexes of the same witness rank, but in
	 * address-ascending order only.
	 */
	malloc_mutex_address_ordered
} malloc_mutex_lock_order_t;

typedef struct malloc_mutex_s malloc_mutex_t;
struct malloc_mutex_s {
	union {
		struct {
			/*
			 * prof_data is defined first to reduce cacheline
			 * bouncing: the data is not touched by the mutex holder
			 * during unlocking, while might be modified by
			 * contenders.  Having it before the mutex itself could
			 * avoid prefetching a modified cacheline (for the
			 * unlocking thread).
			 */
			mutex_prof_data_t	prof_data;
#ifdef _WIN32
#  if _WIN32_WINNT >= 0x0600
			SRWLOCK         	lock;
#  else
			CRITICAL_SECTION	lock;
#  endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
			os_unfair_lock		lock;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
			pthread_mutex_t		lock;
			malloc_mutex_t		*postponed_next;
#else
			pthread_mutex_t		lock;
#endif
			/* 
			 * Hint flag to avoid exclusive cache line contention
			 * during spin waiting
			 */
			atomic_b_t		locked;
		};
		/*
		 * We only touch witness when configured w/ debug.  However we
		 * keep the field in a union when !debug so that we don't have
		 * to pollute the code base with #ifdefs, while avoid paying the
		 * memory cost.
		 */
#if !defined(JEMALLOC_DEBUG)
		witness_t			witness;
		malloc_mutex_lock_order_t	lock_order;
#endif
	};

#if defined(JEMALLOC_DEBUG)
	witness_t			witness;
	malloc_mutex_lock_order_t	lock_order;
#endif
};

/*
 * Based on benchmark results, a fixed spin with this amount of retries works
 * well for our critical sections.
 */
#define MALLOC_MUTEX_MAX_SPIN 250

#ifdef _WIN32
#  if _WIN32_WINNT >= 0x0600
#    define MALLOC_MUTEX_LOCK(m)    AcquireSRWLockExclusive(&(m)->lock)
#    define MALLOC_MUTEX_UNLOCK(m)  ReleaseSRWLockExclusive(&(m)->lock)
#    define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
#  else
#    define MALLOC_MUTEX_LOCK(m)    EnterCriticalSection(&(m)->lock)
#    define MALLOC_MUTEX_UNLOCK(m)  LeaveCriticalSection(&(m)->lock)
#    define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
#  endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
#    define MALLOC_MUTEX_LOCK(m)    os_unfair_lock_lock(&(m)->lock)
#    define MALLOC_MUTEX_UNLOCK(m)  os_unfair_lock_unlock(&(m)->lock)
#    define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
#else
#    define MALLOC_MUTEX_LOCK(m)    pthread_mutex_lock(&(m)->lock)
#    define MALLOC_MUTEX_UNLOCK(m)  pthread_mutex_unlock(&(m)->lock)
#    define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
#endif

#define LOCK_PROF_DATA_INITIALIZER					\
    {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0,		\
	    ATOMIC_INIT(0), 0, NULL, 0}

#ifdef _WIN32
#  define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
#  if defined(JEMALLOC_DEBUG)
#    define MALLOC_MUTEX_INITIALIZER					\
  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
         WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
#  else
#    define MALLOC_MUTEX_INITIALIZER                      \
  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}},  \
      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
#  endif
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
#  if (defined(JEMALLOC_DEBUG))
#     define MALLOC_MUTEX_INITIALIZER					\
      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
#  else
#     define MALLOC_MUTEX_INITIALIZER					\
      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
#  endif

#else
#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
#  if defined(JEMALLOC_DEBUG)
#    define MALLOC_MUTEX_INITIALIZER					\
     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
#  else
#    define MALLOC_MUTEX_INITIALIZER                          \
     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}},	\
      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
#  endif
#endif

#ifdef JEMALLOC_LAZY_LOCK
extern bool isthreaded;
#endif

bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
    witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
bool malloc_mutex_first_thread(void);
bool malloc_mutex_boot(void);
void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);

void malloc_mutex_lock_slow(malloc_mutex_t *mutex);

static inline void
malloc_mutex_lock_final(malloc_mutex_t *mutex) {
	MALLOC_MUTEX_LOCK(mutex);
	atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
}

static inline bool
malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
	return MALLOC_MUTEX_TRYLOCK(mutex);
}

static inline void
mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	if (config_stats) {
		mutex_prof_data_t *data = &mutex->prof_data;
		data->n_lock_ops++;
		if (data->prev_owner != tsdn) {
			data->prev_owner = tsdn;
			data->n_owner_switches++;
		}
	}
}

/* Trylock: return false if the lock is successfully acquired. */
static inline bool
malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
	if (isthreaded) {
		if (malloc_mutex_trylock_final(mutex)) {
			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
			return true;
		}
		mutex_owner_stats_update(tsdn, mutex);
	}
	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);

return false;
}

/* Aggregate lock prof data. */
static inline void
malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
	nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
	if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
		nstime_copy(&sum->max_wait_time, &data->max_wait_time);
	}

sum->n_wait_times += data->n_wait_times;
	sum->n_spin_acquired += data->n_spin_acquired;

if (sum->max_n_thds < data->max_n_thds) {
		sum->max_n_thds = data->max_n_thds;
	}
	uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
	    ATOMIC_RELAXED);
	uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
	    &data->n_waiting_thds, ATOMIC_RELAXED);
	atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
	    ATOMIC_RELAXED);
	sum->n_owner_switches += data->n_owner_switches;
	sum->n_lock_ops += data->n_lock_ops;
}

static inline void
malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
	if (isthreaded) {
		if (malloc_mutex_trylock_final(mutex)) {
			malloc_mutex_lock_slow(mutex);
			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
		}
		mutex_owner_stats_update(tsdn, mutex);
	}
	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
}

static inline void
malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
	witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
	if (isthreaded) {
		MALLOC_MUTEX_UNLOCK(mutex);
	}
}

static inline void
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
}

static inline void
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
}

/* Copy the prof data from mutex for processing. */
static inline void
malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
    malloc_mutex_t *mutex) {
	mutex_prof_data_t *source = &mutex->prof_data;
	/* Can only read holding the mutex. */
	malloc_mutex_assert_owner(tsdn, mutex);

/*
	 * Not *really* allowed (we shouldn't be doing non-atomic loads of
	 * atomic data), but the mutex protection makes this safe, and writing
	 * a member-for-member copy is tedious for this situation.
	 */
	*data = *source;
	/* n_wait_thds is not reported (modified w/o locking). */
	atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
}

static inline void
malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
    malloc_mutex_t *mutex) {
	mutex_prof_data_t *source = &mutex->prof_data;
	/* Can only read holding the mutex. */
	malloc_mutex_assert_owner(tsdn, mutex);

nstime_add(&data->tot_wait_time, &source->tot_wait_time);
	if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
		nstime_copy(&data->max_wait_time, &source->max_wait_time);
	}
	data->n_wait_times += source->n_wait_times;
	data->n_spin_acquired += source->n_spin_acquired;
	if (data->max_n_thds < source->max_n_thds) {
		data->max_n_thds = source->max_n_thds;
	}
	/* n_wait_thds is not reported. */
	atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
	data->n_owner_switches += source->n_owner_switches;
	data->n_lock_ops += source->n_lock_ops;
}

#endif /* JEMALLOC_INTERNAL_MUTEX_H */

003 File Manager

Editing: mutex.h

Upload File

Create Folder