ydb/contrib/libs/jemalloc/src/hook.c

#include "jemalloc/internal/jemalloc_preamble.h"

#include "jemalloc/internal/hook.h"

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/seq.h"

typedef struct hooks_internal_s hooks_internal_t;
struct hooks_internal_s {
	hooks_t hooks;
	bool in_use;
};

seq_define(hooks_internal_t, hooks)

static atomic_u_t nhooks = ATOMIC_INIT(0);
static seq_hooks_t hooks[HOOK_MAX];
static malloc_mutex_t hooks_mu;

bool
hook_boot() {
	return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK,
	    malloc_mutex_rank_exclusive);
}

static void *
hook_install_locked(hooks_t *to_install) {
	hooks_internal_t hooks_internal;
	for (int i = 0; i < HOOK_MAX; i++) {
		bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
		/* We hold mu; no concurrent access. */
		assert(success);
		if (!hooks_internal.in_use) {
			hooks_internal.hooks = *to_install;
			hooks_internal.in_use = true;
			seq_store_hooks(&hooks[i], &hooks_internal);
			atomic_store_u(&nhooks,
			    atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1,
			    ATOMIC_RELAXED);
			return &hooks[i];
		}
	}
	return NULL;
}

void *
hook_install(tsdn_t *tsdn, hooks_t *to_install) {
	malloc_mutex_lock(tsdn, &hooks_mu);
	void *ret = hook_install_locked(to_install);
	if (ret != NULL) {
		tsd_global_slow_inc(tsdn);
	}
	malloc_mutex_unlock(tsdn, &hooks_mu);
	return ret;
}

static void
hook_remove_locked(seq_hooks_t *to_remove) {
	hooks_internal_t hooks_internal;
	bool success = seq_try_load_hooks(&hooks_internal, to_remove);
	/* We hold mu; no concurrent access. */
	assert(success);
	/* Should only remove hooks that were added. */
	assert(hooks_internal.in_use);
	hooks_internal.in_use = false;
	seq_store_hooks(to_remove, &hooks_internal);
	atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1,
	    ATOMIC_RELAXED);
}

void
hook_remove(tsdn_t *tsdn, void *opaque) {
	if (config_debug) {
		char *hooks_begin = (char *)&hooks[0];
		char *hooks_end = (char *)&hooks[HOOK_MAX];
		char *hook = (char *)opaque;
		assert(hooks_begin <= hook && hook < hooks_end
		    && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
	}
	malloc_mutex_lock(tsdn, &hooks_mu);
	hook_remove_locked((seq_hooks_t *)opaque);
	tsd_global_slow_dec(tsdn);
	malloc_mutex_unlock(tsdn, &hooks_mu);
}

#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr)				\
for (int for_each_hook_counter = 0;					\
    for_each_hook_counter < HOOK_MAX;					\
    for_each_hook_counter++) {						\
	bool for_each_hook_success = seq_try_load_hooks(		\
	    (hooks_internal_ptr), &hooks[for_each_hook_counter]);	\
	if (!for_each_hook_success) {					\
		continue;						\
	}								\
	if (!(hooks_internal_ptr)->in_use) {				\
		continue;						\
	}
#define FOR_EACH_HOOK_END						\
}

static bool *
hook_reentrantp() {
	/*
	 * We prevent user reentrancy within hooks.  This is basically just a
	 * thread-local bool that triggers an early-exit.
	 *
	 * We don't fold in_hook into reentrancy.  There are two reasons for
	 * this:
	 * - Right now, we turn on reentrancy during things like extent hook
	 *   execution.  Allocating during extent hooks is not officially
	 *   supported, but we don't want to break it for the time being.  These
	 *   sorts of allocations should probably still be hooked, though.
	 * - If a hook allocates, we may want it to be relatively fast (after
	 *   all, it executes on every allocator operation).  Turning on
	 *   reentrancy is a fairly heavyweight mode (disabling tcache,
	 *   redirecting to arena 0, etc.).  It's possible we may one day want
	 *   to turn on reentrant mode here, if it proves too difficult to keep
	 *   this working.  But that's fairly easy for us to see; OTOH, people
	 *   not using hooks because they're too slow is easy for us to miss.
	 *
	 * The tricky part is
	 * that this code might get invoked even if we don't have access to tsd.
	 * This function mimics getting a pointer to thread-local data, except
	 * that it might secretly return a pointer to some global data if we
	 * know that the caller will take the early-exit path.
	 * If we return a bool that indicates that we are reentrant, then the
	 * caller will go down the early exit path, leaving the global
	 * untouched.
	 */
	static bool in_hook_global = true;
	tsdn_t *tsdn = tsdn_fetch();
	bool *in_hook = tsdn_in_hookp_get(tsdn);
	if (in_hook!= NULL) {
		return in_hook;
	}
	return &in_hook_global;
}

#define HOOK_PROLOGUE							\
	if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) {	\
		return;							\
	}								\
	bool *in_hook = hook_reentrantp();				\
	if (*in_hook) {							\
		return;							\
	}								\
	*in_hook = true;

#define HOOK_EPILOGUE							\
	*in_hook = false;

void
hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
    uintptr_t args_raw[3]) {
	HOOK_PROLOGUE

	hooks_internal_t hook;
	FOR_EACH_HOOK_BEGIN(&hook)
		hook_alloc h = hook.hooks.alloc_hook;
		if (h != NULL) {
			h(hook.hooks.extra, type, result, result_raw, args_raw);
		}
	FOR_EACH_HOOK_END

	HOOK_EPILOGUE
}

void
hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
	HOOK_PROLOGUE
	hooks_internal_t hook;
	FOR_EACH_HOOK_BEGIN(&hook)
		hook_dalloc h = hook.hooks.dalloc_hook;
		if (h != NULL) {
			h(hook.hooks.extra, type, address, args_raw);
		}
	FOR_EACH_HOOK_END
	HOOK_EPILOGUE
}

void
hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
	HOOK_PROLOGUE
	hooks_internal_t hook;
	FOR_EACH_HOOK_BEGIN(&hook)
		hook_expand h = hook.hooks.expand_hook;
		if (h != NULL) {
			h(hook.hooks.extra, type, address, old_usize, new_usize,
			    result_raw, args_raw);
		}
	FOR_EACH_HOOK_END
	HOOK_EPILOGUE
}