#include #include namespace NBalloc { static constexpr size_t ALIVE_SIGNATURE = 0xaULL << 56; static constexpr size_t DISABLED_SIGNATURE = 0xbULL << 56; static constexpr size_t SIGNATURE_MASK = 0xfULL << 56; static constexpr size_t MINIMAL_ALIGNMENT = sizeof(NBalloc::TAllocHeader); static_assert(((MINIMAL_ALIGNMENT - 1) & MINIMAL_ALIGNMENT) == 0, "invalid BALLOC_MINIMAL_ALIGNMENT"); static Y_FORCE_INLINE void* Malloc(size_t size) { TLS& ltls = tls; size = Align(size, sizeof(TAllocHeader)); if (Y_UNLIKELY(ltls.Mode == Empty || ltls.Mode == ToBeEnabled)) { Init(ltls); } if (Y_LIKELY(ltls.Mode != Disabled)) { TAllocHeader* allocHeader = AllocateRaw(size, ALIVE_SIGNATURE); return allocHeader + 1; } else { // ltls.Mode == Disabled const size_t extsize = size + sizeof(TAllocHeader); TAllocHeader* allocHeader = (TAllocHeader*)LibcMalloc(extsize); allocHeader->Encode(allocHeader, size, DISABLED_SIGNATURE); return allocHeader + 1; } } static void Y_FORCE_INLINE Free(void* ptr) { if (ptr == nullptr) { return; } TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; size_t size = allocHeader->AllocSize; const size_t signature = size & SIGNATURE_MASK; if (Y_LIKELY(signature == ALIVE_SIGNATURE)) { allocHeader->AllocSize = 0; // abort later on double free #ifdef DBG_FILL_MEMORY memset(ptr, 0xde, size - signature); #endif FreeRaw(allocHeader->Block); if (NAllocStats::IsEnabled()) { NAllocStats::DecThreadAllocStats(size - signature); } } else if (signature == DISABLED_SIGNATURE) { LibcFree(allocHeader->Block); } else { NMalloc::AbortFromCorruptedAllocator(); } } static bool Y_FORCE_INLINE IsOwnedByBalloc(void* ptr) { TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; size_t size = allocHeader->AllocSize; const size_t signature = size & SIGNATURE_MASK; if (signature == ALIVE_SIGNATURE) { return true; } else if (signature == DISABLED_SIGNATURE) { return false; } NMalloc::AbortFromCorruptedAllocator(); Y_UNREACHABLE(); } static void Y_FORCE_INLINE Disable() { #if defined(_musl_) // just skip it #else tls.Mode = Disabled; #endif } static void Y_FORCE_INLINE Enable() { tls.Mode = ToBeEnabled; } static bool Y_FORCE_INLINE IsDisabled() { return tls.Mode == Disabled; } }; #if defined(Y_COVER_PTR) void* CoverPtr(void* ptr, size_t len) noexcept; void* UncoverPtr(void* ptr) noexcept; #endif extern "C" void* malloc(size_t size) { #if defined(Y_COVER_PTR) return CoverPtr(NBalloc::Malloc(size + 32), size); #else return NBalloc::Malloc(size); #endif } extern "C" void free(void* data) { #if defined(Y_COVER_PTR) NBalloc::Free(UncoverPtr(data)); #else NBalloc::Free(data); #endif } #if defined(USE_INTELCC) || defined(_darwin_) || defined(_freebsd_) || defined(_STLPORT_VERSION) #define OP_THROWNOTHING noexcept #else #define OP_THROWNOTHING #endif void* operator new(size_t size) { #if defined(Y_COVER_PTR) return malloc(size); #else return NBalloc::Malloc(size); #endif } int posix_memalign(void** memptr, const size_t alignment, const size_t size) { #if defined(Y_COVER_PTR) (void)alignment; *memptr = malloc(size); return 0; #else if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void*)) { return EINVAL; } if (alignment <= NBalloc::MINIMAL_ALIGNMENT) { *memptr = NBalloc::Malloc(size); return 0; } size_t bigSize = size + alignment - NBalloc::MINIMAL_ALIGNMENT; void* res = NBalloc::Malloc(bigSize); void* alignedPtr = (void*)NBalloc::Align((size_t)res, alignment); if (alignedPtr != res) { auto oldAllocHeader = (NBalloc::TAllocHeader*)res - 1; auto newAllocHeader = (NBalloc::TAllocHeader*)alignedPtr - 1; void* block = oldAllocHeader->Block; newAllocHeader->Encode(block, size, NBalloc::ALIVE_SIGNATURE); } *memptr = alignedPtr; return 0; #endif } void* operator new(size_t size, const std::nothrow_t&) OP_THROWNOTHING { #if defined(Y_COVER_PTR) return malloc(size); #else return NBalloc::Malloc(size); #endif } void operator delete(void* p)OP_THROWNOTHING { #if defined(Y_COVER_PTR) free(p); #else NBalloc::Free(p); #endif } void operator delete(void* p, const std::nothrow_t&)OP_THROWNOTHING { #if defined(Y_COVER_PTR) free(p); #else NBalloc::Free(p); #endif } void* operator new[](size_t size) { #if defined(Y_COVER_PTR) return malloc(size); #else return NBalloc::Malloc(size); #endif } void* operator new[](size_t size, const std::nothrow_t&) OP_THROWNOTHING { #if defined(Y_COVER_PTR) return malloc(size); #else return NBalloc::Malloc(size); #endif } void operator delete[](void* p) OP_THROWNOTHING { #if defined(Y_COVER_PTR) free(p); #else NBalloc::Free(p); #endif } void operator delete[](void* p, const std::nothrow_t&) OP_THROWNOTHING { #if defined(Y_COVER_PTR) free(p); #else NBalloc::Free(p); #endif } extern "C" void* calloc(size_t n, size_t elemSize) { const size_t size = n * elemSize; if (elemSize != 0 && size / elemSize != n) { return nullptr; } #if defined(Y_COVER_PTR) void* result = malloc(size); #else void* result = NBalloc::Malloc(size); #endif if (result) { memset(result, 0, size); } return result; } extern "C" void cfree(void* ptr) { #if defined(Y_COVER_PTR) free(ptr); #else NBalloc::Free(ptr); #endif } #if defined(Y_COVER_PTR) static inline void* DoRealloc(void* oldPtr, size_t newSize) { #else extern "C" void* realloc(void* oldPtr, size_t newSize) { #endif if (!oldPtr) { void* result = NBalloc::Malloc(newSize); return result; } if (newSize == 0) { NBalloc::Free(oldPtr); return nullptr; } void* newPtr = NBalloc::Malloc(newSize); if (!newPtr) { return nullptr; } NBalloc::TAllocHeader* header = (NBalloc::TAllocHeader*)oldPtr - 1; const size_t oldSize = header->AllocSize & ~NBalloc::SIGNATURE_MASK; const size_t signature = header->AllocSize & NBalloc::SIGNATURE_MASK; if (Y_LIKELY((signature == NBalloc::ALIVE_SIGNATURE) || (signature == NBalloc::DISABLED_SIGNATURE))) { memcpy(newPtr, oldPtr, oldSize < newSize ? oldSize : newSize); NBalloc::Free(oldPtr); return newPtr; } NMalloc::AbortFromCorruptedAllocator(); return nullptr; } #if defined(Y_COVER_PTR) extern "C" void* realloc(void* oldPtr, size_t newSize) { if (!oldPtr) { return malloc(newSize); } if (!newSize) { free(oldPtr); return nullptr; } return CoverPtr(DoRealloc(UncoverPtr(oldPtr), newSize + 32), newSize); } #endif // Only for testing purposes. Never use in production. extern "C" bool IsOwnedByBalloc(void* ptr) { return NBalloc::IsOwnedByBalloc(ptr); } extern "C" bool BallocDisabled() { return NBalloc::IsDisabled(); } extern "C" void DisableBalloc() { NBalloc::Disable(); } extern "C" void EnableBalloc() { NBalloc::Enable(); } extern "C" void* memalign(size_t alignment, size_t size) { void* ptr; int res = posix_memalign(&ptr, alignment, size); return res ? nullptr : ptr; } extern "C" void* valloc(size_t size) { return memalign(NBalloc::PAGE_ELEM, size); } #if !defined(_MSC_VER) && !defined(_freebsd_) // Workaround for pthread_create bug in linux. extern "C" void* __libc_memalign(size_t alignment, size_t size) { return memalign(alignment, size); } #endif