#pragma once #include #include #ifndef NALF_ALLOC_DEFAULTMODE #define NALF_ALLOC_DEFAULTMODE (TAllocHint::Chunked) #endif #ifndef NALF_ALLOC_DEFAULTALIGN #define NALF_ALLOC_DEFAULTALIGN (16) #endif #if defined(_tsan_enabled_) || defined(_msan_enabled_) || defined(_asan_enabled_) || defined(WITH_VALGRIND) #define NALF_FORCE_MALLOC_FREE 1 #define NALF_DONOT_DEFINE_GLOBALS 1 #endif namespace NNumaAwareLockFreeAllocator { struct TAllocHint { enum EHint { Undefined, Incremental, Chunked, System, ForceIncremental, ForceChunked, ForceSystem, Bootstrap, }; // valid op hint values: incremental, chunked, system, force* // valid thread hint values: undefined, incremental, chunked, system // bootstrap is used in node initialization only }; class TPerThreadAllocator; TPerThreadAllocator* GetThreadAllocator(); void* Allocate(ui64 len, TAllocHint::EHint hint = NALF_ALLOC_DEFAULTMODE, ui64 align = NALF_ALLOC_DEFAULTALIGN); void Free(void* mem); void* Realloc(void* mem, ui64 len); TAllocHint::EHint SwapHint(TAllocHint::EHint hint) noexcept; std::pair MemBlockSize(void* mem); void* Allocate(TPerThreadAllocator* pta, ui64 len, TAllocHint::EHint hint = NALF_ALLOC_DEFAULTMODE, ui64 align = NALF_ALLOC_DEFAULTALIGN); void Free(TPerThreadAllocator* pta, void* mem); void* Realloc(TPerThreadAllocator* pta, void* mem, ui64 len); TAllocHint::EHint SwapHint(TPerThreadAllocator* pta, TAllocHint::EHint hint) noexcept; template struct TSwapHint : TNonCopyable { const TAllocHint::EHint Old; TSwapHint() : Old(SwapHint(Hint)) { } ~TSwapHint() { SwapHint(Old); } }; void* SystemAllocation(ui64 size); void SystemFree(void* mem, ui64 size); void* SystemRemap(void* mem, ui64 oldsize, ui64 newsize); ui32 GetNumaNode(); struct TAllocatorStats { ui64 TotalBytesReserved; // w/o system bytes! ui32 PerThreadEntries; struct TSizeStats { ui32 PageSize; ui32 ChunkSize; ui64 TotalPagesReserved; ui64 TotalPagesCached; ui64 TotalAllocations; ui64 TotalReclaimed; ui64 PagesClaimed; ui64 PagesFromCache; ui64 PagesReleased; TSizeStats(); }; struct TIncrementalStats { ui64 TotalPagesReserved; ui64 TotalPagesCached; ui64 TotalAllocations; ui64 TotalReclaimed; ui64 PagesClaimed; ui64 PagesFromCache; ui64 PagesReleased; TIncrementalStats(); }; struct TSysStats { ui64 TotalBytesReserved; ui64 TotalBytesCached; ui64 TotalAllocations; ui64 TotalReclaimed; TSysStats(); }; TVector BySizeStats; TIncrementalStats IncrementalStats; TSysStats SysStats; TAllocatorStats(); void Out(IOutputStream& out) const; }; TVector GetAllocatorStats(); // one entry per numa-node static const ui64 SystemPageSize = 4096; }