ydb/library/cpp/malloc/nalf/nalf_alloc.h

#pragma once

#include <util/generic/vector.h>
#include <util/stream/output.h>

#ifndef NALF_ALLOC_DEFAULTMODE
#define NALF_ALLOC_DEFAULTMODE (TAllocHint::Chunked)
#endif

#ifndef NALF_ALLOC_DEFAULTALIGN
#define NALF_ALLOC_DEFAULTALIGN (16)
#endif

#if defined(_tsan_enabled_) || defined(_msan_enabled_) || defined(_asan_enabled_) || defined(WITH_VALGRIND)
#define NALF_FORCE_MALLOC_FREE 1
#define NALF_DONOT_DEFINE_GLOBALS 1
#endif

namespace NNumaAwareLockFreeAllocator {
    struct TAllocHint {
        enum EHint {
            Undefined,
            Incremental,
            Chunked,
            System,
            ForceIncremental,
            ForceChunked,
            ForceSystem,
            Bootstrap,
        };
        // valid op hint values: incremental, chunked, system, force*
        // valid thread hint values: undefined, incremental, chunked, system
        // bootstrap is used in node initialization only
    };

    class TPerThreadAllocator;

    TPerThreadAllocator* GetThreadAllocator();
    void* Allocate(ui64 len, TAllocHint::EHint hint = NALF_ALLOC_DEFAULTMODE, ui64 align = NALF_ALLOC_DEFAULTALIGN);
    void Free(void* mem);
    void* Realloc(void* mem, ui64 len);
    TAllocHint::EHint SwapHint(TAllocHint::EHint hint) noexcept;
    std::pair<ui64, TAllocHint::EHint> MemBlockSize(void* mem);

    void* Allocate(TPerThreadAllocator* pta, ui64 len, TAllocHint::EHint hint = NALF_ALLOC_DEFAULTMODE, ui64 align = NALF_ALLOC_DEFAULTALIGN);
    void Free(TPerThreadAllocator* pta, void* mem);
    void* Realloc(TPerThreadAllocator* pta, void* mem, ui64 len);

    TAllocHint::EHint SwapHint(TPerThreadAllocator* pta, TAllocHint::EHint hint) noexcept;

    template <TAllocHint::EHint Hint>
    struct TSwapHint : TNonCopyable {
        const TAllocHint::EHint Old;
        TSwapHint()
            : Old(SwapHint(Hint))
        {
        }
        ~TSwapHint() {
            SwapHint(Old);
        }
    };

    void* SystemAllocation(ui64 size);
    void SystemFree(void* mem, ui64 size);
    void* SystemRemap(void* mem, ui64 oldsize, ui64 newsize);
    ui32 GetNumaNode();

    struct TAllocatorStats {
        ui64 TotalBytesReserved; // w/o system bytes!
        ui32 PerThreadEntries;

        struct TSizeStats {
            ui32 PageSize;
            ui32 ChunkSize;

            ui64 TotalPagesReserved;
            ui64 TotalPagesCached;
            ui64 TotalAllocations;
            ui64 TotalReclaimed;
            ui64 PagesClaimed;
            ui64 PagesFromCache;
            ui64 PagesReleased;

            TSizeStats();
        };

        struct TIncrementalStats {
            ui64 TotalPagesReserved;
            ui64 TotalPagesCached;
            ui64 TotalAllocations;
            ui64 TotalReclaimed;
            ui64 PagesClaimed;
            ui64 PagesFromCache;
            ui64 PagesReleased;

            TIncrementalStats();
        };

        struct TSysStats {
            ui64 TotalBytesReserved;
            ui64 TotalBytesCached;
            ui64 TotalAllocations;
            ui64 TotalReclaimed;

            TSysStats();
        };

        TVector<TSizeStats> BySizeStats;
        TIncrementalStats IncrementalStats;
        TSysStats SysStats;

        TAllocatorStats();
        void Out(IOutputStream& out) const;
    };

    TVector<TAllocatorStats> GetAllocatorStats(); // one entry per numa-node

    static const ui64 SystemPageSize = 4096;

}