ydb/library/cpp/coroutine/engine/stack/stack_pool.inl

#include "stack_storage.h"
#include "stack_utils.h"


namespace NCoro::NStack {

    template<typename TGuard>
    TPool<TGuard>::TPool(size_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard)
        : StackSize_(stackSize)
        , RssPagesToKeep_(IsSmallStack() ? settings.SmallStackRssPagesToKeep : settings.RssPagesToKeep)
        , Guard_(guard)
        , ChunkSize_(Guard_.GetPageAlignedSize() + StackSize_ * settings.StacksPerChunk)
    {
        Y_ASSERT(RssPagesToKeep_);
        if (!RssPagesToKeep_) {
            RssPagesToKeep_ = 1; // at least guard should be kept
        }

        const size_t stackSizeInPages = stackSize / PageSize;
        Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);
        if (stackSizeInPages < RssPagesToKeep_) {
            RssPagesToKeep_ = stackSizeInPages; // keep all stack pages
        }

        Y_ASSERT(StackSize_ && !(StackSize_ & PageSizeMask)); // stack size is not zero and page aligned
        Y_ASSERT(Guard_.GetSize() < StackSize_); // stack has enough space to place guard
        Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);

        Storage_ = MakeHolder<TStorage>(StackSize_, RssPagesToKeep_, settings.ReleaseRate);

        AllocNewMemoryChunk();
    }

    template<typename TGuard>
    TPool<TGuard>::TPool(TPool&& other) noexcept = default;

    template<typename TGuard>
    TPool<TGuard>::~TPool() {
        if (!Memory_.empty()) {
            Y_ASSERT(NextToAlloc_ && StackSize_);

            for (const auto& chunk : Memory_) {
                Y_ASSERT(chunk.Raw && chunk.Aligned);

                if (Guard_.ShouldRemoveProtectionBeforeFree()) {
                    Guard_.RemoveProtection(chunk.Aligned, Guard_.GetPageAlignedSize()); // first page in chunk

                    const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
                    for (char* i = chunk.Aligned + Guard_.GetPageAlignedSize(); i < endOfStacksMemory; i += StackSize_) {
                        Guard_.RemoveProtection(i, StackSize_);
                    }
                }

                free(chunk.Raw);
            }
        }
    }

    template<typename TGuard>
    NDetails::TStack TPool<TGuard>::AllocStack(const char* name) {
        Y_ASSERT(!Memory_.empty());

        if (!Storage_->IsEmpty()) {
            return Storage_->GetStack(Guard_, name);
        } else {
            ++NumOfAllocated_;
            return AllocNewStack(name);
        }
    }

    template<typename TGuard>
    void TPool<TGuard>::FreeStack(NDetails::TStack& stack) {
        Y_ASSERT(Storage_->Size() < ((ChunkSize_ - Guard_.GetPageAlignedSize()) / StackSize_) * Memory_.size());
        Y_ASSERT(IsStackFromThisPool(stack));

        Storage_->ReturnStack(stack);
    }

    template<typename TGuard>
    size_t TPool<TGuard>::GetReleasedSize() const noexcept {
        return Storage_->GetReleasedSize();
    }
    template<typename TGuard>
    size_t TPool<TGuard>::GetFullSize() const noexcept {
        return Storage_->GetFullSize();
    }

    template<typename TGuard>
    void TPool<TGuard>::AllocNewMemoryChunk() {
        const size_t totalSizeInPages = ChunkSize_ / PageSize;

        TMemory memory;
        const auto res = GetAlignedMemory(totalSizeInPages, memory.Raw, memory.Aligned);
        Y_ABORT_UNLESS(res, "Failed to allocate memory for coro stack pool");

        NextToAlloc_ = memory.Aligned + Guard_.GetPageAlignedSize(); // skip first guard page
        Guard_.Protect(memory.Aligned, Guard_.GetPageAlignedSize(), false); // protect first guard page

        Memory_.push_back(std::move(memory));
    }

    template<typename TGuard>
    bool TPool<TGuard>::IsSmallStack() const noexcept {
        return StackSize_ / PageSize <= SmallStackMaxSizeInPages;
    }

    template<typename TGuard>
    bool TPool<TGuard>::IsStackFromThisPool(const NDetails::TStack& stack) const noexcept {
        for (const auto& chunk : Memory_) {
            const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
            if (chunk.Raw <= stack.GetRawMemory() && stack.GetRawMemory() < endOfStacksMemory) {
                return true;
            }
        }
        return false;
    }

    template<typename TGuard>
    NDetails::TStack TPool<TGuard>::AllocNewStack(const char* name) {
        if (NextToAlloc_ + StackSize_ > Memory_.rbegin()->Aligned + ChunkSize_) {
            AllocNewMemoryChunk(); // also sets NextToAlloc_ to first stack position in new allocated chunk of memory
        }
        Y_ASSERT(NextToAlloc_ + StackSize_ <= Memory_.rbegin()->Aligned + ChunkSize_);

        char* newStack = NextToAlloc_;
        NextToAlloc_ += StackSize_;

        Guard_.Protect(newStack, StackSize_, true);
        return NDetails::TStack{newStack, newStack, StackSize_, name};
    }

}