ydb/yql/essentials/udfs/common/unicode_base/lib/unicode_base_udf.h

#pragma once

#include <yql/essentials/public/udf/udf_allocator.h>
#include <yql/essentials/public/udf/udf_helpers.h>
#include <yql/essentials/utils/utf8.h>
#include <yql/essentials/public/udf/arrow/udf_arrow_helpers.h>

#include <library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h>
#include <library/cpp/unicode/normalization/normalization.h>
#include <library/cpp/unicode/set/unicode_set.h>

#include <library/cpp/deprecated/split/split_iterator.h>
#include <util/string/join.h>
#include <util/string/reverse.h>
#include <util/string/split.h>
#include <util/string/subst.h>
#include <util/charset/wide.h>
#include <util/charset/utf8.h>
#include <util/string/strip.h>
#include <util/string/ascii.h>
#include <util/charset/unidata.h>

using namespace NYql;
using namespace NUdf;
using namespace NUnicode;

namespace {
    inline constexpr bool IsAscii(wchar32 c) noexcept {
        return ::IsAscii(c);
    }

    template <class It>
    struct TIsUnicodeSpaceAdapter {
        bool operator()(const It& it) const noexcept {
            return IsSpace(*it);
        }
    };

    template <class It>
    TIsUnicodeSpaceAdapter<It> IsUnicodeSpaceAdapter(It) {
        return {};
    }

    struct TNoChangesTag {};

    template <typename TDerived>
    struct TScalarOperationMixin {
        static TUnboxedValue DoExecute(const IValueBuilder* builder, const TUnboxedValuePod* args) {
            Y_DEBUG_ABORT_UNLESS(IsUtf8(args[0].AsStringRef()));
            auto&& executeResult = TDerived::Execute(args[0].AsStringRef());
            return ProcessResult(builder, std::move(executeResult), args);
        }

    private:
        static TUnboxedValue ProcessResult(const IValueBuilder* builder, TString&& newString, const TUnboxedValuePod*) {
            return builder->NewString(std::move(newString));
        }

        template <typename T>
        static TUnboxedValue ProcessResult(const IValueBuilder* builder, std::variant<TNoChangesTag, T> newValue, const TUnboxedValuePod* initialArg) {
            if (std::holds_alternative<T>(newValue)) {
                return ProcessResult(builder, std::move(std::get<T>(newValue)), initialArg);
            } else {
                return initialArg[0];
            }
        }

        static TUnboxedValue ProcessResult(const IValueBuilder* builder, bool result, const TUnboxedValuePod*) {
            Y_UNUSED(builder);
            return TUnboxedValuePod(result);
        }
    };

    template <typename TDerived>
    struct TBlockOperationMixin {
        template <typename Sync>
        static void DoExecute(const TBlockItem arg, const Sync& sync) {
            Y_DEBUG_ABORT_UNLESS(IsUtf8(arg.AsStringRef()));
            auto&& executeResult = TDerived::Execute(arg.AsStringRef());
            TBlockItem boxedValue = ProcessResult(std::move(executeResult), arg);
            sync(boxedValue);
        }

    private:
        static TBlockItem ProcessResult(const TString& newString, const TBlockItem arg) {
            Y_UNUSED(arg);
            return TBlockItem(std::move(newString));
        }

        template <typename T>
        static TBlockItem ProcessResult(const std::variant<TNoChangesTag, T>& newValue, const TBlockItem arg) {
            if (std::holds_alternative<T>(newValue)) {
                return ProcessResult(std::get<T>(newValue), arg);
            } else {
                return arg;
            }
        }

        static TBlockItem ProcessResult(bool result, const TBlockItem arg) {
            Y_UNUSED(arg);
            return TBlockItem(result);
        }
    };

    template <typename TDerived>
    struct TOperationMixin: public TBlockOperationMixin<TDerived>, public TScalarOperationMixin<TDerived> {
        using TBlockOperationMixin<TDerived>::DoExecute;
        using TScalarOperationMixin<TDerived>::DoExecute;
    };

    template <auto mode>
    struct TNormalizeUTF8: public TOperationMixin<TNormalizeUTF8<mode>> {
        static TString Execute(TStringRef arg) {
            const TUtf16String& input = UTF8ToWide(arg.Data(), arg.Size());
            return WideToUTF8(Normalize<mode>(input));
        }
    };

    template <bool (*Function)(wchar32)>
    struct TCheckAllChars: public TOperationMixin<TCheckAllChars<Function>> {
        static bool Execute(TStringRef arg) {
            const TStringBuf input(arg);
            wchar32 rune;
            const unsigned char* cur = reinterpret_cast<const unsigned char*>(input.begin());
            const unsigned char* last = reinterpret_cast<const unsigned char*>(input.end());
            while (cur != last) {
                ReadUTF8CharAndAdvance(rune, cur, last);
                if (!static_cast<bool (*)(wchar32)>(Function)(rune)) {
                    return false;
                }
            }
            return true;
        }
    };

    template <bool (*Function)(TUtf16String&, size_t pos, size_t count)>
    struct TStringToStringMapper: public TOperationMixin<TStringToStringMapper<Function>> {
        static std::variant<TNoChangesTag, TString> Execute(TStringRef arg) {
            if (auto wide = UTF8ToWide(arg);
                static_cast<bool (*)(TUtf16String&, size_t pos, size_t count)>(Function)(wide, 0, TUtf16String::npos)) {
                return WideToUTF8(std::move(wide));
            } else {
                return TNoChangesTag{};
            }
        }
    };

#define DEFINE_UTF8_OPERATION(udfName, Executor, signature)                                          \
    BEGIN_SIMPLE_STRICT_ARROW_UDF(T##udfName, signature) {                                           \
        return Executor::DoExecute(valueBuilder, args);                                              \
    }                                                                                                \
                                                                                                     \
    struct T##udfName##KernelExec                                                                    \
        : public TUnaryKernelExec<T##udfName##KernelExec> {                                          \
        template <typename TSink>                                                                    \
        static void Process(const IValueBuilder* valueBuilder, TBlockItem arg1, const TSink& sink) { \
            Y_UNUSED(valueBuilder);                                                                  \
            Executor::DoExecute(arg1, sink);                                                         \
        }                                                                                            \
    };                                                                                               \
                                                                                                     \
    END_SIMPLE_ARROW_UDF(T##udfName, T##udfName##KernelExec::Do)

    DEFINE_UTF8_OPERATION(Normalize, TNormalizeUTF8<NFC>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(NormalizeNFD, TNormalizeUTF8<NFD>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(NormalizeNFC, TNormalizeUTF8<NFC>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(NormalizeNFKD, TNormalizeUTF8<NFKD>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(NormalizeNFKC, TNormalizeUTF8<NFKC>, TUtf8(TAutoMap<TUtf8>));

    DEFINE_UTF8_OPERATION(IsAscii, TCheckAllChars<IsAscii>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsSpace, TCheckAllChars<IsSpace>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsUpper, TCheckAllChars<IsUpper>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsLower, TCheckAllChars<IsLower>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsDigit, TCheckAllChars<IsDigit>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsAlpha, TCheckAllChars<IsAlpha>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsAlnum, TCheckAllChars<IsAlnum>, bool(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(IsHex, TCheckAllChars<IsHexdigit>, bool(TAutoMap<TUtf8>));

    DEFINE_UTF8_OPERATION(ToTitle, TStringToStringMapper<ToTitle>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(ToUpper, TStringToStringMapper<ToUpper>, TUtf8(TAutoMap<TUtf8>));
    DEFINE_UTF8_OPERATION(ToLower, TStringToStringMapper<ToLower>, TUtf8(TAutoMap<TUtf8>));

    SIMPLE_UDF(TIsUtf, bool(TOptional<char*>)) {
        Y_UNUSED(valueBuilder);
        if (args[0]) {
            return TUnboxedValuePod(IsUtf8(args[0].AsStringRef()));
        } else {
            return TUnboxedValuePod(false);
        }
    }

    SIMPLE_UDF(TGetLength, ui64(TAutoMap<TUtf8>)) {
        Y_UNUSED(valueBuilder);
        const auto& inputRef = args[0].AsStringRef();
        size_t result;
        GetNumberOfUTF8Chars(inputRef.Data(), inputRef.Size(), result);
        return TUnboxedValuePod(static_cast<ui64>(result));
    }

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TToUint64, ui64(TAutoMap<TUtf8>, TOptional<ui16>), 1) {
        Y_UNUSED(valueBuilder);
        const TString inputStr(args[0].AsStringRef());
        const char* input = inputStr.data();
        const int base = static_cast<int>(args[1].GetOrDefault<ui16>(0));
        char* pos = nullptr;
        errno = 0;
        unsigned long long res = std::strtoull(input, &pos, base);
        if (!res && errno == EINVAL) {
            UdfTerminate("Incorrect base");
        }

        ui64 ret = static_cast<ui64>(res);
        if (!res && pos == input) {
            UdfTerminate("Input string is not a number");
        } else if ((res == ULLONG_MAX && errno == ERANGE) || ret != res) {
            UdfTerminate("Converted value falls out of Uint64 range");
        } else if (*pos) {
            UdfTerminate("Input string contains junk after the number");
        }
        return TUnboxedValuePod(ret);
    }

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TTryToUint64, TOptional<ui64>(TAutoMap<TUtf8>, TOptional<ui16>), 1) {
        Y_UNUSED(valueBuilder);
        const TString inputStr(args[0].AsStringRef());
        const char* input = inputStr.data();
        const int base = static_cast<int>(args[1].GetOrDefault<ui16>(0));
        char* pos = nullptr;
        errno = 0;
        unsigned long long res = std::strtoull(input, &pos, base);
        if (!res && errno == EINVAL) {
            return TUnboxedValuePod();
        }

        ui64 ret = static_cast<ui64>(res);
        if (!res && pos == input) {
            return TUnboxedValuePod();
        }
        if ((res == ULLONG_MAX && errno == ERANGE) || ret != res) {
            return TUnboxedValuePod();
        }
        if (*pos) {
            return TUnboxedValuePod();
        }
        return TUnboxedValuePod(ret);
    }

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TSubstring, TUtf8(TAutoMap<TUtf8>, TOptional<ui64>, TOptional<ui64>), 1) {
        const TStringBuf input(args[0].AsStringRef());
        size_t from = args[1].GetOrDefault<ui64>(0);
        size_t len = !args[2] ? TStringBuf::npos : size_t(args[2].Get<ui64>());
        return valueBuilder->NewString(SubstrUTF8(input, from, len));
    }

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TFind, TOptional<ui64>(TAutoMap<TUtf8>, TUtf8, TOptional<ui64>), 1) {
        Y_UNUSED(valueBuilder);
        const std::string_view string(args[0].AsStringRef());
        const std::string_view needle(args[1].AsStringRef());
        std::string_view::size_type pos = 0U;

        if (auto p = args[2].GetOrDefault<ui64>(0ULL)) {
            for (auto ptr = string.data(); p && pos < string.size(); --p) {
                const auto width = WideCharSize(*ptr);
                pos += width;
                ptr += width;
            }
        }

        if (const auto find = string.find(needle, pos); std::string_view::npos != find) {
            size_t result;
            GetNumberOfUTF8Chars(string.data(), find, result);
            return TUnboxedValuePod(static_cast<ui64>(result));
        }
        return TUnboxedValuePod();
    }

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TRFind, TOptional<ui64>(TAutoMap<TUtf8>, TUtf8, TOptional<ui64>), 1) {
        Y_UNUSED(valueBuilder);
        const std::string_view string(args[0].AsStringRef());
        const std::string_view needle(args[1].AsStringRef());
        std::string_view::size_type pos = std::string_view::npos;

        if (auto p = args[2].GetOrDefault<ui64>(std::string_view::npos); std::string_view::npos != p) {
            pos = 0ULL;
            for (auto ptr = string.data(); p && pos < string.size(); --p) {
                const auto width = WideCharSize(*ptr);
                pos += width;
                ptr += width;
            }
        }

        if (const auto find = string.rfind(needle, pos); std::string_view::npos != find) {
            size_t result;
            GetNumberOfUTF8Chars(string.data(), find, result);
            return TUnboxedValuePod(static_cast<ui64>(result));
        }
        return TUnboxedValuePod();
    }

    using TTmpVector = TSmallVec<TUnboxedValue, TUnboxedValue::TAllocator>;

    template <typename TIt>
    static void SplitToListImpl(
            const IValueBuilder* valueBuilder,
            const TUnboxedValue& input,
            const std::string_view::const_iterator from,
            const TIt& it,
            TTmpVector& result) {
        for (const auto& elem : it) {
            result.emplace_back(valueBuilder->SubString(input, std::distance(from, elem.TokenStart()), std::distance(elem.TokenStart(), elem.TokenDelim())));
        }
    }

    template <typename TIt>
    static void SplitToListImpl(
            const IValueBuilder* valueBuilder,
            const TUnboxedValue& input,
            const TUtf32String::const_iterator start,
            const TIt& it,
            TTmpVector& result) {
        const std::string_view& original = input.AsStringRef();
        size_t charPos = 0U, bytePos = 0U;
        for (const auto& elem : it) {
            for (const size_t next = std::distance(start, elem.TokenStart()); charPos < next; ++charPos)
                bytePos += WideCharSize(original[bytePos]);
            const auto from = bytePos;

            for (const size_t next = charPos + std::distance(elem.TokenStart(), elem.TokenDelim()); charPos < next; ++charPos)
                bytePos += WideCharSize(original[bytePos]);
            const auto size = bytePos - from;
            result.emplace_back(valueBuilder->SubString(input, from, size));
        }
    }

    template <typename TIt, typename TStrIt>
    static void SplitToListImpl(
            const IValueBuilder* valueBuilder,
            const TUnboxedValue& input,
            const TStrIt from,
            TIt& it,
            bool skipEmpty,
            TTmpVector& result) {
        if (skipEmpty) {
            SplitToListImpl(valueBuilder, input, from, it.SkipEmpty(), result);
        } else {
            SplitToListImpl(valueBuilder, input, from, it, result);
        }
    }

    constexpr char delimeterStringName[] = "DelimeterString";
    constexpr char skipEmptyName[] = "SkipEmpty";
    constexpr char limitName[] = "Limit";
    using TDelimeterStringArg = TNamedArg<bool, delimeterStringName>;
    using TSkipEmptyArg = TNamedArg<bool, skipEmptyName>;
    using TLimitArg = TNamedArg<ui64, limitName>;

    SIMPLE_UDF_WITH_OPTIONAL_ARGS(TSplitToList, TListType<TUtf8>(
                            TOptional<TUtf8>,
                            TUtf8,
                            TDelimeterStringArg,
                            TSkipEmptyArg,
                            TLimitArg
                       ),
                       3) {
        TTmpVector result;
        if (args[0]) {
            const bool delimiterString = args[2].GetOrDefault<bool>(true);
            const bool skipEmpty = args[3].GetOrDefault<bool>(false);
            const auto limit = args[4].GetOrDefault<ui64>(0);
            if (delimiterString) {
                const std::string_view input(args[0].AsStringRef());
                const std::string_view delimeter(args[1].AsStringRef());
                if (limit) {
                    auto it = StringSplitter(input).SplitByString(delimeter).Limit(limit + 1);
                    SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result);
                } else {
                    auto it = StringSplitter(input).SplitByString(delimeter);
                    SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result);
                }
            } else {
                const auto& input = UTF8ToUTF32<true>(args[0].AsStringRef());
                const auto& delimeter = UTF8ToUTF32<true>(args[1].AsStringRef());
                if (limit) {
                    auto it = StringSplitter(input).SplitBySet(delimeter.c_str()).Limit(limit + 1);
                    SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result);
                } else {
                    auto it = StringSplitter(input).SplitBySet(delimeter.c_str());
                    SplitToListImpl(valueBuilder, args[0], input.cbegin(), it, skipEmpty, result);
                }
            }
        }
        return valueBuilder->NewList(result.data(), result.size());
    }

    SIMPLE_UDF(TJoinFromList, TUtf8(TAutoMap<TListType<TOptional<TUtf8>>>, TUtf8)) {
        const auto input = args[0].GetListIterator();
        const std::string_view delimeter(args[1].AsStringRef());
        std::vector<TString> items;

        for (TUnboxedValue current; input.Next(current);) {
            if (current) {
                items.emplace_back(current.AsStringRef());
            }
        }

        return valueBuilder->NewString(JoinSeq(delimeter, items));
    }

    SIMPLE_UDF(TLevensteinDistance, ui64(TAutoMap<TUtf8>, TAutoMap<TUtf8>)) {
        Y_UNUSED(valueBuilder);
        const TStringBuf left(args[0].AsStringRef());
        const TStringBuf right(args[1].AsStringRef());
        const auto& leftUtf32 = UTF8ToUTF32<true>(left);
        const auto& rightUtf32 = UTF8ToUTF32<true>(right);
        const ui64 result = NLevenshtein::Distance(leftUtf32, rightUtf32);
        return TUnboxedValuePod(result);
    }

    SIMPLE_UDF(TReplaceAll, TUtf8(TAutoMap<TUtf8>, TUtf8, TUtf8)) {
        if (TString result(args[0].AsStringRef()); SubstGlobal(result, args[1].AsStringRef(), args[2].AsStringRef()))
            return valueBuilder->NewString(result);
        else
            return args[0];
    }

    SIMPLE_UDF(TReplaceFirst, TUtf8(TAutoMap<TUtf8>, TUtf8, TUtf8)) {
        std::string result(args[0].AsStringRef());
        const std::string_view what(args[1].AsStringRef());
        if (const auto index = result.find(what); index != std::string::npos) {
            result.replace(index, what.size(), std::string_view(args[2].AsStringRef()));
            return valueBuilder->NewString(result);
        }
        return args[0];
    }

    SIMPLE_UDF(TReplaceLast, TUtf8(TAutoMap<TUtf8>, TUtf8, TUtf8)) {
        std::string result(args[0].AsStringRef());
        const std::string_view what(args[1].AsStringRef());
        if (const auto index = result.rfind(what); index != std::string::npos) {
            result.replace(index, what.size(), std::string_view(args[2].AsStringRef()));
            return valueBuilder->NewString(result);
        }
        return args[0];
    }

    SIMPLE_UDF(TRemoveAll, TUtf8(TAutoMap<TUtf8>, TUtf8)) {
        TUtf32String input = UTF8ToUTF32<true>(args[0].AsStringRef());
        const TUtf32String remove = UTF8ToUTF32<true>(args[1].AsStringRef());
        const std::unordered_set<wchar32> chars(remove.cbegin(), remove.cend());
        size_t tpos = 0;
        for (const wchar32 c : input) {
            if (!chars.contains(c)) {
                input[tpos++] = c;
            }
        }
        if (tpos != input.size()) {
            input.resize(tpos);
            return valueBuilder->NewString(WideToUTF8(input));
        }
        return args[0];
    }

    SIMPLE_UDF(TRemoveFirst, TUtf8(TAutoMap<TUtf8>, TUtf8)) {
        TUtf32String input = UTF8ToUTF32<true>(args[0].AsStringRef());
        const TUtf32String remove =  UTF8ToUTF32<true>(args[1].AsStringRef());
        const std::unordered_set<wchar32> chars(remove.cbegin(), remove.cend());
        for (auto it = input.cbegin(); it != input.cend(); ++it) {
            if (chars.contains(*it)) {
                input.erase(it);
                return valueBuilder->NewString(WideToUTF8(input));
            }
        }
        return args[0];
    }

    SIMPLE_UDF(TRemoveLast, TUtf8(TAutoMap<TUtf8>, TUtf8)) {
        TUtf32String input = UTF8ToUTF32<true>(args[0].AsStringRef());
        const TUtf32String remove = UTF8ToUTF32<true>(args[1].AsStringRef());
        const std::unordered_set<wchar32> chars(remove.cbegin(), remove.cend());
        for (auto it = input.crbegin(); it != input.crend(); ++it) {
            if (chars.contains(*it)) {
                input.erase(input.crend() - it - 1, 1);
                return valueBuilder->NewString(WideToUTF8(input));
            }
        }
        return args[0];
    }

    SIMPLE_UDF(TToCodePointList, TListType<ui32>(TAutoMap<TUtf8>)) {
        size_t codePointCount = 0;
        const auto& inputRef = args[0].AsStringRef();
        if (!GetNumberOfUTF8Chars(inputRef.Data(), inputRef.Size(), codePointCount)) {
            // should not happen but still we have to check return code
            ythrow yexception() << "Unable to count code points";
        }

        TUnboxedValue* itemsPtr = nullptr;
        auto result = valueBuilder->NewArray(codePointCount, itemsPtr);
        const unsigned char* current = reinterpret_cast<const unsigned char*>(inputRef.Data());
        const unsigned char* end = current + inputRef.Size();
        wchar32 rune = BROKEN_RUNE;
        ui32 codePointIndex = 0;
        RECODE_RESULT retcode = RECODE_OK;
        while (current < end && RECODE_OK == (retcode = ReadUTF8CharAndAdvance(rune, current, end))) {
            if (codePointIndex >= codePointCount) {
                // sanity check
                ythrow yexception() << "Too big code point index " << codePointIndex << ", expecting only " << codePointCount << " code points";
            }
            itemsPtr[codePointIndex++] = TUnboxedValuePod(static_cast<ui32>(rune));
        }

        if (retcode != RECODE_OK) {
            ythrow yexception() << "Malformed UTF-8 string";
        }

        return result;
    }

    SIMPLE_UDF(TFromCodePointList, TUtf8(TAutoMap<TListType<ui32>>)) {
        auto input = args[0];
        if (auto elems = input.GetElements()) {
            const auto elemCount = input.GetListLength();
            auto bufferSize = WideToUTF8BufferSize(elemCount);
            TTempBuf buffer(bufferSize);
            auto bufferPtr = buffer.Data();
            auto bufferEnd = buffer.Data() + bufferSize;
            for (ui64 i = 0; i != elemCount; ++i) {
                const auto& item = elems[i];
                const wchar32 rune = item.Get<ui32>();
                size_t written = 0;
                WideToUTF8(&rune, 1, bufferPtr, written);
                Y_ENSURE(written <= 4);
                bufferPtr += written;
                Y_ENSURE(bufferPtr <= bufferEnd);
            }
            return valueBuilder->NewString(TStringRef(buffer.Data(), bufferPtr - buffer.Data()));
        }

        std::vector<char, NUdf::TStdAllocatorForUdf<char>> buffer;
        buffer.reserve(TUnboxedValuePod::InternalBufferSize);

        const auto& iter = input.GetListIterator();
        char runeBuffer[4] = {};
        for (NUdf::TUnboxedValue item; iter.Next(item); ) {
            const wchar32 rune = item.Get<ui32>();
            size_t written = 0;
            WideToUTF8(&rune, 1, runeBuffer, written);
            Y_ENSURE(written <= 4);
            buffer.insert(buffer.end(), runeBuffer, runeBuffer + written);
        }

        return valueBuilder->NewString(TStringRef(buffer.data(), buffer.size()));
    }

    SIMPLE_UDF(TReverse, TUtf8(TAutoMap<TUtf8>)) {
        auto wide = UTF8ToWide(args[0].AsStringRef());
        ReverseInPlace(wide);
        return valueBuilder->NewString(WideToUTF8(wide));
    }

    SIMPLE_UDF(TStrip, TUtf8(TAutoMap<TUtf8>)) {
        const TUtf32String input = UTF8ToUTF32<true>(args[0].AsStringRef());
        const auto& result = StripString(input, IsUnicodeSpaceAdapter(input.begin()));
        return valueBuilder->NewString(WideToUTF8(result));
    }

    SIMPLE_UDF(TIsUnicodeSet, bool(TAutoMap<TUtf8>, TUtf8)) {
        Y_UNUSED(valueBuilder);
        const TStringBuf input(args[0].AsStringRef());
        const TUtf16String& customCategory = UTF8ToWide(args[1].AsStringRef());
        TUnicodeSet unicodeSet;
        try {
            unicodeSet.Parse(customCategory);
        } catch (...) {
            UdfTerminate((TStringBuilder() << "Failed to parse unicode set: " << CurrentExceptionMessage()).c_str());
        }
        bool result = true;
        wchar32 rune;
        const unsigned char* cur = reinterpret_cast<const unsigned char*>(input.begin());
        const unsigned char* last = reinterpret_cast<const unsigned char*>(input.end());
        while (cur != last) {
            ReadUTF8CharAndAdvance(rune, cur, last);
            if (!unicodeSet.Has(rune)) {
                result = false;
                break;
            }
        }
        return TUnboxedValuePod(result);
    }

#define EXPORTED_UNICODE_BASE_UDF \
        TIsUtf,                   \
        TGetLength,               \
        TSubstring,               \
        TFind,                    \
        TRFind,                   \
        TSplitToList,             \
        TJoinFromList,            \
        TLevensteinDistance,      \
        TReplaceAll,              \
        TReplaceFirst,            \
        TReplaceLast,             \
        TRemoveAll,               \
        TRemoveFirst,             \
        TRemoveLast,              \
        TToCodePointList,         \
        TFromCodePointList,       \
        TReverse,                 \
        TToLower,                 \
        TToUpper,                 \
        TToTitle,                 \
        TToUint64,                \
        TTryToUint64,             \
        TStrip,                   \
        TIsUnicodeSet,            \
        TNormalize,               \
        TNormalizeNFD,            \
        TNormalizeNFC,            \
        TNormalizeNFKD,           \
        TNormalizeNFKC,           \
        TIsAscii,                 \
        TIsSpace,                 \
        TIsUpper,                 \
        TIsLower,                 \
        TIsDigit,                 \
        TIsAlpha,                 \
        TIsAlnum,                 \
        TIsHex
}