Clone protobuf_udf to ydb/library

library/cpp/protobuf/CMakeLists.txt

@@ -6,6 +6,8 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(dynamic_prototype)
 add_subdirectory(interop)
 add_subdirectory(json)
 add_subdirectory(util)
+add_subdirectory(yql)

library/cpp/protobuf/dynamic_prototype/CMakeLists.darwin-x86_64.txt

@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-protobuf-dynamic_prototype)
+target_link_libraries(cpp-protobuf-dynamic_prototype PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_sources(cpp-protobuf-dynamic_prototype PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/dynamic_prototype.cpp
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.cpp
+)

library/cpp/protobuf/dynamic_prototype/CMakeLists.linux-aarch64.txt

@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-protobuf-dynamic_prototype)
+target_link_libraries(cpp-protobuf-dynamic_prototype PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_sources(cpp-protobuf-dynamic_prototype PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/dynamic_prototype.cpp
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.cpp
+)

library/cpp/protobuf/dynamic_prototype/CMakeLists.linux-x86_64.txt

@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-protobuf-dynamic_prototype)
+target_link_libraries(cpp-protobuf-dynamic_prototype PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_sources(cpp-protobuf-dynamic_prototype PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/dynamic_prototype.cpp
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.cpp
+)

library/cpp/protobuf/dynamic_prototype/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()

library/cpp/protobuf/dynamic_prototype/CMakeLists.windows-x86_64.txt

@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(cpp-protobuf-dynamic_prototype)
+target_link_libraries(cpp-protobuf-dynamic_prototype PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  contrib-libs-protobuf
+)
+target_sources(cpp-protobuf-dynamic_prototype PRIVATE
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/dynamic_prototype.cpp
+  ${CMAKE_SOURCE_DIR}/library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.cpp
+)

library/cpp/protobuf/dynamic_prototype/dynamic_prototype.cpp

@@ -0,0 +1,63 @@
+#include "dynamic_prototype.h"
+
+#include <util/generic/yexception.h>
+
+TDynamicPrototype::TDynamicMessage::TDynamicMessage(THolder<NProtoBuf::Message> message, TIntrusivePtr<TDynamicPrototype> prototype)
+    : Prototype(std::move(prototype))
+    , Message(std::move(message))
+{
+}
+
+NProtoBuf::Message& TDynamicPrototype::TDynamicMessage::operator*() {
+    return *Message;
+}
+
+NProtoBuf::Message* TDynamicPrototype::TDynamicMessage::operator->() {
+    return Get();
+}
+
+NProtoBuf::Message* TDynamicPrototype::TDynamicMessage::Get() {
+    return Message.Get();
+}
+
+TDynamicPrototypePtr TDynamicPrototype::Create(const NProtoBuf::FileDescriptorSet& fds, const TString& messageName, bool yqlHack) {
+    return new TDynamicPrototype(fds, messageName, yqlHack);
+}
+
+TDynamicPrototype::TDynamicPrototype(const NProtoBuf::FileDescriptorSet& fileDescriptorSet, const TString& messageName, bool yqlHack)
+{
+    const NProtoBuf::FileDescriptor* fileDescriptor;
+
+    for (int i = 0; i < fileDescriptorSet.file_size(); ++i) {
+        fileDescriptor = Pool.BuildFile(fileDescriptorSet.file(i));
+        if (fileDescriptor == nullptr) {
+            ythrow yexception() << "can't build file descriptor";
+        }
+    }
+
+    Descriptor = Pool.FindMessageTypeByName(messageName);
+
+    // Первоначальный вариант поведения, когда тип определялся
+    // по имени сообщения верхнего уровня в заданном файле.
+    if (yqlHack && !Descriptor) {
+        Descriptor = fileDescriptor->FindMessageTypeByName(messageName);
+    }
+
+    if (!Descriptor) {
+        ythrow yexception() << "no descriptor for " << messageName;
+    }
+
+    Prototype = Factory.GetPrototype(Descriptor);
+}
+
+TDynamicPrototype::TDynamicMessage TDynamicPrototype::Create() {
+    return TDynamicMessage(CreateUnsafe(), this);
+}
+
+THolder<NProtoBuf::Message> TDynamicPrototype::CreateUnsafe() const {
+    return THolder<NProtoBuf::Message>(Prototype->New());
+}
+
+const NProtoBuf::Descriptor* TDynamicPrototype::GetDescriptor() const {
+    return Descriptor;
+}

library/cpp/protobuf/dynamic_prototype/dynamic_prototype.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <google/protobuf/dynamic_message.h>
+#include <google/protobuf/descriptor.pb.h>
+
+#include <util/generic/ptr.h>
+
+class TDynamicPrototype;
+
+using TDynamicPrototypePtr = TIntrusivePtr<TDynamicPrototype>;
+
+class TDynamicPrototype : public TThrRefBase {
+public:
+    class TDynamicMessage {
+    public:
+        TDynamicMessage() = default;
+        TDynamicMessage(THolder<NProtoBuf::Message> message, TIntrusivePtr<TDynamicPrototype> prototype);
+        NProtoBuf::Message& operator*();
+        NProtoBuf::Message* operator->();
+        NProtoBuf::Message* Get();
+    private:
+        TIntrusivePtr<TDynamicPrototype> Prototype;
+        THolder<NProtoBuf::Message> Message;
+    };
+
+    static TDynamicPrototypePtr Create(const NProtoBuf::FileDescriptorSet& fds, const TString& messageName, bool yqlHack = false);
+
+    // https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.dynamic_message#DynamicMessageFactory
+    // Must outlive created messages
+    TDynamicMessage Create();
+
+    THolder<NProtoBuf::Message> CreateUnsafe() const;
+
+    const NProtoBuf::Descriptor* GetDescriptor() const;
+
+private:
+    TDynamicPrototype(const NProtoBuf::FileDescriptorSet& fds, const TString& messageName, bool yqlHack);
+
+    NProtoBuf::DescriptorPool Pool;
+    const NProtoBuf::Descriptor* Descriptor;
+    NProtoBuf::DynamicMessageFactory Factory;
+    const NProtoBuf::Message* Prototype;
+};

library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.cpp

@@ -0,0 +1,33 @@
+#include "generate_file_descriptor_set.h"
+
+#include <util/generic/queue.h>
+#include <util/generic/set.h>
+#include <util/generic/vector.h>
+
+using namespace NProtoBuf;
+
+FileDescriptorSet GenerateFileDescriptorSet(const Descriptor* desc) {
+    TVector<const FileDescriptor*> flat;
+    TQueue<const FileDescriptor*> descriptors;
+
+    for (descriptors.push(desc->file()); !descriptors.empty(); descriptors.pop()) {
+        const FileDescriptor* file = descriptors.front();
+
+        for (int i = 0; i < file->dependency_count(); ++i) {
+            descriptors.push(file->dependency(i));
+        }
+
+        flat.push_back(file);
+    }
+
+    FileDescriptorSet result;
+    TSet<TString> visited;
+
+    for (auto ri = flat.rbegin(); ri != flat.rend(); ++ri) {
+        if (visited.insert((*ri)->name()).second == true) {
+            (*ri)->CopyTo(result.add_file());
+        }
+    }
+
+    return result;
+}

library/cpp/protobuf/dynamic_prototype/generate_file_descriptor_set.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <google/protobuf/descriptor.pb.h>
+
+NProtoBuf::FileDescriptorSet GenerateFileDescriptorSet(const NProtoBuf::Descriptor* desc);