YDB Import 550

yt/yt/core/concurrency/config.h

@@ -40,7 +40,7 @@ namespace NDetail {
 ////////////////////////////////////////////////////////////////////////////////
 
 class TPeriodicExecutorOptionsSerializer
-    : public NYTree::TExternalizedYsonStruct<TPeriodicExecutorOptions>
+    : public NYTree::TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TPeriodicExecutorOptions, TPeriodicExecutorOptionsSerializer);
@@ -51,7 +51,7 @@ public:
 ////////////////////////////////////////////////////////////////////////////////
 
 class TRetryingPeriodicExecutorOptionsSerializer
-    : public NYTree::TExternalizedYsonStruct<TRetryingPeriodicExecutorOptions>
+    : public NYTree::TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TRetryingPeriodicExecutorOptions, TRetryingPeriodicExecutorOptionsSerializer);

yt/yt/core/misc/backoff_strategy_config.h

@@ -9,7 +9,7 @@ namespace NYT {
 ////////////////////////////////////////////////////////////////////////////////
 
 class TExponentialBackoffOptionsSerializer
-    : public NYTree::TExternalizedYsonStruct<TExponentialBackoffOptions>
+    : public NYTree::TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TExponentialBackoffOptions, TExponentialBackoffOptionsSerializer);
@@ -22,7 +22,7 @@ ASSIGN_EXTERNAL_YSON_SERIALIZER(TExponentialBackoffOptions, TExponentialBackoffO
 ////////////////////////////////////////////////////////////////////////////////
 
 class TConstantBackoffOptionsSerializer
-    : public NYTree::TExternalizedYsonStruct<TConstantBackoffOptions>
+    : public NYTree::TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TConstantBackoffOptions, TConstantBackoffOptionsSerializer);

yt/yt/core/ytree/serialization_traits.h

@@ -25,7 +25,7 @@ concept CSerializableByTraits = TSerializationTraits<T>::IsSerializable;
     template <> \
     struct ::NYT::NYTree::TSerializationTraits<Type> \
     { \
-        static constexpr bool IsSerializable = true; \
+        [[maybe_unused]] static constexpr bool IsSerializable = true; \
         using TSerializer = SerializerType; \
     }; \
 

yt/yt/core/ytree/serialize-inl.h

@@ -467,7 +467,7 @@ template <class T>
 void Serialize(const T& value, NYson::IYsonConsumer* consumer)
 {
     using TSerializer = typename TSerializationTraits<T>::TSerializer;
-    auto serializer = TSerializer::template CreateReadOnly<TSerializer>(value);
+    auto serializer = TSerializer::template CreateReadOnly<T, TSerializer>(value);
     Serialize(serializer, consumer);
 }
 
@@ -666,7 +666,7 @@ template <class T>
 void Deserialize(T& value, INodePtr node)
 {
     using TSerializer = typename TSerializationTraits<T>::TSerializer;
-    auto serializer = TSerializer::template CreateWritable<TSerializer>(value);
+    auto serializer = TSerializer::template CreateWritable<T, TSerializer>(value);
     Deserialize(serializer, node);
 }
 

yt/yt/core/ytree/unittests/yson_struct_ut.cpp

@@ -30,7 +30,7 @@ struct TTestTraitConfig
 };
 
 class TTestTraitConfigSerializer
-    : public TExternalizedYsonStruct<TTestTraitConfig>
+    : public virtual TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TTestTraitConfig, TTestTraitConfigSerializer);
@@ -54,7 +54,7 @@ struct TTestProcessorsTraitConfig
 };
 
 class TTestProcessorsTraitConfigSerializer
-    : public TExternalizedYsonStruct<TTestProcessorsTraitConfig>
+    : public TExternalizedYsonStruct
 {
 public:
     REGISTER_EXTERNALIZED_YSON_STRUCT(TTestProcessorsTraitConfig, TTestProcessorsTraitConfigSerializer);
@@ -88,12 +88,97 @@ public:
     }
 };
 
+////////////////////////////////////////////////////////////////////////////////
+
+struct TTestDerivedPodConfig
+    : public TTestTraitConfig
+{
+    int Field3;
+};
+
+class TTestDerivedPodConfigSerializer
+    : public TTestTraitConfigSerializer
+{
+public:
+    REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT(TTestDerivedPodConfig, TTestDerivedPodConfigSerializer, (TTestTraitConfigSerializer));
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.ExternalClassParameter("field_3", &TThat::Field3);
+    }
+};
+
+struct TTestDoubleDerivedPodConfig
+    : public TTestDerivedPodConfig
+{
+    int Field4;
+};
+
+class TTestDoubleDerivedPodConfigSerializer
+    : public TTestDerivedPodConfigSerializer
+{
+public:
+    REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT(TTestDoubleDerivedPodConfig, TTestDoubleDerivedPodConfigSerializer, (TTestDerivedPodConfigSerializer));
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.ExternalClassParameter("field_4", &TThat::Field4);
+    }
+};
+
+struct TTestDerivedSecondBase
+{
+    int Field5;
+    int Field6;
+};
+
+class TTestDerivedSecondBaseSerializer
+    : public virtual TExternalizedYsonStruct
+{
+public:
+    REGISTER_EXTERNALIZED_YSON_STRUCT(TTestDerivedSecondBase, TTestDerivedSecondBaseSerializer);
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.ExternalClassParameter("field_5", &TThat::Field5);
+        registrar.ExternalClassParameter("field_6", &TThat::Field6);
+    }
+};
+
+struct TTestDerivedTwoBasesConfig
+    : public TTestDoubleDerivedPodConfig
+    , public TTestDerivedSecondBase
+{ };
+
+class TTestDerivedTwoBasesConfigSerializer
+    : public TTestDoubleDerivedPodConfigSerializer
+    , public TTestDerivedSecondBaseSerializer
+{
+public:
+    REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT(
+        TTestDerivedTwoBasesConfig,
+        TTestDerivedTwoBasesConfigSerializer,
+        (TTestDoubleDerivedPodConfigSerializer)
+        (TTestDerivedSecondBaseSerializer));
+
+    static void Register(TRegistrar)
+    { }
+};
+
 } // namespace
 
 ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestTraitConfig, TTestTraitConfigSerializer);
 
 ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestProcessorsTraitConfig, TTestProcessorsTraitConfigSerializer);
 
+ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestDerivedPodConfig, TTestDerivedPodConfigSerializer);
+
+ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestDoubleDerivedPodConfig, TTestDoubleDerivedPodConfigSerializer);
+
+ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestDerivedSecondBase, TTestDerivedSecondBaseSerializer);
+
+ASSIGN_EXTERNAL_YSON_SERIALIZER(TTestDerivedTwoBasesConfig, TTestDerivedTwoBasesConfigSerializer);
+
 ////////////////////////////////////////////////////////////////////////////////
 
 namespace {
@@ -1830,6 +1915,113 @@ TEST(TYsonStructTest, SerializableByTraitsPostPreprocessors)
     EXPECT_EQ(reader.Field.Field2, 33);
 }
 
+class TDerivedFieldTester
+    : public NYT::NYTree::TYsonStructLite
+{
+public:
+    TTestDerivedPodConfig Field;
+
+    REGISTER_YSON_STRUCT_LITE(TDerivedFieldTester);
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.Parameter("field", &TThis::Field);
+    }
+};
+
+TEST(TYsonStructTest, SerializableByTraitsDerivedFromExternalized)
+{
+    TDerivedFieldTester writer = {};
+    writer.Field = {{55, 34}, 37};
+
+    TBufferStream stream;
+
+    ::Save(&stream, writer);
+
+    TDerivedFieldTester reader = {};
+    ::Load(&stream, reader);
+    EXPECT_EQ(writer.Field.Field1, 55);
+    EXPECT_EQ(writer.Field.Field2, 34);
+    EXPECT_EQ(writer.Field.Field3, 37);
+    EXPECT_EQ(reader.Field.Field1, 55);
+    EXPECT_EQ(reader.Field.Field2, 34);
+    EXPECT_EQ(reader.Field.Field3, 37);
+}
+
+class TDoubleDerivedFieldTester
+    : public NYT::NYTree::TYsonStructLite
+{
+public:
+    TTestDoubleDerivedPodConfig Field;
+
+    REGISTER_YSON_STRUCT_LITE(TDoubleDerivedFieldTester);
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.Parameter("field", &TThis::Field);
+    }
+};
+
+TEST(TYsonStructTest, SerializableByTraitsDoubleDerivedFromExternalized)
+{
+    TDoubleDerivedFieldTester writer = {};
+    writer.Field = {{{55, 34}, 37}, 77};
+
+    TBufferStream stream;
+
+    ::Save(&stream, writer);
+
+    TDoubleDerivedFieldTester reader = {};
+    ::Load(&stream, reader);
+    EXPECT_EQ(writer.Field.Field1, 55);
+    EXPECT_EQ(writer.Field.Field2, 34);
+    EXPECT_EQ(writer.Field.Field3, 37);
+    EXPECT_EQ(writer.Field.Field4, 77);
+    EXPECT_EQ(reader.Field.Field1, 55);
+    EXPECT_EQ(reader.Field.Field2, 34);
+    EXPECT_EQ(reader.Field.Field3, 37);
+    EXPECT_EQ(reader.Field.Field4, 77);
+}
+
+class TTwoBasesFieldTester
+    : public NYT::NYTree::TYsonStructLite
+{
+public:
+    TTestDerivedTwoBasesConfig Field;
+
+    REGISTER_YSON_STRUCT_LITE(TTwoBasesFieldTester);
+
+    static void Register(TRegistrar registrar)
+    {
+        registrar.Parameter("field", &TThis::Field);
+    }
+};
+
+TEST(TYsonStructTest, SerializableByTraitsDerivedFromTwoExternalizedBases)
+{
+    TTwoBasesFieldTester writer = {};
+    writer.Field = {{{{55, 34}, 37}, 77}, {7, 8}};
+
+    TBufferStream stream;
+
+    ::Save(&stream, writer);
+
+    TTwoBasesFieldTester reader = {};
+    ::Load(&stream, reader);
+    EXPECT_EQ(writer.Field.Field1, 55);
+    EXPECT_EQ(writer.Field.Field2, 34);
+    EXPECT_EQ(writer.Field.Field3, 37);
+    EXPECT_EQ(writer.Field.Field4, 77);
+    EXPECT_EQ(writer.Field.Field5, 7);
+    EXPECT_EQ(writer.Field.Field6, 8);
+    EXPECT_EQ(reader.Field.Field1, 55);
+    EXPECT_EQ(reader.Field.Field2, 34);
+    EXPECT_EQ(reader.Field.Field3, 37);
+    EXPECT_EQ(reader.Field.Field4, 77);
+    EXPECT_EQ(reader.Field.Field5, 7);
+    EXPECT_EQ(reader.Field.Field6, 8);
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 } // namespace

yt/yt/core/ytree/yson_struct-inl.h

@@ -53,30 +53,28 @@ const std::type_info& CallCtor()
 
 //! Creates TSerializer object which has preprocessors applied
 //! to a TStruct object referred to by writable.
-template <std::default_initializable TStruct>
-template <class TSerializer>
-TSerializer TExternalizedYsonStruct<TStruct>::CreateWritable(TStruct& writable)
+template <std::default_initializable TStruct, class TSerializer>
+TSerializer TExternalizedYsonStruct::CreateWritable(TStruct& writable)
 {
-    static_assert(std::derived_from<TSerializer, TExternalizedYsonStruct<TStruct>>);
+    static_assert(std::derived_from<TSerializer, TExternalizedYsonStruct>);
     return TSerializer(&writable);
 }
 
 //! Creates TSerializer object which has preprocessors applied
 //! to a dummy object and has pointer to readOnly assigned afterwards.
-template <std::default_initializable TStruct>
-template <class TSerializer>
-TSerializer TExternalizedYsonStruct<TStruct>::CreateReadOnly(const TStruct& readOnly)
+template <std::default_initializable TStruct, class TSerializer>
+TSerializer TExternalizedYsonStruct::CreateReadOnly(const TStruct& readOnly)
 {
-    static_assert(std::derived_from<TSerializer, TExternalizedYsonStruct<TStruct>>);
+    static_assert(std::derived_from<TSerializer, TExternalizedYsonStruct>);
     auto ret = TSerializer();
-    ret.That = const_cast<TStruct*>(&readOnly);
+    ret.SetThat(const_cast<TStruct*>(&readOnly));
     return ret;
 }
 
 //! We need some writable instance of TStruct to refer to in order
 //! to have a default constructor required by TYsonStructRegistry::InitializeStruct.
 template <std::default_initializable TStruct>
-TStruct* TExternalizedYsonStruct<TStruct>::GetDefault() noexcept
+TStruct* TExternalizedYsonStruct::GetDefault() noexcept
 {
     thread_local TStruct defaultThat = {};
     //! NB: We reset default after every invocation
@@ -211,9 +209,10 @@ template <class TExternal, class TValue>
     // requires std::derived_from<TStruct, TExternalizedYsonStruct<TExternal, TStruct>>
 TYsonStructParameter<TValue>& TYsonStructRegistrar<TStruct>::ExternalClassParameter(const TString& key, TValue(TExternal::*field))
 {
-    static_assert(std::derived_from<TStruct, TExternalizedYsonStruct<TExternal>>);
+    static_assert(std::derived_from<TStruct, TExternalizedYsonStruct>);
+    static_assert(std::same_as<typename TStruct::TExternal, TExternal>);
     auto universalAccessor = [field] (TStruct* serializer) -> auto& {
-        return serializer->That->*field;
+        return serializer->That_->*field;
     };
 
     return ParameterWithUniversalAccessor<TValue>(key, universalAccessor);
@@ -226,7 +225,7 @@ void TYsonStructRegistrar<TStruct>::ExternalPreprocessor(TExternalPreprocessor p
 {
     static_assert(CInvocable<TExternalPreprocessor, void(typename TStruct::TExternal*)>);
     Meta_->RegisterPreprocessor([preprocessor = std::move(preprocessor)] (TYsonStructBase* target) {
-        preprocessor(TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(target)->That);
+        preprocessor(TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(target)->That_);
     });
 }
 
@@ -237,7 +236,7 @@ void TYsonStructRegistrar<TStruct>::ExternalPostprocessor(TExternalPostprocessor
 {
     static_assert(CInvocable<TExternalPostprocessor, void(typename TStruct::TExternal*)>);
     Meta_->RegisterPostprocessor([postprocessor = std::move(postprocessor)] (TYsonStructBase* target) {
-        postprocessor(TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(target)->That);
+        postprocessor(TYsonStructRegistry::Get()->template CachedDynamicCast<TStruct>(target)->That_);
     });
 }
 
@@ -381,6 +380,7 @@ void UpdateYsonStructField(TIntrusivePtr<TDst>& dst, const TIntrusivePtr<TSrc>&
 #undef DEFINE_YSON_STRUCT
 #undef DEFINE_YSON_STRUCT_LITE
 #undef REGISTER_EXTERNALIZED_YSON_STRUCT
+#undef REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT
 
 #define YSON_STRUCT_IMPL__DECLARE_ALIASES(TStruct) \
 private: \
@@ -397,12 +397,16 @@ private: \
         this->SetDefaults(); \
     } \
 
+//! NB(arkady-e1ppa): Alias is used by registrar postprocessors
+//! in order to properly infer template argument.
 #define YSON_STRUCT_EXTERNAL_SERIALIZER_IMPL__DECLARE_ALIASES(TStruct, TSerializer) \
+public: \
+    using TExternal = TStruct; \
+    TStruct* That_ = nullptr; \
 private: \
     using TRegistrar = ::NYT::NYTree::TYsonStructRegistrar<TSerializer>; \
     using TThat = TStruct; \
     friend class ::NYT::NYTree::TYsonStructRegistry; \
-    template <std::default_initializable T> \
     friend class ::NYT::NYTree::TExternalizedYsonStruct; \
     template <class T> \
     friend const std::type_info& ::NYT::NYTree::CallCtor(); \
@@ -453,25 +457,70 @@ TStruct::TStruct() \
     YSON_STRUCT_IMPL__CTOR_BODY(TStruct) \
 }
 
-
 //! NB(arkady-e1ppa): These constructors are only used internally.
 //! Default one is required by TYsonStructRegistry::InitializeStruct.
 //! If you want to create an instance of a TStruct then:
 //! 1) Unless you are working with some serialization logic you shouldn't.
 //! 2) Use TExternalizedYsonStruct::CreateWritable and TExternalizedYsonStruct::CreateReadOnly
 //! instead to guarantee proper initialization as well as const-correctness.
-#define REGISTER_EXTERNALIZED_YSON_STRUCT(TStruct, TSerializer) \
-private: \
+#define EXTERNALIZED_YSON_STRUCT_IMPL__CTORS(TStruct, TSerializer) \
+public: \
+    TSerializer() \
+        : TSerializer(::NYT::NYTree::TExternalizedYsonStruct::template GetDefault<TStruct>()) \
+    { }; \
     explicit TSerializer(TStruct* ptr) \
         : ::NYT::NYTree::TYsonStructFinalClassHolder(std::type_index(typeid(TSerializer))) \
     { \
-        static_assert(std::derived_from<TSerializer, ::NYT::NYTree::TExternalizedYsonStruct<TStruct>>, "Class must inherit from TExternalizedYsonStruct"); \
-        ::NYT::NYTree::TExternalizedYsonStruct<TStruct>::That = ptr; \
+        static_assert(std::derived_from<TSerializer, ::NYT::NYTree::TExternalizedYsonStruct>, "Class must inherit from TExternalizedYsonStruct"); \
+        SetThat(ptr); \
         YSON_STRUCT_LITE_IMPL__CTOR_BODY(TSerializer); \
     } \
-    TSerializer() \
-        : TSerializer(::NYT::NYTree::TExternalizedYsonStruct<TStruct>::GetDefault()) \
-    { }; \
+
+#define REGISTER_EXTERNALIZED_YSON_STRUCT(TStruct, TSerializer) \
+    EXTERNALIZED_YSON_STRUCT_IMPL__CTORS(TStruct, TSerializer) \
+public: \
+    void SetThat(TStruct* ptr) \
+    { \
+        That_ = ptr; \
+    } \
+    YSON_STRUCT_EXTERNAL_SERIALIZER_IMPL__DECLARE_ALIASES(TStruct, TSerializer) \
+
+//! TODO(arkady-e1ppa):
+/*
+    Code below is terrible both in terms of internal implementation and user experience.
+    Ideally we would want to abolish the macro below and instead use magic method
+    "GetThat" whenever we want to assign That_. Properly implementing CreateReadOnly
+    is likely to require some more tinkering but should ultimately be possible to do.
+
+    Magic GetThat should be something along the lines
+    static TStruct* GetThat(this auto& self)
+    {
+        return self.That_;
+    }
+
+    and called as TSerializer::GetThat(*this) which would trigger the correct overload of it and
+    deduce the most derived type in auto which would resolve .That_ as a field of the most derived
+    class.
+
+    This "deducing this" feature will come in C++23 and is likely to be properly supported in clang-18 or 19.
+
+    Another thing to consider is using some method (which is likely outside of standard) to list all
+    direct bases of the type so we can at least remove explicit enumeration of them in the user code.
+
+    Perhaps, there is another approach to consider?
+*/
+
+#define BASE_SET_THAT_ENTRY(TBase) \
+    TBase::SetThat(ptr); \
+
+#define REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT(TStruct, TSerializer, TBases) \
+    EXTERNALIZED_YSON_STRUCT_IMPL__CTORS(TStruct, TSerializer) \
+public: \
+    void SetThat(TStruct* ptr) \
+    { \
+        That_ = ptr; \
+        PP_FOR_EACH(BASE_SET_THAT_ENTRY, TBases) \
+    } \
     YSON_STRUCT_EXTERNAL_SERIALIZER_IMPL__DECLARE_ALIASES(TStruct, TSerializer) \
 
 ////////////////////////////////////////////////////////////////////////////////

yt/yt/core/ytree/yson_struct.h

@@ -147,17 +147,10 @@ class TYsonStructLite
 
 ////////////////////////////////////////////////////////////////////////////////
 
-template <std::default_initializable TStruct>
 class TExternalizedYsonStruct
     : public TYsonStructLite
 {
 public:
-    //! NB(arkady-e1ppa): Alias is used by registrar postprocessors
-    //! in order to properly infer template argument.
-    using TExternal = TStruct;
-
-    TStruct* That = nullptr;
-
     //! NB(arkady-e1ppa): Due to still present bug in clang which makes it
     //! incapable of processing constraints, which refer to class template
     //! parameters, we cannot properly constraint TSerializer here and
@@ -167,15 +160,16 @@ public:
     //! replace class with std::derived_from<TExternalizedYsonStruct<TStruct>>
     //! and remove exposition-only "requires" statements
 
-    template <class TSerializer>
+    template <std::default_initializable TStruct, class TSerializer>
         // requires std::derived_from<TSerializer, TExternalizedYsonStruct<TStruct>>
     static TSerializer CreateWritable(TStruct& writable);
 
-    template <class TSerializer>
+    template <std::default_initializable TStruct, class TSerializer>
         // requires std::derived_from<TSerializer, TExternalizedYsonStruct<TStruct>>
     static TSerializer CreateReadOnly(const TStruct& readOnly);
 
 protected:
+    template <std::default_initializable TStruct>
     static TStruct* GetDefault() noexcept;
 };
 
@@ -356,6 +350,8 @@ void UpdateYsonStructField(TIntrusivePtr<TDst>& dst, const TIntrusivePtr<TSrc>&
 //! Define non-ref-counted Yson external serializer methods and fields.
 #define REGISTER_EXTERNALIZED_YSON_STRUCT(TStruct, TSerializer)
 
+#define REGISTER_DERIVED_EXTERNALIZED_YSON_STRUCT(TStruct, TSerializer, TBases)
+
 ////////////////////////////////////////////////////////////////////////////////
 
 } // namespace NYT::NYTree