YQL-16602 support of conversions in catalog, start transaction during

init

ydb/library/yql/parser/pg_catalog/CMakeLists.darwin-x86_64.txt

@@ -61,6 +61,12 @@ get_built_tool_path(
   tools/rescompiler/bin
   rescompiler
 )
+get_built_tool_path(
+  TOOL_rescompiler_bin
+  TOOL_rescompiler_dependency
+  tools/rescompiler/bin
+  rescompiler
+)
 
 add_library(yql-parser-pg_catalog)
 target_link_libraries(yql-parser-pg_catalog PUBLIC
@@ -88,6 +94,7 @@ target_sources(yql-parser-pg_catalog.global PRIVATE
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/69148841cb58a458173a42fc4a2d34a5.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/ec0b500b671e1ef6896201d71c27c3a9.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/35d2aa467812d7360a78c1fae061c821.cpp
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
 )
 resources(yql-parser-pg_catalog.global
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/eadca93c2963e941669f9c2a2aa57764.cpp
@@ -152,3 +159,10 @@ resources(yql-parser-pg_catalog.global
   KEYS
   pg_amop.dat
 )
+resources(yql-parser-pg_catalog.global
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
+  INPUTS
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat
+  KEYS
+  pg_conversion.dat
+)

ydb/library/yql/parser/pg_catalog/CMakeLists.linux-aarch64.txt

@@ -61,6 +61,12 @@ get_built_tool_path(
   tools/rescompiler/bin
   rescompiler
 )
+get_built_tool_path(
+  TOOL_rescompiler_bin
+  TOOL_rescompiler_dependency
+  tools/rescompiler/bin
+  rescompiler
+)
 
 add_library(yql-parser-pg_catalog)
 target_link_libraries(yql-parser-pg_catalog PUBLIC
@@ -90,6 +96,7 @@ target_sources(yql-parser-pg_catalog.global PRIVATE
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/69148841cb58a458173a42fc4a2d34a5.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/ec0b500b671e1ef6896201d71c27c3a9.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/35d2aa467812d7360a78c1fae061c821.cpp
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
 )
 resources(yql-parser-pg_catalog.global
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/eadca93c2963e941669f9c2a2aa57764.cpp
@@ -154,3 +161,10 @@ resources(yql-parser-pg_catalog.global
   KEYS
   pg_amop.dat
 )
+resources(yql-parser-pg_catalog.global
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
+  INPUTS
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat
+  KEYS
+  pg_conversion.dat
+)

ydb/library/yql/parser/pg_catalog/CMakeLists.linux-x86_64.txt

@@ -61,6 +61,12 @@ get_built_tool_path(
   tools/rescompiler/bin
   rescompiler
 )
+get_built_tool_path(
+  TOOL_rescompiler_bin
+  TOOL_rescompiler_dependency
+  tools/rescompiler/bin
+  rescompiler
+)
 
 add_library(yql-parser-pg_catalog)
 target_link_libraries(yql-parser-pg_catalog PUBLIC
@@ -90,6 +96,7 @@ target_sources(yql-parser-pg_catalog.global PRIVATE
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/69148841cb58a458173a42fc4a2d34a5.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/ec0b500b671e1ef6896201d71c27c3a9.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/35d2aa467812d7360a78c1fae061c821.cpp
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
 )
 resources(yql-parser-pg_catalog.global
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/eadca93c2963e941669f9c2a2aa57764.cpp
@@ -154,3 +161,10 @@ resources(yql-parser-pg_catalog.global
   KEYS
   pg_amop.dat
 )
+resources(yql-parser-pg_catalog.global
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
+  INPUTS
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat
+  KEYS
+  pg_conversion.dat
+)

ydb/library/yql/parser/pg_catalog/CMakeLists.windows-x86_64.txt

@@ -61,6 +61,12 @@ get_built_tool_path(
   tools/rescompiler/bin
   rescompiler
 )
+get_built_tool_path(
+  TOOL_rescompiler_bin
+  TOOL_rescompiler_dependency
+  tools/rescompiler/bin
+  rescompiler
+)
 
 add_library(yql-parser-pg_catalog)
 target_link_libraries(yql-parser-pg_catalog PUBLIC
@@ -88,6 +94,7 @@ target_sources(yql-parser-pg_catalog.global PRIVATE
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/69148841cb58a458173a42fc4a2d34a5.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/ec0b500b671e1ef6896201d71c27c3a9.cpp
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/35d2aa467812d7360a78c1fae061c821.cpp
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
 )
 resources(yql-parser-pg_catalog.global
   ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/eadca93c2963e941669f9c2a2aa57764.cpp
@@ -152,3 +159,10 @@ resources(yql-parser-pg_catalog.global
   KEYS
   pg_amop.dat
 )
+resources(yql-parser-pg_catalog.global
+  ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/pg_catalog/d248668b32b6a4d36f76898f91cda187.cpp
+  INPUTS
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat
+  KEYS
+  pg_conversion.dat
+)

ydb/library/yql/parser/pg_catalog/catalog.cpp

@@ -38,6 +38,8 @@ using TAmOps = THashMap<std::tuple<ui32, ui32, ui32, ui32>, TAmOpDesc>;
 
 using TAmProcs = THashMap<std::tuple<ui32, ui32, ui32, ui32>, TAmProcDesc>;
 
+using TConversions = THashMap<std::pair<TString, TString>, TConversionDesc>;
+
 bool IsCompatibleTo(ui32 actualTypeId, ui32 expectedTypeId, const TTypes& types) {
     if (actualTypeId == expectedTypeId) {
         return true;
@@ -372,7 +374,9 @@ public:
         } else if (key == "prosrc") {
             LastProc.Src = value;
         } else if (key == "prolang") {
-            IsSupported = false;
+            if (value != "c") {
+                IsSupported = false;
+            }
         } else if (key == "proargtypes") {
             TVector<TString> strArgs;
             Split(value, " ", strArgs);
@@ -1092,6 +1096,46 @@ private:
     bool IsSupported = true;
 };
 
+class TConversionsParser : public TParser {
+public:
+    TConversionsParser(TConversions& conversions, const THashMap<TString, TVector<ui32>>& procByName)
+        : Conversions(conversions)
+        , ProcByName(procByName)
+    {}
+
+    void OnKey(const TString& key, const TString& value) override {
+        if (key == "oid") {
+            LastConversion.ConversionId = FromString<ui32>(value);
+        } else if (key == "conforencoding") {
+            Y_ENSURE(value.StartsWith("PG_"));
+            LastConversion.From = value.substr(3);
+        } else if (key == "contoencoding") {
+            Y_ENSURE(value.StartsWith("PG_"));
+            LastConversion.To = value.substr(3);
+        } else if (key == "conproc") {
+            auto found = ProcByName.FindPtr(value);
+            if (found && found->size() == 1) {
+                LastConversion.ProcId = found->front();
+            }
+        }
+    }
+
+    void OnFinish() override {
+        if (LastConversion.ProcId) {
+            Conversions[std::make_pair(LastConversion.From, LastConversion.To)] = LastConversion;
+        }
+
+        LastConversion = TConversionDesc();
+    }
+
+private:
+    TConversions& Conversions;
+
+    const THashMap<TString, TVector<ui32>>& ProcByName;
+
+    TConversionDesc LastConversion;
+};
+
 TOperators ParseOperators(const TString& dat, const THashMap<TString, ui32>& typeByName,
     const TTypes& types, const THashMap<TString, TVector<ui32>>& procByName, const TProcs& procs) {
     TOperators ret;
@@ -1163,6 +1207,13 @@ TAmProcs ParseAmProcs(const TString& dat, const THashMap<TString, ui32>& typeByN
     return ret;
 }
 
+TConversions ParseConversions(const TString& dat, const THashMap<TString, TVector<ui32>>& procByName) {
+    TConversions ret;
+    TConversionsParser parser(ret, procByName);
+    parser.Do(dat);
+    return ret;
+}
+
 struct TCatalog {
     TCatalog() {
         TString typeData;
@@ -1183,6 +1234,8 @@ struct TCatalog {
         Y_ENSURE(NResource::FindExact("pg_amproc.dat", &amProcData));
         TString amOpData;
         Y_ENSURE(NResource::FindExact("pg_amop.dat", &amOpData));
+        TString conversionData;
+        Y_ENSURE(NResource::FindExact("pg_conversion.dat", &conversionData));
         THashMap<ui32, TLazyTypeInfo> lazyTypeInfos;
         Types = ParseTypes(typeData, lazyTypeInfos);
         for (const auto& [k, v] : Types) {
@@ -1333,6 +1386,8 @@ struct TCatalog {
                 }
             }
         }
+
+        Conversions = ParseConversions(conversionData, ProcByName);
     }
 
     static const TCatalog& Instance() {
@@ -1347,6 +1402,7 @@ struct TCatalog {
     TOpClasses OpClasses;
     TAmOps AmOps;
     TAmProcs AmProcs;
+    TConversions Conversions;
     THashMap<TString, TVector<ui32>> ProcByName;
     THashMap<TString, ui32> TypeByName;
     THashMap<std::pair<ui32, ui32>, ui32> CastsByDir;
@@ -2421,6 +2477,21 @@ const TAmProcDesc& LookupAmProc(ui32 familyId, ui32 num, ui32 leftType, ui32 rig
     return *amProcPtr;
 }
 
+bool HasConversion(const TString& from, const TString& to) {
+    const auto &catalog = TCatalog::Instance();
+    return catalog.Conversions.contains(std::make_pair(from, to));
+}
+
+const TConversionDesc& LookupConversion(const TString& from, const TString& to) {
+    const auto& catalog = TCatalog::Instance();
+    auto convPtr = catalog.Conversions.FindPtr(std::make_pair(from, to));
+    if (!convPtr) {
+        throw yexception() << "No such conversion from " << from << " to " << to;
+    }
+
+    return *convPtr;
+}
+
 bool IsCompatibleTo(ui32 actualType, ui32 expectedType) {
     const auto& catalog = TCatalog::Instance();
     return IsCompatibleTo(actualType, expectedType, catalog.Types);

ydb/library/yql/parser/pg_catalog/catalog.h

@@ -195,6 +195,13 @@ enum class EHashAmProcNum {
     Hash = 1
 };
 
+struct TConversionDesc {
+    ui32 ConversionId = 0;
+    TString From;
+    TString To;
+    ui32 ProcId = 0;
+};
+
 const TProcDesc& LookupProc(const TString& name, const TVector<ui32>& argTypeIds);
 const TProcDesc& LookupProc(ui32 procId, const TVector<ui32>& argTypeIds);
 const TProcDesc& LookupProc(ui32 procId);
@@ -230,6 +237,9 @@ const TAmOpDesc& LookupAmOp(ui32 familyId, ui32 strategy, ui32 leftType, ui32 ri
 bool HasAmProc(ui32 familyId, ui32 num, ui32 leftType, ui32 rightType);
 const TAmProcDesc& LookupAmProc(ui32 familyId, ui32 num, ui32 leftType, ui32 rightType);
 
+bool HasConversion(const TString& from, const TString& to);
+const TConversionDesc& LookupConversion(const TString& from, const TString& to);
+
 bool IsCompatibleTo(ui32 actualType, ui32 expectedType);
 
 inline bool IsArrayType(const TTypeDesc& typeDesc) noexcept {

ydb/library/yql/parser/pg_catalog/ut/catalog_ut.cpp

@@ -197,3 +197,14 @@ Y_UNIT_TEST_SUITE(TOpClassesTests) {
         UNIT_ASSERT_VALUES_EQUAL(ret.FamilyId, 1977);
    }
 }
+
+Y_UNIT_TEST_SUITE(TConversionTests) {
+    Y_UNIT_TEST(TestMissing) {
+        UNIT_ASSERT_EXCEPTION(LookupConversion("foo", "bar"), yexception);
+    }
+
+   Y_UNIT_TEST(TestOk) {
+        auto procId = LookupConversion("LATIN1", "UTF8").ProcId;
+        UNIT_ASSERT_VALUES_EQUAL(LookupProc(procId).Name, "iso8859_1_to_utf8");
+   }
+}

ydb/library/yql/parser/pg_catalog/ya.make

@@ -9,6 +9,7 @@ RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_opfamily.dat pg_opfamil
 RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_opclass.dat pg_opclass.dat)
 RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_amproc.dat pg_amproc.dat)
 RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_amop.dat pg_amop.dat)
+RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat pg_conversion.dat)
 
 SRCS(
     catalog.cpp

ydb/library/yql/parser/pg_wrapper/CMakeLists.darwin-x86_64.txt

@@ -147,6 +147,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arrow.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arrow_impl.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/conversion.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/parser.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/thread_inits.c
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
@@ -156,6 +157,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.1.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.2.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.3.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/recovery.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp

ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-aarch64.txt

@@ -146,6 +146,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arrow.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/arrow_impl.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/conversion.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/parser.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/thread_inits.c
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
@@ -155,6 +156,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.1.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.2.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/pg_kernels.3.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/recovery.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp