YQ Connector: unit test for generic provider in KQP

1. Добавлен юнит-тест на интеграцию YQL Generic provider и KQP.
2. В Generic провайдере обрабатываются все новые параметры из запросов `CREATE EXTERNAL DATA SOURCE`.
3. В Connector API добавлена сущность `Protocol`, которая позволяет указывать протокол соединения между Коннектором и источником данных.

ydb/core/external_sources/CMakeLists.darwin-x86_64.txt

@@ -15,6 +15,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC
   library-cpp-scheme
   ydb-core-base
   ydb-core-protos
+  generic-connector-libcpp
   providers-s3-path_generator
   cpp-client-ydb_params
   cpp-client-ydb_value

ydb/core/external_sources/CMakeLists.linux-aarch64.txt

@@ -16,6 +16,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC
   library-cpp-scheme
   ydb-core-base
   ydb-core-protos
+  generic-connector-libcpp
   providers-s3-path_generator
   cpp-client-ydb_params
   cpp-client-ydb_value

ydb/core/external_sources/CMakeLists.linux-x86_64.txt

@@ -16,6 +16,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC
   library-cpp-scheme
   ydb-core-base
   ydb-core-protos
+  generic-connector-libcpp
   providers-s3-path_generator
   cpp-client-ydb_params
   cpp-client-ydb_value

ydb/core/external_sources/CMakeLists.windows-x86_64.txt

@@ -15,6 +15,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC
   library-cpp-scheme
   ydb-core-base
   ydb-core-protos
+  generic-connector-libcpp
   providers-s3-path_generator
   cpp-client-ydb_params
   cpp-client-ydb_value

ydb/core/external_sources/external_source_factory.cpp

@@ -3,8 +3,10 @@
 #include "external_data_source.h"
 
 #include <util/generic/map.h>
+#include <util/string/cast.h>
 
 #include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
+#include <ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h>
 
 
 namespace NKikimr::NExternalSource {
@@ -33,8 +35,14 @@ private:
 IExternalSourceFactory::TPtr CreateExternalSourceFactory() {
     return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
         {"ObjectStorage", CreateObjectStorageExternalSource()},
-        {"ClickHouse", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})},
-        {"PostgreSQL", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})}
+        {
+            ToString(NYql::NConnector::EExternalDataSource::ClickHouse),
+            CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})
+        },
+        {
+            ToString(NYql::NConnector::EExternalDataSource::PostgreSQL), 
+            CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})
+        }
     });
 }
 

ydb/core/external_sources/ya.make

@@ -10,6 +10,7 @@ PEERDIR(
     library/cpp/scheme
     ydb/core/base
     ydb/core/protos
+    ydb/library/yql/providers/generic/connector/libcpp
     ydb/library/yql/providers/s3/path_generator
     ydb/public/sdk/cpp/client/ydb_params
     ydb/public/sdk/cpp/client/ydb_value

ydb/core/fq/libs/actors/clusters_from_connections.cpp

@@ -102,10 +102,25 @@ void FillGenericClusterConfig(
         clusterCfg.mutable_credentials()->mutable_basic()->set_username(connection.login());
         clusterCfg.mutable_credentials()->mutable_basic()->set_password(connection.password());
         FillClusterAuth(clusterCfg, connection.auth(), authToken, accountIdSignatures);
+
         // Since resolver always returns secure ports, we'll always ask for secure connections
         // between remote Connector and the data source:
         // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp#L24
         clusterCfg.SetUseSsl(true);
+
+        // In YQv1 we just hardcode desired protocols here.
+        // In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params.
+        switch (dataSourceKind) {
+            case NYql::NConnector::NApi::CLICKHOUSE:
+                clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::HTTP);
+                break;
+            case NYql::NConnector::NApi::POSTGRESQL:
+                clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
+                break;
+            default:
+                ythrow yexception() << "Unexpected data source kind: '" 
+                                    << NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
+        }
 }
 
 } //namespace

ydb/core/kqp/ut/federated_query/CMakeLists.txt

@@ -7,4 +7,5 @@
 
 
 add_subdirectory(common)
+add_subdirectory(generic)
 add_subdirectory(s3)

ydb/core/kqp/ut/federated_query/generic/CMakeLists.darwin-x86_64.txt

@@ -0,0 +1,75 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-kqp-ut-federated_query-generic)
+target_compile_options(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/core/kqp
+)
+target_link_libraries(ydb-core-kqp-ut-federated_query-generic PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-testing-unittest_main
+  ydb-core-kqp
+  libs-apache-arrow
+  contrib-libs-fmt
+  kqp-ut-common
+  ut-federated_query-common
+  yql-sql-pg_dummy
+)
+target_link_options(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  -Wl,-platform_version,macos,11.0,11.0
+  -fPIC
+  -fPIC
+  -framework
+  CoreFoundation
+)
+target_sources(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp
+)
+set_property(
+  TARGET
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  SPLIT_FACTOR
+  1
+)
+add_yunittest(
+  NAME
+  ydb-core-kqp-ut-federated_query-generic
+  TEST_TARGET
+  ydb-core-kqp-ut-federated_query-generic
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  PROCESSORS
+  1
+)
+target_allocator(ydb-core-kqp-ut-federated_query-generic
+  system_allocator
+)
+vcs_info(ydb-core-kqp-ut-federated_query-generic)

ydb/core/kqp/ut/federated_query/generic/CMakeLists.linux-aarch64.txt

@@ -0,0 +1,78 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-kqp-ut-federated_query-generic)
+target_compile_options(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/core/kqp
+)
+target_link_libraries(ydb-core-kqp-ut-federated_query-generic PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  cpp-testing-unittest_main
+  ydb-core-kqp
+  libs-apache-arrow
+  contrib-libs-fmt
+  kqp-ut-common
+  ut-federated_query-common
+  yql-sql-pg_dummy
+)
+target_link_options(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+)
+target_sources(ydb-core-kqp-ut-federated_query-generic PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp
+)
+set_property(
+  TARGET
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  SPLIT_FACTOR
+  1
+)
+add_yunittest(
+  NAME
+  ydb-core-kqp-ut-federated_query-generic
+  TEST_TARGET
+  ydb-core-kqp-ut-federated_query-generic
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  ydb-core-kqp-ut-federated_query-generic
+  PROPERTY
+  PROCESSORS
+  1
+)
+target_allocator(ydb-core-kqp-ut-federated_query-generic
+  cpp-malloc-jemalloc
+)
+vcs_info(ydb-core-kqp-ut-federated_query-generic)