golang pg test (#8132)

.github/config/muted_ya.txt

@@ -101,6 +101,81 @@ ydb/tests/functional/tenants test_dynamic_tenants.py.*
 ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
 ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestArrayValueBackend]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBinaryByteSliceToInt]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBinaryByteSlicetoUUID]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBindError]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCommit]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnListen]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnPing]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnUnlistenAll]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnUnlisten]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnectorWithNoticeHandler_Simple]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestConnectorWithNotificationHandler_Simple]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelBegin]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestContextCancelExec]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryCancelRace]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyFromError]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInBinaryError]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInMultipleValues]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInRaiseStmtTrigger]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInStmtAffectedRows]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInTypes]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyInWrongType]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopyRespLoopConnectionError]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestCopySyntaxError]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEmptyQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEncodeAndParseTs]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestEncodeDecode]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorClass]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorDuringStartup]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnExec]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnQueryRowSimpleQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestErrorOnQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestExec]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestFormatTsBackend]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestHasCorrectRootGroupPermissions]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestInfinityTimestamp]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue1046]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue1062]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestIssue186]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerFailedQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerListen]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerPing]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerReconnect]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerUnlistenAll]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestListenerUnlisten]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_Connect]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_Driver]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNewConnector_WorksWithOpenDB]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNotifyExtra]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestNullAfterNonNull]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestParseErrorInExtendedQuery]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestPing]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryCancelledReused]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestQueryRowBugWorkaround]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestReconnect]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestReturning]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestRowsResultTag]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestRuntimeParameters]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.Background]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.WithTimeout]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext/context.WithTimeout_exceeded]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtExecContext]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.Background]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.WithTimeout]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext/context.WithTimeout_exceeded]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStmtQueryContext]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestStringWithNul]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimeWithTimezone]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimestampWithTimeZone]
+ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTxOptions]
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
 ydb/core/kqp/ut/query KqpStats.SysViewClientLost
 ydb/core/kqp/ut/olap KqpOlap.ManyColumnShards

.gitignore

@@ -7,6 +7,8 @@
 !*/
 # Unignore all files inside canondata dir
 !**/canondata/**
+# Allow docker files
+!Dockerfile
 
 /canonization_show_res.log
 

ydb/tests/postgres_integrations/go-libpq/conftest.py

@@ -0,0 +1,6 @@
+import ydb.tests.postgres_integrations.library
+import pytest
+
+
+def pytest_collection_finish(session: pytest.Session):
+    ydb.tests.postgres_integrations.library.pytest_collection_finish(session)

ydb/tests/postgres_integrations/go-libpq/data/.gitignore

@@ -0,0 +1,3 @@
+/exchange/
+/sources/
+/test-result/

ydb/tests/postgres_integrations/go-libpq/data/Dockerfile

@@ -0,0 +1,15 @@
+# For docker context at root git directory
+
+FROM golang:1.20
+
+WORKDIR /project/sources/
+
+COPY patch.diff /patch.diff
+COPY docker-init.bash /docker-init.bash
+RUN /docker-init.bash
+
+COPY common-go-scripts/go-run-separate-tests.bash /go-run-separate-tests.bash
+
+COPY docker-start.bash /docker-start.bash
+
+CMD [ "/docker-start.bash" ]

ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eu
+
+ONE_TEST_TIMEOUT=5s
+TEST_BINARY=./test.binary
+
+echo "Get test list"
+TESTS=$($TEST_BINARY --test.list "^Test" | sort)
+
+
+echo "Shell $SHELL"
+
+rm -f /test-result/raw/result.txt
+for TEST_NAME in $TESTS; do
+    echo -n "Test: $TEST_NAME "
+    if echo "$TEST_NAME" | grep -Eq "$YDB_PG_TESTFILTER"; then
+        echo start
+    else
+        echo skip
+        continue
+    fi
+    CMD="$TEST_BINARY --test.run '^$TEST_NAME\$' --test.v --test.timeout='$ONE_TEST_TIMEOUT'"
+    echo "$CMD"
+    bash -c "$CMD" >> /test-result/raw/result.txt 2>&1 || true
+done
+
+go-junit-report < /test-result/raw/result.txt > /test-result/raw/result.xml

ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml

@@ -0,0 +1,22 @@
+version: "3"
+services:
+  project:
+    network_mode: host
+
+    image: ydb-test/go-pqlib
+    build:
+      context: ../../..
+      dockerfile: languages/go/libpq/Dockerfile
+      network: host
+    environment:
+      - PGUSER=${YDB_PG_USER:-root}
+      - PGPASSWORD=${YDB_PG_PASSWORD:-1234}
+      - PGHOST=${YDB_PG_HOST:-ydb}
+      - PGPORT=${YDB_PG_PORT:-5432}
+      - PGDATABASE=${YDB_PG_DATABASE:-local}
+      - PQGOSSLTESTS=0
+      - PQSSLCERTTEST_PATH=certs
+      - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-}
+    volumes:
+      - ./exchange:/exchange
+      - ./test-result/:/test-result

ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml

@@ -0,0 +1,40 @@
+version: "3"
+services:
+  ydb:
+    image: ghcr.io/ydb-platform/local-ydb:nightly
+    environment:
+      - "YDB_DEFAULT_LOG_LEVEL=DEBUG"
+      - "GRPC_TLS_PORT=2135"
+      - "GRPC_PORT=2136"
+      - "MON_PORT=8765"
+      - "YDB_USE_IN_MEMORY_PDISKS=true"
+      - "POSTGRES_USER=${YDB_PG_USER:-root}"
+      - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}"
+      - "YDB_FEATURE_FLAGS=enable_temp_tables"
+      - "YDB_TABLE_ENABLE_PREPARED_DDL=true"
+    healthcheck:
+      test: "/bin/sh /health_check"
+      interval: 1s
+      start_period: 1m
+  project:
+    depends_on:
+      ydb:
+        condition: service_healthy
+
+    image: ydb-test/go-pqlib
+    build:
+      context: ../../..
+      dockerfile: languages/go/libpq/Dockerfile
+      network: host
+    environment:
+      - PGUSER=${YDB_PG_USER:-root}
+      - PGPASSWORD=${YDB_PG_PASSWORD:-1234}
+      - PGHOST=${YDB_PG_HOST:-ydb}
+      - PGPORT=${YDB_PG_PORT:-5432}
+      - PGDATABASE=${YDB_PG_DATABASE:-/local}
+      - PQGOSSLTESTS=0
+      - PQSSLCERTTEST_PATH=certs
+      - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-}
+    volumes:
+      - ./exchange:/exchange
+      - ./test-result/:/test-result

ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -eu
+
+apt-get update && apt-get install -y patch
+
+go install github.com/jstemmer/go-junit-report/v2@v2.0.0
+
+mkdir -p /original-sources
+cd /original-sources
+
+wget https://github.com/lib/pq/archive/refs/tags/v1.10.9.tar.gz -O libpq.tar.gz
+tar --strip-components=1 -zxvf libpq.tar.gz
+rm -f libpq.tar.gz
+
+mkdir -p /project/sources/
+cp -R /original-sources/. /project/sources/
+
+cd /project/sources/
+[ -e /patch.diff ] && patch -s -p0 < /patch.diff
+
+# cache binary
+echo "Build test binary"
+go test -c -o ./test.binary

ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -eu
+
+echo "Start script"
+
+rm -rf /test-result 2> /dev/null || true
+
+mkdir -p /exchange
+mkdir -p /test-result/raw
+
+if [ -e /exchange/sources ]; then
+    echo "Skip prepare sources, because it is exist"
+else
+    echo "Copy sources"
+    mkdir -p /exchange/sources
+    cp -R /project/sources/. /exchange/sources
+    chmod -R a+rw /exchange/sources
+fi
+
+cd /project/sources/
+
+export YDB_PG_TESTFILTER="${YDB_PG_TESTFILTER:-}"  # set YDB_PG_TESTNAME to empty string if it not set
+
+echo "Run tests: '$YDB_PG_TESTFILTER'"
+
+echo "Start test"
+
+mkdir -p /test-result/raw
+PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash
+
+if [ -n "${YDB_PG_TESTFILTER:-}" ]; then
+    cat /test-result/raw/result.txt
+fi
+
+chmod -R a+rw /test-result