Replay GetTableRange (#8760)

ydb/library/yql/ast/ya.make

@@ -24,6 +24,7 @@ SRCS(
 )
 
 PEERDIR(
+    contrib/libs/openssl
     library/cpp/colorizer
     library/cpp/containers/sorted_vector
     library/cpp/containers/stack_vector

ydb/library/yql/ast/yql_expr.cpp

@@ -19,6 +19,8 @@
 #include <util/digest/numeric.h>
 #include <util/string/cast.h>
 
+#include <openssl/sha.h>
+
 #include <map>
 #include <unordered_set>
 
@@ -3667,6 +3669,53 @@ bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNod
     return CompareExpressions(l, r, map, level, visited);
 }
 
+class TCacheKeyBuilder {
+public:
+    TString Process(const TExprNode& root) {
+        SHA256_Init(&Sha);
+        unsigned char hash[SHA256_DIGEST_LENGTH];
+        Visit(root);
+        SHA256_Final(hash, &Sha);
+        return TString((const char*)hash, sizeof(hash));
+    }
+
+private:
+    void Visit(const TExprNode& node) {
+        auto [it, inserted] = Visited.emplace(&node, Visited.size());
+        SHA256_Update(&Sha, &it->second, sizeof(it->second));
+        if (!inserted) {
+            return;
+        }
+
+        ui32 type = node.Type();
+        SHA256_Update(&Sha, &type, sizeof(type));
+        if (node.Type() == TExprNode::EType::Atom || node.Type() == TExprNode::EType::Callable) {
+            ui32 textLen = node.Content().size();
+            SHA256_Update(&Sha, &textLen, sizeof(textLen));
+            SHA256_Update(&Sha, node.Content().Data(), textLen);
+        }
+
+        if (node.Type() == TExprNode::EType::Atom || node.Type() == TExprNode::EType::Argument || node.Type() == TExprNode::EType::World) {
+            return;
+        }
+
+        ui32 len = node.ChildrenSize();
+        SHA256_Update(&Sha, &len, sizeof(len));
+        for (const auto& child : node.Children()) {
+            Visit(*child);
+        }
+    }
+
+private:
+    SHA256_CTX Sha;
+    TNodeMap<ui64> Visited;
+};
+
+TString MakeCacheKey(const TExprNode& root) {
+    TCacheKeyBuilder builder;
+    return builder.Process(root);
+}
+
 void GatherParents(const TExprNode& node, TParentsMap& parentsMap) {
     parentsMap.clear();
     TNodeSet visisted;

ydb/library/yql/ast/yql_expr.h

@@ -2837,6 +2837,8 @@ bool CompareExprTrees(const TExprNode*& one, const TExprNode*& two);
 
 bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNodeMap<ui32>& argsMap);
 
+TString MakeCacheKey(const TExprNode& root);
+
 void GatherParents(const TExprNode& node, TParentsMap& parentsMap);
 
 struct TConvertToAstSettings {

ydb/library/yql/core/services/ya.make

@@ -16,7 +16,6 @@ SRCS(
 )
 
 PEERDIR(
-    contrib/libs/openssl
     library/cpp/string_utils/base64
     library/cpp/yson
     ydb/library/yql/ast/serialize

ydb/library/yql/core/services/yql_eval_expr.cpp

@@ -17,8 +17,6 @@
 #include <library/cpp/yson/node/node_io.h>
 #include <library/cpp/string_utils/base64/base64.h>
 
-#include <openssl/sha.h>
-
 #include <util/string/builder.h>
 
 namespace NYql {
@@ -45,53 +43,6 @@ static THashSet<TStringBuf> SubqueryExpandFuncs = {
     TStringBuf("SubqueryAssumeOrderBy")
 };
 
-class TCacheKeyBuilder {
-public:
-    TString Process(const TExprNode& root) {
-        SHA256_Init(&Sha);
-        unsigned char hash[SHA256_DIGEST_LENGTH];
-        Visit(root);
-        SHA256_Final(hash, &Sha);
-        return TString((const char*)hash, sizeof(hash));
-    }
-
-private:
-    void Visit(const TExprNode& node) {
-        auto [it, inserted] = Visited.emplace(&node, Visited.size());
-        SHA256_Update(&Sha, &it->second, sizeof(it->second));
-        if (!inserted) {
-            return;
-        }
-
-        ui32 type = node.Type();
-        SHA256_Update(&Sha, &type, sizeof(type));
-        if (node.Type() == TExprNode::EType::Atom || node.Type() == TExprNode::EType::Callable) {
-            ui32 textLen = node.Content().size();
-            SHA256_Update(&Sha, &textLen, sizeof(textLen));
-            SHA256_Update(&Sha, node.Content().Data(), textLen);
-        }
-
-        if (node.Type() == TExprNode::EType::Atom || node.Type() == TExprNode::EType::Argument || node.Type() == TExprNode::EType::World) {
-            return;
-        }
-
-        ui32 len = node.ChildrenSize();
-        SHA256_Update(&Sha, &len, sizeof(len));
-        for (const auto& child : node.Children()) {
-            Visit(*child);
-        }
-    }
-
-private:
-    SHA256_CTX Sha;
-    TNodeMap<ui64> Visited;
-};
-
-TString MakeCacheKey(const TExprNode& root) {
-    TCacheKeyBuilder builder;
-    return builder.Process(root);
-}
-
 bool CheckPendingArgs(const TExprNode& root, TNodeSet& visited, TNodeMap<const TExprNode*>& activeArgs, const TNodeMap<ui32>& externalWorlds, TExprContext& ctx,
     bool underTypeOf, bool& hasUnresolvedTypes) {
     if (!visited.emplace(&root).second) {

ydb/library/yql/providers/yt/gateway/qplayer/yql_yt_qplayer_gateway.cpp

@@ -20,6 +20,7 @@ namespace {
 
 const TString YtGateway_CanonizePaths = "YtGateway_CanonizePaths";
 const TString YtGateway_GetTableInfo = "YtGateway_GetTableInfo";
+const TString YtGateway_GetTableRange = "YtGateway_GetTableRange";
 const TString YtGateway_GetFolder = "YtGateway_GetFolder";
 const TString YtGateway_GetFolders = "YtGateway_GetFolders";
 const TString YtGateway_ResolveLinks = "YtGateway_ResolveLinks";
@@ -300,12 +301,106 @@ public:
             });
     }
 
+    static TString MakeGetTableRangeKey(const TTableRangeOptions& options) {
+        auto keyNode = NYT::TNode()
+            ("Cluster", options.Cluster())
+            ("Prefix", options.Prefix())
+            ("Suffix", options.Suffix());
+
+        if (options.Filter()) {
+            keyNode("Filter", MakeCacheKey(*options.Filter()));
+        }
+
+        return MakeHash(NYT::NodeToCanonicalYsonString(keyNode, NYT::NYson::EYsonFormat::Binary));
+    }
+
     NThreading::TFuture<TTableRangeResult> GetTableRange(TTableRangeOptions&& options) final {
         if (QContext_.CanRead()) {
-            throw yexception() << "Can't replay GetTableRange";
+            TTableRangeResult res;
+            res.SetSuccess();
+            auto key = MakeGetTableRangeKey(options);
+            auto item = QContext_.GetReader()->Get({YtGateway_GetTableRange, key}).GetValueSync();
+            if (!item) {
+                throw yexception() << "Missing replay data";
+            }
+
+            auto listNode = NYT::NodeFromYsonString(item->Value);
+            for (const auto& valueNode : listNode.AsList()) {
+                TCanonizedPath p;
+                p.Path = valueNode["Path"].AsString();
+                if (valueNode.HasKey("Columns")) {
+                    p.Columns.ConstructInPlace();
+                    for (const auto& c : valueNode["Columns"].AsList()) {
+                        p.Columns->push_back(c.AsString());
+                    }
+                }
+
+                if (valueNode.HasKey("Ranges")) {
+                    p.Ranges.ConstructInPlace();
+                    for (const auto& r : valueNode["Ranges"].AsString()) {
+                        NYT::TReadRange range;
+                        NYT::Deserialize(range, r);
+                        p.Ranges->push_back(range);
+                    }
+                }
+
+                if (valueNode.HasKey("AdditionalAttributes")) {
+                    p.AdditionalAttributes = valueNode["AdditionalAttributes"].AsString();
+                }
+
+                res.Tables.push_back(p);
+            }
+
+            return NThreading::MakeFuture<TTableRangeResult>(res);
         }
 
-        return Inner_->GetTableRange(std::move(options));
+        auto optionsDup = options;
+        return Inner_->GetTableRange(std::move(options))
+            .Subscribe([optionsDup, qContext = QContext_](const NThreading::TFuture<TTableRangeResult>& future) {
+                if (!qContext.CanWrite() || future.HasException()) {
+                    return;
+                }
+
+                const auto& res = future.GetValueSync();
+                if (!res.Success()) {
+                    return;
+                }
+
+                const auto& key = MakeGetTableRangeKey(optionsDup);
+                auto listNode = NYT::TNode::CreateList();
+                for (const auto& t : res.Tables) {
+                    listNode.Add();
+                    auto& valueNode = listNode.AsList().back();
+                    valueNode("Path", t.Path);
+                    if (t.Columns) {
+                        NYT::TNode columnsNode = NYT::TNode::CreateList();
+                        for (const auto& c : *t.Columns) {
+                            columnsNode.Add(NYT::TNode(c));
+                        }
+
+                        valueNode("Columns", columnsNode);
+                    }
+
+                    if (t.Ranges) {
+                        NYT::TNode rangesNode = NYT::TNode::CreateList();
+                        for (const auto& r : *t.Ranges) {
+                            NYT::TNode rangeNode;
+                            NYT::TNodeBuilder builder(&rangeNode);
+                            NYT::Serialize(r, &builder);
+                            rangesNode.Add(rangeNode);
+                        }
+
+                        valueNode("Ranges", rangesNode);
+                    }
+
+                    if (t.AdditionalAttributes) {
+                        valueNode("AdditionalAttributes", NYT::TNode(*t.AdditionalAttributes));
+                    }
+                }
+
+                auto value = NYT::NodeToYsonString(listNode, NYT::NYson::EYsonFormat::Binary);
+                qContext.GetWriter()->Put({YtGateway_GetTableRange, key}, value).GetValueSync();
+        });
     }
 
     static TString MakeGetFolderKey(const TFolderOptions& options) {