[yt provider] Properly clear constraints in YqlRowSpec (#8350)

ydb/library/yql/providers/yt/lib/row_spec/yql_row_spec.cpp

@@ -437,7 +437,7 @@ bool TYqlRowSpecInfo::ParseType(const NYT::TNode& rowSpecAttr, TExprContext& ctx
 
 bool TYqlRowSpecInfo::ParseSort(const NYT::TNode& rowSpecAttr, TExprContext& ctx, const TPositionHandle& pos) {
     if (rowSpecAttr.HasKey(RowSpecAttrSortMembers) || rowSpecAttr.HasKey(RowSpecAttrSortedBy) || rowSpecAttr.HasKey(RowSpecAttrSortDirections)) {
-        ClearSortness();
+        ClearSortness(ctx);
     }
     if (rowSpecAttr.HasKey(RowSpecAttrSortDirections)) {
         for (auto& item: rowSpecAttr[RowSpecAttrSortDirections].AsList()) {
@@ -559,7 +559,7 @@ void TYqlRowSpecInfo::ParseConstraints(const NYT::TNode& rowSpecAttr) {
 
 bool TYqlRowSpecInfo::ValidateSort(const TYTSortInfo& sortInfo, TExprContext& ctx, const TPositionHandle& pos) {
     if (sortInfo.Keys.empty() && IsSorted()) {
-        ClearSortness();
+        ClearSortness(ctx);
         if (!ctx.AddWarning(YqlIssue(ctx.GetPosition(pos), EYqlIssueCode::TIssuesIds_EIssueCode_YT_ROWSPEC_DIFF_SORT,
             "Table attribute '_yql_row_spec' defines sorting, but the table is not really sorted. The sorting will be ignored."))) {
             return false;
@@ -573,13 +573,13 @@ bool TYqlRowSpecInfo::ValidateSort(const TYTSortInfo& sortInfo, TExprContext& ct
     } else if (IsSorted()) {
         bool diff = false;
         if (SortedBy.size() > sortInfo.Keys.size()) {
-            ClearSortness(sortInfo.Keys.size());
+            ClearSortness(ctx, sortInfo.Keys.size());
             diff = true;
         }
         auto backendSort = GetForeignSort();
         for (size_t i = 0; i < backendSort.size(); ++i) {
             if (backendSort[i].first != sortInfo.Keys[i].first || backendSort[i].second != (bool)sortInfo.Keys[i].second) {
-                ClearSortness(i);
+                ClearSortness(ctx, i);
                 diff = true;
                 break;
             }
@@ -1396,7 +1396,7 @@ const TStructExprType* TYqlRowSpecInfo::GetExtendedType(TExprContext& ctx) const
     return extended ? ctx.MakeType<TStructExprType>(items) : Type;
 }
 
-bool TYqlRowSpecInfo::CopySortness(const TYqlRowSpecInfo& from, ECopySort mode) {
+bool TYqlRowSpecInfo::CopySortness(TExprContext& ctx, const TYqlRowSpecInfo& from, ECopySort mode) {
     SortDirections = from.SortDirections;
     SortMembers = from.SortMembers;
     SortedBy = from.SortedBy;
@@ -1408,16 +1408,16 @@ bool TYqlRowSpecInfo::CopySortness(const TYqlRowSpecInfo& from, ECopySort mode)
         for (size_t i = 0; i < SortMembers.size(); ++i) {
             const auto itemNdx = Type->FindItem(SortMembers[i]);
             if (!itemNdx || (SortedBy[i] == SortMembers[i] && Type->GetItems()[*itemNdx]->GetItemType() != SortedByTypes[i])) {
-                sortIsChanged = ClearSortness(i);
+                sortIsChanged = ClearSortness(ctx, i);
                 break;
             } else if (ECopySort::Pure == mode && SortedBy[i] != SortMembers[i]) {
-                sortIsChanged = ClearSortness(i);
+                sortIsChanged = ClearSortness(ctx, i);
                 break;
             }
         }
         if (ECopySort::WithCalc != mode) {
             if (SortMembers.size() < SortedBy.size()) {
-                sortIsChanged = ClearSortness(SortMembers.size()) || sortIsChanged;
+                sortIsChanged = ClearSortness(ctx, SortMembers.size()) || sortIsChanged;
             }
         }
     }
@@ -1431,42 +1431,42 @@ void TYqlRowSpecInfo::CopyConstraints(const TYqlRowSpecInfo& from) {
     Distinct = from.Distinct;
 }
 
-bool TYqlRowSpecInfo::KeepPureSortOnly() {
+bool TYqlRowSpecInfo::KeepPureSortOnly(TExprContext& ctx) {
     bool sortIsChanged = false;
     for (size_t i = 0; i < SortMembers.size(); ++i) {
         if (!Type->FindItem(SortMembers[i])) {
-            sortIsChanged = ClearSortness(i);
+            sortIsChanged = ClearSortness(ctx, i);
             break;
         } else if (SortedBy[i] != SortMembers[i]) {
-            sortIsChanged = ClearSortness(i);
+            sortIsChanged = ClearSortness(ctx, i);
             break;
         }
     }
     if (SortMembers.size() < SortedBy.size()) {
-        sortIsChanged = ClearSortness(SortMembers.size()) || sortIsChanged;
+        sortIsChanged = ClearSortness(ctx, SortMembers.size()) || sortIsChanged;
     }
     return sortIsChanged;
 }
 
-bool TYqlRowSpecInfo::ClearNativeDescendingSort() {
+bool TYqlRowSpecInfo::ClearNativeDescendingSort(TExprContext& ctx) {
     for (size_t i = 0; i < SortDirections.size(); ++i) {
         if (!SortDirections[i] && Type->FindItem(SortedBy[i])) {
-            return ClearSortness(i);
+            return ClearSortness(ctx, i);
         }
     }
     return false;
 }
 
-bool TYqlRowSpecInfo::MakeCommonSortness(const TYqlRowSpecInfo& from) {
+bool TYqlRowSpecInfo::MakeCommonSortness(TExprContext& ctx, const TYqlRowSpecInfo& from) {
     bool sortIsChanged = false;
     UniqueKeys = false; // Merge of two and more tables cannot have unique keys
     const size_t resultSize = Min<size_t>(SortMembers.size(), from.SortMembers.size()); // Truncate all calculated columns
     if (SortedBy.size() > resultSize) {
-        sortIsChanged = ClearSortness(resultSize);
+        sortIsChanged = ClearSortness(ctx, resultSize);
     }
     for (size_t i = 0; i < resultSize; ++i) {
         if (SortMembers[i] != from.SortMembers[i] || SortedBy[i] != from.SortedBy[i] || SortedByTypes[i] != from.SortedByTypes[i] || SortDirections[i] != from.SortDirections[i]) {
-            sortIsChanged = ClearSortness(i) || sortIsChanged;
+            sortIsChanged = ClearSortness(ctx, i) || sortIsChanged;
             break;
         }
     }
@@ -1482,13 +1482,15 @@ bool TYqlRowSpecInfo::CompareSortness(const TYqlRowSpecInfo& with, bool checkUni
         && (!checkUniqueFlag || UniqueKeys == with.UniqueKeys);
 }
 
-bool TYqlRowSpecInfo::ClearSortness(size_t fromMember) {
+bool TYqlRowSpecInfo::ClearSortness(TExprContext& ctx, size_t fromMember) {
     if (fromMember <= SortMembers.size()) {
         SortMembers.erase(SortMembers.begin() + fromMember, SortMembers.end());
         SortedBy.erase(SortedBy.begin() + fromMember, SortedBy.end());
         SortedByTypes.erase(SortedByTypes.begin() + fromMember, SortedByTypes.end());
         SortDirections.erase(SortDirections.begin() + fromMember, SortDirections.end());
         UniqueKeys = false;
+        ConstraintsNode.Clear();
+        Sorted = MakeSortConstraint(ctx);
         return true;
     }
     return false;

ydb/library/yql/providers/yt/lib/row_spec/yql_row_spec.h

@@ -60,15 +60,15 @@ struct TYqlRowSpecInfo: public TThrRefBase {
     // Includes aux columns
     const TStructExprType* GetExtendedType(TExprContext& ctx) const;
     // Returns true if sortness is changed
-    bool CopySortness(const TYqlRowSpecInfo& from, ECopySort mode = ECopySort::Pure);
+    bool CopySortness(TExprContext& ctx, const TYqlRowSpecInfo& from, ECopySort mode = ECopySort::Pure);
     // Returns true if sortness is changed
-    bool MakeCommonSortness(const TYqlRowSpecInfo& from);
+    bool MakeCommonSortness(TExprContext& ctx, const TYqlRowSpecInfo& from);
     bool CompareSortness(const TYqlRowSpecInfo& with, bool checkUniqueFlag = true) const;
     // Returns true if sortness is changed
-    bool ClearSortness(size_t fromMember = 0);
+    bool ClearSortness(TExprContext& ctx, size_t fromMember = 0);
     // Returns true if sortness is changed
-    bool KeepPureSortOnly();
-    bool ClearNativeDescendingSort();
+    bool KeepPureSortOnly(TExprContext& ctx);
+    bool ClearNativeDescendingSort(TExprContext& ctx);
     const TSortedConstraintNode* MakeSortConstraint(TExprContext& ctx) const;
     const TDistinctConstraintNode* MakeDistinctConstraint(TExprContext& ctx) const;
     void CopyConstraints(const TYqlRowSpecInfo& from);

ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp

@@ -345,7 +345,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::TakeOrSkip(TExprBase no
             if (!IsOutputUsedMultipleTimes(map.Ref(), *getParents())) {
                 TYtOutTableInfo mapOut(map.Output().Item(0));
                 if (mapOut.RowSpec->IsSorted()) {
-                    mapOut.RowSpec->ClearSortness();
+                    mapOut.RowSpec->ClearSortness(ctx);
                     input = Build<TYtOutput>(ctx, input.Pos())
                         .InitFrom(input.Cast<TYtOutput>())
                         .Operation<TYtMap>()

ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_sort.cpp

@@ -29,7 +29,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::YtSortOverAlreadySorted
     TYqlRowSpecInfo commonSorted = outRowSpec;
     auto section = sort.Input().Item(0);
     for (auto path: section.Paths()) {
-        commonSorted.MakeCommonSortness(*TYtTableBaseInfo::GetRowSpec(path.Table()));
+        commonSorted.MakeCommonSortness(ctx, *TYtTableBaseInfo::GetRowSpec(path.Table()));
     }
 
     if (outRowSpec.CompareSortness(commonSorted)) {
@@ -198,7 +198,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::Sort(TExprBase node, TE
     if (canUseMerge) {
         TYqlRowSpecInfo commonSorted = *sortOut.RowSpec;
         for (auto& pathInfo: inputInfos) {
-            commonSorted.MakeCommonSortness(*pathInfo->Table->RowSpec);
+            commonSorted.MakeCommonSortness(ctx, *pathInfo->Table->RowSpec);
         }
         // input is sorted at least as strictly as output
         if (!sortOut.RowSpec->CompareSortness(commonSorted)) {
@@ -533,8 +533,8 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::AssumeSorted(TExprBase
                 && firstNativeType == path->GetNativeYtType();
         });
         if (canMerge) {
-            outTable.RowSpec->CopySortness(*inputPaths.front()->Table->RowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
-            outTable.RowSpec->ClearSortness(sorted->GetContent().size());
+            outTable.RowSpec->CopySortness(ctx, *inputPaths.front()->Table->RowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
+            outTable.RowSpec->ClearSortness(ctx, sorted->GetContent().size());
             outTable.SetUnique(assume.Ref().GetConstraint<TDistinctConstraintNode>(), assume.Pos(), ctx);
             if (firstNativeType) {
                 outTable.RowSpec->CopyTypeOrders(*firstNativeType);

ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp

@@ -392,7 +392,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::Write(TExprBase node, T
                 }
             } else {
                 if (inputPaths.size() == 1 && inputPaths.front()->Table->RowSpec && inputPaths.front()->Table->RowSpec->IsSorted()) {
-                    outTable.RowSpec->CopySortness(*inputPaths.front()->Table->RowSpec);
+                    outTable.RowSpec->CopySortness(ctx, *inputPaths.front()->Table->RowSpec);
                 }
             }
         }
@@ -406,12 +406,12 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::Write(TExprBase node, T
                 bool hasAux = inputPaths.front()->Table->RowSpec->HasAuxColumns();
                 bool sortIsChanged = inputPaths.front()->Table->IsUnordered
                     ? inputPaths.front()->Table->RowSpec->IsSorted()
-                    : outTable.RowSpec->CopySortness(*inputPaths.front()->Table->RowSpec,
+                    : outTable.RowSpec->CopySortness(ctx, *inputPaths.front()->Table->RowSpec,
                         exactCopySort ? TYqlRowSpecInfo::ECopySort::Exact : TYqlRowSpecInfo::ECopySort::WithDesc);
                 useExplicitColumns = useExplicitColumns || (inputPaths.front()->HasColumns() && hasAux);
 
                 for (size_t i = 1; i < inputPaths.size(); ++i) {
-                    sortIsChanged = outTable.RowSpec->MakeCommonSortness(*inputPaths[i]->Table->RowSpec) || sortIsChanged;
+                    sortIsChanged = outTable.RowSpec->MakeCommonSortness(ctx, *inputPaths[i]->Table->RowSpec) || sortIsChanged;
                     const bool tableHasAux = inputPaths[i]->Table->RowSpec->HasAuxColumns();
                     hasAux = hasAux || tableHasAux;
                     if (inputPaths[i]->HasColumns() && tableHasAux) {

ydb/library/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp

@@ -608,7 +608,7 @@ private:
                 if (initialWrite) {
                     ++nextDescription.WriteValidateCount;
                     if (nextDescription.IsReplaced) {
-                        nextDescription.RowSpec->CopySortness(*contentRowSpecs.front(), TYqlRowSpecInfo::ECopySort::Exact);
+                        nextDescription.RowSpec->CopySortness(ctx, *contentRowSpecs.front(), TYqlRowSpecInfo::ECopySort::Exact);
                         if (auto contentNativeType = contentRowSpecs.front()->GetNativeYtType()) {
                             nextDescription.RowSpec->CopyTypeOrders(*contentNativeType);
                         }
@@ -616,7 +616,7 @@ private:
                     } else {
                         nextDescription.MonotonicKeys = monotonicKeys;
                         if (description.RowSpec) {
-                            nextDescription.RowSpec->CopySortness(*description.RowSpec, TYqlRowSpecInfo::ECopySort::Exact);
+                            nextDescription.RowSpec->CopySortness(ctx, *description.RowSpec, TYqlRowSpecInfo::ECopySort::Exact);
                             const auto currNativeType = description.RowSpec->GetNativeYtType();
                             if (currNativeType && nextDescription.RowSpec->GetNativeYtType() != currNativeType) {
                                 nextDescription.RowSpec->CopyTypeOrders(*currNativeType);
@@ -644,7 +644,7 @@ private:
 
                 const bool uniqueKeys = nextDescription.RowSpec->UniqueKeys;
                 for (size_t s = from; s < contentRowSpecs.size(); ++s) {
-                    const bool hasSortChanges = nextDescription.RowSpec->MakeCommonSortness(*contentRowSpecs[s]);
+                    const bool hasSortChanges = nextDescription.RowSpec->MakeCommonSortness(ctx, *contentRowSpecs[s]);
                     const bool breaksSorting = hasSortChanges || !nextDescription.RowSpec->CompareSortness(*contentRowSpecs[s], false);
                     if (monotonicKeys) {
                         if (breaksSorting) {
@@ -1763,7 +1763,7 @@ private:
             for (auto out: publish.Input()) {
                 contentRowSpecs.push_back(MakeIntrusive<TYqlRowSpecInfo>(GetOutTable(out).Cast<TYtOutTable>().RowSpec()));
                 if (IsUnorderedOutput(out)) {
-                    contentRowSpecs.back()->ClearSortness();
+                    contentRowSpecs.back()->ClearSortness(ctx);
                 }
             }
             TExprNode::TPtr content; // Don't try to convert content

ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp

@@ -974,8 +974,8 @@ IGraphTransformer::TStatus UpdateTableMeta(const TExprNode::TPtr& tableNode, TEx
                     tableInfo.RowSpec->CopyTypeOrders(*nativeType);
                 }
                 if (prevRowSpec->IsSorted()) {
-                    tableInfo.RowSpec->CopySortness(*prevRowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
-                    tableInfo.RowSpec->MakeCommonSortness(*prevRowSpec); // Truncated keys with changed types
+                    tableInfo.RowSpec->CopySortness(ctx, *prevRowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
+                    tableInfo.RowSpec->MakeCommonSortness(ctx, *prevRowSpec); // Truncated keys with changed types
                 }
             }
         }
@@ -1419,7 +1419,7 @@ TYtPath CopyOrTrivialMap(TPositionHandle pos, TExprBase world, TYtDSink dataSink
         bool sortIsChanged = false;
         for (size_t i = 0; i < rowSpecs.size(); ++i) {
             if (!rowSpecs[i].first) {
-                sortIsChanged = outTable.RowSpec->ClearSortness();
+                sortIsChanged = outTable.RowSpec->ClearSortness(ctx);
                 continue;
             }
             if (0 == i) {
@@ -1433,11 +1433,11 @@ TYtPath CopyOrTrivialMap(TPositionHandle pos, TExprBase world, TYtDSink dataSink
                         ? TYqlRowSpecInfo::ECopySort::Exact
                         : TYqlRowSpecInfo::ECopySort::WithDesc;
                 }
-                sortIsChanged = outTable.RowSpec->CopySortness(*rowSpecs[i].first, mode);
+                sortIsChanged = outTable.RowSpec->CopySortness(ctx, *rowSpecs[i].first, mode);
             } else {
-                sortIsChanged = outTable.RowSpec->MakeCommonSortness(*rowSpecs[i].first) || sortIsChanged;
+                sortIsChanged = outTable.RowSpec->MakeCommonSortness(ctx, *rowSpecs[i].first) || sortIsChanged;
                 if (rowSpecs[i].second && !sortConstraintEnabled) {
-                    sortIsChanged = outTable.RowSpec->KeepPureSortOnly() || sortIsChanged;
+                    sortIsChanged = outTable.RowSpec->KeepPureSortOnly(ctx) || sortIsChanged;
                 }
             }
         }
@@ -1507,7 +1507,7 @@ TYtPath CopyOrTrivialMap(TPositionHandle pos, TExprBase world, TYtDSink dataSink
                         .Body("stream")
                         .Done().Ptr();
 
-                    mapOutTable.RowSpec->CopySortness(*rowSpecs[i].first, sortConstraintEnabled ? TYqlRowSpecInfo::ECopySort::WithDesc : TYqlRowSpecInfo::ECopySort::Pure);
+                    mapOutTable.RowSpec->CopySortness(ctx, *rowSpecs[i].first, sortConstraintEnabled ? TYqlRowSpecInfo::ECopySort::WithDesc : TYqlRowSpecInfo::ECopySort::Pure);
                     if (sortConstraintEnabled) {
                         TKeySelectorBuilder builder(path.Pos(), ctx, useNativeDescSort, scheme.Cast<TStructExprType>());
                         builder.ProcessRowSpec(*mapOutTable.RowSpec);

ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp

@@ -192,7 +192,7 @@ public:
                     if (!rowSpec->Parse(tableDesc.Meta->Attrs, ctx)) {
                         return TStatus::Error;
                     }
-                    if (!State_->Configuration->UseNativeDescSort.Get().GetOrElse(false) && rowSpec->ClearNativeDescendingSort()) {
+                    if (!State_->Configuration->UseNativeDescSort.Get().GetOrElse(false) && rowSpec->ClearNativeDescendingSort(ctx)) {
                         if (!ctx.AddWarning(YqlIssue(TPosition(), EYqlIssueCode::TIssuesIds_EIssueCode_YT_NATIVE_DESC_SORT_IGNORED, "Native descending sort is ignored"))) {
                             return TStatus::Error;
                         }

ydb/library/yql/providers/yt/provider/yql_yt_logical_optimize.cpp

@@ -227,7 +227,7 @@ protected:
                             }
                         }
                     } else {
-                        mapOut.RowSpec->CopySortness(TYqlRowSpecInfo(map.Output().Item(0).RowSpec()));
+                        mapOut.RowSpec->CopySortness(ctx, TYqlRowSpecInfo(map.Output().Item(0).RowSpec()));
                     }
                     mapOut.SetUnique(path.Ref().GetConstraint<TDistinctConstraintNode>(), map.Mapper().Pos(), ctx);
 
@@ -266,7 +266,7 @@ protected:
                     }
 
                     TYtOutTableInfo mergeOut(ctx.MakeType<TStructExprType>(structItems), prevRowSpec.GetNativeYtTypeFlags());
-                    mergeOut.RowSpec->CopySortness(prevRowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
+                    mergeOut.RowSpec->CopySortness(ctx, prevRowSpec, TYqlRowSpecInfo::ECopySort::WithDesc);
                     if (auto nativeType = prevRowSpec.GetNativeYtType()) {
                         mergeOut.RowSpec->CopyTypeOrders(*nativeType);
                     }

ydb/library/yql/providers/yt/provider/yql_yt_optimize.cpp

@@ -370,7 +370,7 @@ TYtSection MakeEmptySection(TYtSection section, NNodes::TYtDSink dataSink, bool
     if (section.Paths().Size() == 1) {
         auto srcTableInfo = TYtTableBaseInfo::Parse(section.Paths().Item(0).Table());
         if (keepSortness && srcTableInfo->RowSpec && srcTableInfo->RowSpec->IsSorted()) {
-            outTable.RowSpec->CopySortness(*srcTableInfo->RowSpec, TYqlRowSpecInfo::ECopySort::WithCalc);
+            outTable.RowSpec->CopySortness(ctx, *srcTableInfo->RowSpec, TYqlRowSpecInfo::ECopySort::WithCalc);
         }
     }
     outTable.SetUnique(section.Ref().GetConstraint<TDistinctConstraintNode>(), section.Pos(), ctx);