#include #include #include "mkql_validate.h" #include #include namespace NKikimr { namespace NMiniKQL { namespace { static const TString VERIFY_DELIMITER = "\n - "; using namespace NUdf; template struct TLazyVerifyListValue; template struct TLazyVerifyListIterator: public TBoxedValue { TLazyVerifyListIterator(const TLazyVerifyListValue& lazyList, ui64 index = 0) : LazyList(lazyList) , OrigIter(TBoxedValueAccessor::GetListIterator(*lazyList.Orig)) , Index(index) { if (!OrigIter) { TValidateErrorPolicy::Generate(TStringBuilder() << "TLazyVerifyListIterator constructor expect non empty result of GetListIterator" << VERIFY_DELIMITER << LazyList.Message); } } private: const TLazyVerifyListValue& LazyList; TUnboxedValue OrigIter; ui64 Index; bool Next(NUdf::TUnboxedValue& item) final { ++Index; if (!OrigIter.Next(item)) return false; TType* itemType = LazyList.ListType->GetItemType(); item = TValidate>::Value(LazyList.ValueBuilder, itemType, std::move(item), TStringBuilder() << "LazyList[" << Index << "]" << VERIFY_DELIMITER << LazyList.Message); return true; } bool Skip() final { ++Index; return OrigIter.Skip(); } }; template struct TLazyVerifyDictValue; template struct TLazyVerifyListValue: public TBoxedValue { TLazyVerifyListValue(const IValueBuilder* valueBuilder, const TListType* listType, IBoxedValuePtr&& orig, const TString& message) : ValueBuilder(valueBuilder) , ListType(listType) , Orig(std::move(orig)) , Message(message) {} const IValueBuilder *const ValueBuilder; const TListType *const ListType; const IBoxedValuePtr Orig; const TString Message; private: bool HasFastListLength() const override { return TBoxedValueAccessor::HasFastListLength(*Orig); } ui64 GetListLength() const override { return TBoxedValueAccessor::GetListLength(*Orig); } ui64 GetEstimatedListLength() const override { return TBoxedValueAccessor::GetEstimatedListLength(*Orig); } TUnboxedValue GetListIterator() const override { return TUnboxedValuePod(new TLazyVerifyListIterator(*this)); } const TOpaqueListRepresentation* GetListRepresentation() const override { return nullptr; } IBoxedValuePtr ReverseListImpl(const IValueBuilder& builder) const override { auto upList = TBoxedValueAccessor::ReverseListImpl(*Orig, builder); if (upList) { return new TLazyVerifyListValue(ValueBuilder, ListType, std::move(upList), Message); } return upList; } IBoxedValuePtr SkipListImpl(const IValueBuilder& builder, ui64 count) const override { auto upList = TBoxedValueAccessor::SkipListImpl(*Orig, builder, count); if (upList) { return new TLazyVerifyListValue(ValueBuilder, ListType, std::move(upList), Message); } return upList; } IBoxedValuePtr TakeListImpl(const IValueBuilder& builder, ui64 count) const override { auto upList = TBoxedValueAccessor::TakeListImpl(*Orig, builder, count); if (upList) { return new TLazyVerifyListValue(ValueBuilder, ListType, std::move(upList), Message); } return upList; } IBoxedValuePtr ToIndexDictImpl(const IValueBuilder& builder) const override { auto dictImpl = TBoxedValueAccessor::ToIndexDictImpl(*Orig, builder); if (!dictImpl) { return dictImpl; } return IBoxedValuePtr(new TLazyVerifyDictValue(&builder, ListType->IndexDictKeyType(), ListType->GetItemType(), std::move(dictImpl), TStringBuilder() << "LazyDict over IndexToDict" << VERIFY_DELIMITER << Message)); } bool HasListItems() const override { return TBoxedValueAccessor::HasListItems(*Orig); } bool HasDictItems() const override { const bool result = TBoxedValueAccessor::HasDictItems(*Orig); if (result) { TValidateErrorPolicy::Generate(TStringBuilder() << "TLazyVerifyListValue::HasDictItems expect false, but got true" << VERIFY_DELIMITER << Message); } return result; } }; template struct TLazyVerifyDictIterator: public TBoxedValue { TLazyVerifyDictIterator(const TLazyVerifyDictValue& lazyDict, ui64 index = 0) : LazyDict(lazyDict) , OrigIter((Keys ? &TBoxedValueAccessor::GetKeysIterator : &TBoxedValueAccessor::GetDictIterator)(*LazyDict.Orig)) , Index(index) { if (!OrigIter) { TValidateErrorPolicy::Generate(TStringBuilder() << "TLazyVerifyDictIterator constructor expect non empty result of GetDictIterator" << VERIFY_DELIMITER << LazyDict.Message); } } private: const TLazyVerifyDictValue& LazyDict; TUnboxedValue OrigIter; ui64 Index; bool Skip() final { ++Index; return OrigIter.Skip(); } bool Next(NUdf::TUnboxedValue& key) final { ++Index; if (!OrigIter.Next(key)) return false; key = TValidate>::Value(LazyDict.ValueBuilder, LazyDict.KeyType, std::move(key), TStringBuilder() << "LazyDict[" << Index << "], validate key" << VERIFY_DELIMITER << LazyDict.Message); return true; } bool NextPair(NUdf::TUnboxedValue& key, NUdf::TUnboxedValue& payload) final { ++Index; if (!OrigIter.NextPair(key, payload)) return false; key = TValidate>::Value(LazyDict.ValueBuilder, LazyDict.KeyType, std::move(key), TStringBuilder() << "LazyDict[" << Index << "], validate key" << VERIFY_DELIMITER << LazyDict.Message); payload = TValidate>::Value(LazyDict.ValueBuilder, LazyDict.PayloadType, std::move(payload), TStringBuilder() << "LazyDict[" << Index << "], validate payload" << VERIFY_DELIMITER << LazyDict.Message); return true; } }; template struct TLazyVerifyDictValue: public TBoxedValue { TLazyVerifyDictValue(const IValueBuilder* valueBuilder, const TDictType* dictType, IBoxedValuePtr&& orig, const TString& message) : TLazyVerifyDictValue(valueBuilder, dictType->GetKeyType(), dictType->GetPayloadType(), std::move(orig), message) {} TLazyVerifyDictValue(const IValueBuilder* valueBuilder, const TType* keyType, const TType* payloadType, IBoxedValuePtr&& orig, const TString& message) : ValueBuilder(valueBuilder) , KeyType(keyType) , PayloadType(payloadType) , Orig(std::move(orig)) , Message(message) {} const IValueBuilder *const ValueBuilder; const TType *const KeyType; const TType *const PayloadType; const IBoxedValuePtr Orig; const TString Message; private: ui64 GetDictLength() const override { return TBoxedValueAccessor::GetDictLength(*Orig); } TUnboxedValue GetKeysIterator() const override { return TUnboxedValuePod(new TLazyVerifyDictIterator(*this)); } TUnboxedValue GetDictIterator() const override { return TUnboxedValuePod(new TLazyVerifyDictIterator(*this)); } bool Contains(const TUnboxedValuePod& key) const override { return TBoxedValueAccessor::Contains(*Orig, key); } TUnboxedValue Lookup(const TUnboxedValuePod& key) const override { if (auto lookup = TBoxedValueAccessor::Lookup(*Orig, key)) { return TValidate>::Value(ValueBuilder, PayloadType, lookup.Release().GetOptionalValue(), TStringBuilder() << "LazyDict, validate Lookup payload" << VERIFY_DELIMITER << Message).Release().MakeOptional(); } return TUnboxedValuePod(); } bool HasListItems() const override { const bool result = TBoxedValueAccessor::HasListItems(*Orig); if (result) { TValidateErrorPolicy::Generate(TStringBuilder() << "TLazyVerifyDictValue::HasListItems expect false, but got true" << VERIFY_DELIMITER << Message); } return result; } bool HasDictItems() const override { return TBoxedValueAccessor::HasDictItems(*Orig); } }; template class WrapCallableValue: public TBoxedValue { public: WrapCallableValue(const TCallableType* callableType, TUnboxedValue&& callable, const TString& message); private: const TCallableType *const CallableType; const TUnboxedValue Callable; const TString Message; TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const final; }; template WrapCallableValue::WrapCallableValue(const TCallableType* callableType, TUnboxedValue&& callable, const TString& message) : CallableType(callableType) , Callable(std::move(callable)) , Message(message) { } template TUnboxedValue WrapCallableValue::Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { const ui32 argsCount = CallableType->GetArgumentsCount(); TSmallVec wrapArgs(argsCount); bool childWrapped = false; for (ui32 indexArg = 0; indexArg < argsCount; ++indexArg) { const auto argType = CallableType->GetArgumentType(indexArg); wrapArgs[indexArg] = TValidate::Value(valueBuilder, argType, TUnboxedValuePod(args[indexArg]), TStringBuilder() << "CallableWrapper<" << CallableType->GetName() << ">.arg[" << indexArg << "]" << VERIFY_DELIMITER << Message, &childWrapped); } return TValidate::Value(valueBuilder, CallableType->GetReturnType(), Callable.Run(valueBuilder, childWrapped ? wrapArgs.data() : args), TStringBuilder() << "CallableWrapper<" << CallableType->GetName() << ">.result" << VERIFY_DELIMITER << Message); } } // anonymous namespace template struct TValidateModeLazy { static NUdf::TUnboxedValue ProcessList(const IValueBuilder* valueBuilder, const TListType* listType, IBoxedValuePtr&& boxed, const TString& message, bool* wrapped) { if (wrapped) { *wrapped = true; } return NUdf::TUnboxedValuePod(new TLazyVerifyListValue(valueBuilder, listType, std::move(boxed), message)); } static NUdf::TUnboxedValue ProcessDict(const IValueBuilder* valueBuilder, const TDictType* dictType, IBoxedValuePtr&& boxed, const TString& message, bool* wrapped) { if (wrapped) { *wrapped = true; } return NUdf::TUnboxedValuePod(new TLazyVerifyDictValue(valueBuilder, dictType, std::move(boxed), message)); } }; template struct TValidateModeGreedy { static NUdf::TUnboxedValue ProcessList(const IValueBuilder* valueBuilder, const TListType* listType, IBoxedValuePtr&& boxed, const TString& message, bool* wrapped) { if (!TBoxedValueAccessor::HasFastListLength(*boxed)) { return NUdf::TUnboxedValuePod(new TLazyVerifyListValue(valueBuilder, listType, std::move(boxed), message)); } const TType* itemType = listType->GetItemType(); std::vector list; if (TBoxedValueAccessor::HasFastListLength(*boxed)) list.reserve(TBoxedValueAccessor::GetListLength(*boxed)); bool childWrapped = false; ui64 curIndex = 0; const auto iter = TBoxedValueAccessor::GetListIterator(*boxed); for (NUdf::TUnboxedValue current; iter.Next(current); ++curIndex) { list.emplace_back(TValidate::Value(valueBuilder, itemType, std::move(current), TStringBuilder() << "LazyList[" << curIndex << "]" << VERIFY_DELIMITER << message, &childWrapped)); } const auto elementsCount = TBoxedValueAccessor::GetListLength(*boxed); if (curIndex != elementsCount) { TValidateErrorPolicy::Generate(TStringBuilder() << "TValidateModeGreedy::ProcessList, wrong list length returned, expected: " << curIndex << ", but got: " << elementsCount << VERIFY_DELIMITER << message); } if (childWrapped) { if (wrapped) { *wrapped = true; } return valueBuilder->NewList(list.data(), list.size()); } return NUdf::TUnboxedValuePod(std::move(boxed)); } static TUnboxedValue ProcessDict(const IValueBuilder* valueBuilder, const TDictType* dictType, IBoxedValuePtr&& boxed, const TString& message, bool* wrapped) { auto keyType = dictType->GetKeyType(); auto payloadType = dictType->GetPayloadType(); auto dictBuilder = valueBuilder->NewDict(dictType, TDictFlags::Sorted); bool childWrapped = false; ui64 curIndex = 0; const auto iter = TBoxedValueAccessor::GetDictIterator(*boxed); for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload); ++curIndex) { key = TValidate>::Value(valueBuilder, keyType, std::move(key), TStringBuilder() << "GreedyDict[" << curIndex << "], validate key" << VERIFY_DELIMITER << message); payload = TValidate>::Value(valueBuilder, payloadType, std::move(payload), TStringBuilder() << "GreedyDict[" << curIndex << "], validate payload" << VERIFY_DELIMITER << message); dictBuilder->Add(std::move(key), std::move(payload)); } const auto elementsCount = TBoxedValueAccessor::GetDictLength(*boxed); if (curIndex != elementsCount) { TValidateErrorPolicy::Generate(TStringBuilder() << "TValidateModeGreedy::ProcessDict, wrong dict length returned, expected: " << curIndex << ", but got: " << elementsCount << VERIFY_DELIMITER << message); } if (childWrapped) { if (wrapped) { *wrapped = true; } return dictBuilder->Build(); } return NUdf::TUnboxedValuePod(std::move(boxed)); } }; template NUdf::TUnboxedValue TValidate::Value(const IValueBuilder* valueBuilder, const TType* type, NUdf::TUnboxedValue&& value, const TString& message, bool* wrapped) { if (!value && !(type->IsOptional() || type->IsNull())) { TValidateErrorPolicy::Generate(TStringBuilder() << "Expected value '" << PrintNode(type, true) << "', but got Empty" << VERIFY_DELIMITER << message); } switch (type->GetKind()) { case TType::EKind::Void: case TType::EKind::Null: case TType::EKind::EmptyList: case TType::EKind::EmptyDict: break; case TType::EKind::Data: { auto dataType = static_cast(type); auto dataTypeId = dataType->GetSchemeType(); auto slot = NUdf::FindDataSlot(dataTypeId); if (!slot) { TValidateErrorPolicy::GenerateExc(TUdfValidateException() << "Unregistered TypeId: " << dataTypeId << VERIFY_DELIMITER << message); } if (!IsValidValue(*slot, value)) { TValidateErrorPolicy::Generate(TStringBuilder() << "Expected value '" << PrintNode(type, true) << "' does not conform" << VERIFY_DELIMITER << message); } break; } case TType::EKind::Optional: { if (!value) { break; } auto optionalType = static_cast(type); bool childWrapped = false; auto upValue = TValidate::Value(valueBuilder, optionalType->GetItemType(), value.GetOptionalValue(), TStringBuilder() << "Optional" << message, &childWrapped); if (childWrapped) { if (wrapped) { *wrapped = true; } return upValue.Release().MakeOptional(); } break; } case TType::EKind::List: { if (!value.IsBoxed()) { TValidateErrorPolicy::Generate(TStringBuilder() << "expected value '" << PrintNode(type, true) << "' not conform" << VERIFY_DELIMITER << message); } auto listType = static_cast(type); return TValidateMode::ProcessList(valueBuilder, listType, std::move(value.Release().AsBoxed()), message, wrapped); } case TType::EKind::Struct: { if (!value.IsBoxed()) { TValidateErrorPolicy::Generate(TStringBuilder() << "expected value '" << PrintNode(type, true) << "' not conform" << VERIFY_DELIMITER << message); } auto structType = static_cast(type); bool childWrapped = false; TSmallVec stackItems(structType->GetMembersCount()); for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { TType* memberType = structType->GetMemberType(index); stackItems[index] = TValidate::Value(valueBuilder, memberType, value.GetElement(index), TStringBuilder() << "Struct[" << structType->GetMemberName(index) << "]" << VERIFY_DELIMITER << message, &childWrapped); } if (childWrapped) { TUnboxedValue* items = nullptr; const auto wrappedStruct = valueBuilder->NewArray(structType->GetMembersCount(), items); for (ui32 index = 0; index < structType->GetMembersCount(); ++index) { items[index] = std::move(stackItems[index]); } if (wrapped) { *wrapped = true; } return wrappedStruct; } break; } case TType::EKind::Tuple: { if (!value.IsBoxed()) { TValidateErrorPolicy::Generate(TStringBuilder() << "expected value '" << PrintNode(type, true) << "' not conform" << VERIFY_DELIMITER << message); } auto tupleType = static_cast(type); bool childWrapped = false; TSmallVec stackItems(tupleType->GetElementsCount()); for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { TType* elementType = tupleType->GetElementType(index); stackItems[index] = TValidate::Value(valueBuilder, elementType, value.GetElement(index), TStringBuilder() << "Tuple[" << index << "]" << VERIFY_DELIMITER << message, &childWrapped); } if (childWrapped) { TUnboxedValue* items = nullptr; const auto wrappedTuple = valueBuilder->NewArray(tupleType->GetElementsCount(), items); for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) { items[index] = std::move(stackItems[index]); } if (wrapped) { *wrapped = true; } return wrappedTuple; } break; } case TType::EKind::Dict: { if (!value.IsBoxed()) { TValidateErrorPolicy::Generate(TStringBuilder() << "expected value '" << PrintNode(type, true) << "' not conform" << VERIFY_DELIMITER << message); } auto dictType = static_cast(type); return TValidateMode::ProcessDict(valueBuilder, dictType, std::move(value.Release().AsBoxed()), message, wrapped); } case TType::EKind::Callable: { if (!value.IsBoxed()) { TValidateErrorPolicy::Generate(TStringBuilder() << "expected value '" << PrintNode(type, true) << "' not conform" << VERIFY_DELIMITER << message); } auto callableType = static_cast(type); if (wrapped) { *wrapped = true; } TValidate::WrapCallable(callableType, value, message); return value; } case TType::EKind::Type: // metatype allways valid if we meet it break; case TType::EKind::Resource: // resource allways valid if we meet it break; case TType::EKind::Stream: case TType::EKind::Variant: case TType::EKind::Block: // TODO validate it break; case TType::EKind::Tagged: { auto taggedType = static_cast(type); bool childWrapped = false; auto upValue = TValidate::Value(valueBuilder, taggedType->GetBaseType(), TUnboxedValue(value), TStringBuilder() << "Tagged[" << taggedType->GetTag() << "]" << message, &childWrapped); if (childWrapped) { if (wrapped) { *wrapped = true; } return upValue; } break; } default: Y_ABORT("Verify value meet unexpected type kind: %s", type->GetKindAsStr().data()); } return std::move(value); } template void TValidate::WrapCallable(const TCallableType* callableType, NUdf::TUnboxedValue& callable, const TString& message) { callable = NUdf::TUnboxedValuePod(new WrapCallableValue(callableType, std::move(callable), TStringBuilder() << "CallableWrapper<" << callableType->GetName() << ">" << VERIFY_DELIMITER << message)); } template struct TValidate>; template struct TValidate>; template struct TValidate>; template struct TValidate>; } // namespace MiniKQL } // namespace NKikimr