mkql_computation_node_impl.cpp 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. #include "mkql_computation_node_impl.h"
  2. #include "yql/essentials/minikql/mkql_string_util.h"
  3. namespace NKikimr {
  4. namespace NMiniKQL {
  5. void ThrowNotSupportedImplForClass(const TString& className, const char *func) {
  6. THROW yexception() << "Unsupported access to '" << func << "' method of: " << className;
  7. }
  8. template <class IComputationNodeInterface>
  9. void TRefCountedComputationNode<IComputationNodeInterface>::Ref() {
  10. ++Refs_;
  11. }
  12. template <class IComputationNodeInterface>
  13. void TRefCountedComputationNode<IComputationNodeInterface>::UnRef() {
  14. Y_ABORT_UNLESS(Refs_ > 0);
  15. if (--Refs_ == 0) {
  16. delete this;
  17. }
  18. }
  19. template <class IComputationNodeInterface>
  20. ui32 TRefCountedComputationNode<IComputationNodeInterface>::RefCount() const {
  21. return Refs_;
  22. }
  23. template class TRefCountedComputationNode<IComputationWideFlowNode>;
  24. template class TRefCountedComputationNode<IComputationWideFlowProxyNode>;
  25. TUnboxedImmutableComputationNode::TUnboxedImmutableComputationNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value)
  26. : MemInfo(memInfo)
  27. , UnboxedValue(std::move(value))
  28. , RepresentationKind(UnboxedValue.HasValue() ? (UnboxedValue.IsBoxed() ? EValueRepresentation::Boxed : (UnboxedValue.IsString() ? EValueRepresentation::String : EValueRepresentation::Embedded)) : EValueRepresentation::Embedded)
  29. {
  30. MKQL_MEM_TAKE(MemInfo, this, sizeof(*this), __MKQL_LOCATION__);
  31. TlsAllocState->LockObject(UnboxedValue);
  32. }
  33. TUnboxedImmutableComputationNode::~TUnboxedImmutableComputationNode() {
  34. MKQL_MEM_RETURN(MemInfo, this, sizeof(*this));
  35. TlsAllocState->UnlockObject(UnboxedValue);
  36. }
  37. NUdf::TUnboxedValue TUnboxedImmutableComputationNode::GetValue(TComputationContext& compCtx) const {
  38. Y_UNUSED(compCtx);
  39. if (!TlsAllocState->UseRefLocking && RepresentationKind == EValueRepresentation::String) {
  40. /// TODO: YQL-4461
  41. return MakeString(UnboxedValue.AsStringRef());
  42. }
  43. return UnboxedValue;
  44. }
  45. const IComputationNode* TUnboxedImmutableComputationNode::GetSource() const { return nullptr; }
  46. IComputationNode* TUnboxedImmutableComputationNode::AddDependence(const IComputationNode*) { return nullptr; }
  47. void TUnboxedImmutableComputationNode::RegisterDependencies() const {}
  48. ui32 TUnboxedImmutableComputationNode::GetIndex() const {
  49. THROW yexception() << "Failed to get index.";
  50. }
  51. void TUnboxedImmutableComputationNode::CollectDependentIndexes(const IComputationNode*, TIndexesMap&) const {
  52. THROW yexception() << "Failed to collect dependent indexes.";
  53. }
  54. ui32 TUnboxedImmutableComputationNode::GetDependencyWeight() const {
  55. THROW yexception() << "Can't get dependency weight from const node.";
  56. }
  57. ui32 TUnboxedImmutableComputationNode::GetDependencesCount() const {
  58. THROW yexception() << "Can't get dependences count from const node.";
  59. }
  60. bool TUnboxedImmutableComputationNode::IsTemporaryValue() const { return false; }
  61. void TUnboxedImmutableComputationNode::PrepareStageOne() {}
  62. void TUnboxedImmutableComputationNode::PrepareStageTwo() {}
  63. TString TUnboxedImmutableComputationNode::DebugString() const {
  64. return UnboxedValue ? (UnboxedValue.IsBoxed() ? "Boxed" : "Literal") : "Empty";
  65. }
  66. EValueRepresentation TUnboxedImmutableComputationNode::GetRepresentation() const {
  67. return RepresentationKind;
  68. }
  69. Y_NO_INLINE TStatefulComputationNodeBase::TStatefulComputationNodeBase(ui32 valueIndex, EValueRepresentation kind)
  70. : ValueIndex(valueIndex)
  71. , RepresentationKind(kind)
  72. {}
  73. Y_NO_INLINE TStatefulComputationNodeBase::~TStatefulComputationNodeBase()
  74. {}
  75. Y_NO_INLINE void TStatefulComputationNodeBase::AddDependenceImpl(const IComputationNode* node) {
  76. Dependencies.emplace_back(node);
  77. }
  78. Y_NO_INLINE void TStatefulComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self, const IComputationNode* owner,
  79. IComputationNode::TIndexesMap& dependencies, bool stateless) const {
  80. if (self == owner)
  81. return;
  82. if (const auto ins = dependencies.emplace(ValueIndex, RepresentationKind); ins.second) {
  83. std::for_each(Dependencies.cbegin(), Dependencies.cend(), std::bind(&IComputationNode::CollectDependentIndexes, std::placeholders::_1, owner, std::ref(dependencies)));
  84. if (stateless) {
  85. dependencies.erase(ins.first);
  86. }
  87. }
  88. }
  89. Y_NO_INLINE TStatefulSourceComputationNodeBase::TStatefulSourceComputationNodeBase()
  90. {}
  91. Y_NO_INLINE TStatefulSourceComputationNodeBase::~TStatefulSourceComputationNodeBase()
  92. {}
  93. Y_NO_INLINE void TStatefulSourceComputationNodeBase::PrepareStageOneImpl(const TConstComputationNodePtrVector& dependencies) {
  94. if (!Stateless) {
  95. Stateless = std::accumulate(dependencies.cbegin(), dependencies.cend(), 0,
  96. std::bind(std::plus<i32>(), std::placeholders::_1, std::bind(&IComputationNode::GetDependencyWeight, std::placeholders::_2))) <= 1;
  97. }
  98. }
  99. Y_NO_INLINE void TStatefulSourceComputationNodeBase::AddSource(IComputationNode* source) const {
  100. Sources.emplace(source);
  101. }
  102. template <class IComputationNodeInterface, bool SerializableState>
  103. TStatefulComputationNode<IComputationNodeInterface, SerializableState>::TStatefulComputationNode(TComputationMutables& mutables, EValueRepresentation kind)
  104. : TStatefulComputationNodeBase(mutables.CurValueIndex++, kind)
  105. {
  106. if constexpr (SerializableState) {
  107. mutables.SerializableValues.push_back(ValueIndex);
  108. }
  109. }
  110. template <class IComputationNodeInterface, bool SerializableState>
  111. IComputationNode* TStatefulComputationNode<IComputationNodeInterface, SerializableState>::AddDependence(const IComputationNode* node) {
  112. AddDependenceImpl(node);
  113. return this;
  114. }
  115. template <class IComputationNodeInterface, bool SerializableState>
  116. EValueRepresentation TStatefulComputationNode<IComputationNodeInterface, SerializableState>::GetRepresentation() const {
  117. return RepresentationKind;
  118. }
  119. template <class IComputationNodeInterface, bool SerializableState>
  120. void TStatefulComputationNode<IComputationNodeInterface, SerializableState>::InitNode(TComputationContext&) const {}
  121. template <class IComputationNodeInterface, bool SerializableState>
  122. ui32 TStatefulComputationNode<IComputationNodeInterface, SerializableState>::GetIndex() const { return ValueIndex; }
  123. template <class IComputationNodeInterface, bool SerializableState>
  124. ui32 TStatefulComputationNode<IComputationNodeInterface, SerializableState>::GetDependencesCount() const { return Dependencies.size(); }
  125. template class TStatefulComputationNode<IComputationNode, false>;
  126. template class TStatefulComputationNode<IComputationWideFlowNode, false>;
  127. template class TStatefulComputationNode<IComputationExternalNode, false>;
  128. template class TStatefulComputationNode<IComputationNode, true>;
  129. template class TStatefulComputationNode<IComputationWideFlowNode, true>;
  130. template class TStatefulComputationNode<IComputationExternalNode, true>;
  131. Y_NO_INLINE ui32 TStatelessFlowComputationNodeBase::GetIndexImpl() const {
  132. THROW yexception() << "Failed to get stateless node index.";
  133. }
  134. Y_NO_INLINE void TStatelessFlowComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self,
  135. const IComputationNode* owner, IComputationNode::TIndexesMap& dependencies,
  136. const IComputationNode* dependence) const {
  137. if (self == owner)
  138. return;
  139. if (dependence) {
  140. dependence->CollectDependentIndexes(owner, dependencies);
  141. }
  142. }
  143. Y_NO_INLINE TStatefulFlowComputationNodeBase::TStatefulFlowComputationNodeBase(ui32 stateIndex, EValueRepresentation stateKind)
  144. : StateIndex(stateIndex)
  145. , StateKind(stateKind)
  146. {}
  147. Y_NO_INLINE void TStatefulFlowComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self, const IComputationNode* owner,
  148. IComputationNode::TIndexesMap& dependencies, const IComputationNode* dependence) const {
  149. if (self == owner)
  150. return;
  151. const auto ins = dependencies.emplace(StateIndex, StateKind);
  152. if (ins.second && dependence) {
  153. dependence->CollectDependentIndexes(owner, dependencies);
  154. }
  155. }
  156. Y_NO_INLINE TPairStateFlowComputationNodeBase::TPairStateFlowComputationNodeBase(ui32 stateIndex, EValueRepresentation firstKind, EValueRepresentation secondKind)
  157. : StateIndex(stateIndex)
  158. , FirstKind(firstKind)
  159. , SecondKind(secondKind)
  160. {}
  161. Y_NO_INLINE void TPairStateFlowComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self, const IComputationNode* owner,
  162. IComputationNode::TIndexesMap& dependencies, const IComputationNode* dependence) const {
  163. if (self == owner)
  164. return;
  165. const auto ins1 = dependencies.emplace(StateIndex, FirstKind);
  166. const auto ins2 = dependencies.emplace(StateIndex + 1U, SecondKind);
  167. if (ins1.second && ins2.second && dependence) {
  168. dependence->CollectDependentIndexes(owner, dependencies);
  169. }
  170. }
  171. Y_NO_INLINE ui32 TStatelessWideFlowComputationNodeBase::GetIndexImpl() const {
  172. THROW yexception() << "Failed to get stateless node index.";
  173. }
  174. Y_NO_INLINE void TStatelessWideFlowComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self, const IComputationNode* owner,
  175. IComputationNode::TIndexesMap& dependencies, const IComputationNode* dependence) const {
  176. if (self == owner)
  177. return;
  178. if (dependence) {
  179. dependence->CollectDependentIndexes(owner, dependencies);
  180. }
  181. }
  182. Y_NO_INLINE EValueRepresentation TWideFlowBaseComputationNodeBase::GetRepresentationImpl() const {
  183. THROW yexception() << "Failed to get representation kind.";
  184. }
  185. Y_NO_INLINE NUdf::TUnboxedValue TWideFlowBaseComputationNodeBase::GetValueImpl(TComputationContext&) const {
  186. THROW yexception() << "Failed to get value from wide flow node.";
  187. }
  188. Y_NO_INLINE TStatefulWideFlowComputationNodeBase::TStatefulWideFlowComputationNodeBase(ui32 stateIndex, EValueRepresentation stateKind)
  189. : StateIndex(stateIndex)
  190. , StateKind(stateKind)
  191. {}
  192. Y_NO_INLINE void TStatefulWideFlowComputationNodeBase::CollectDependentIndexesImpl(const IComputationNode* self,
  193. const IComputationNode* owner, IComputationNode::TIndexesMap& dependencies, const IComputationNode* dependence) const {
  194. if (self == owner)
  195. return;
  196. const auto ins = dependencies.emplace(StateIndex, StateKind);
  197. if (ins.second && dependence) {
  198. dependence->CollectDependentIndexes(owner, dependencies);
  199. }
  200. }
  201. Y_NO_INLINE TPairStateWideFlowComputationNodeBase::TPairStateWideFlowComputationNodeBase(
  202. ui32 stateIndex, EValueRepresentation firstKind, EValueRepresentation secondKind)
  203. : StateIndex(stateIndex)
  204. , FirstKind(firstKind)
  205. , SecondKind(secondKind)
  206. {}
  207. Y_NO_INLINE void TPairStateWideFlowComputationNodeBase::CollectDependentIndexesImpl(
  208. const IComputationNode* self, const IComputationNode* owner,
  209. IComputationNode::TIndexesMap& dependencies, const IComputationNode* dependence) const {
  210. if (self == owner)
  211. return;
  212. const auto ins1 = dependencies.emplace(StateIndex, FirstKind);
  213. const auto ins2 = dependencies.emplace(StateIndex + 1U, SecondKind);
  214. if (ins1.second && ins2.second && dependence) {
  215. dependence->CollectDependentIndexes(owner, dependencies);
  216. }
  217. }
  218. Y_NO_INLINE TDecoratorComputationNodeBase::TDecoratorComputationNodeBase(IComputationNode* node, EValueRepresentation kind)
  219. : Node(node)
  220. , Kind(kind)
  221. {}
  222. Y_NO_INLINE ui32 TDecoratorComputationNodeBase::GetIndexImpl() const {
  223. THROW yexception() << "Can't get index from decorator node.";
  224. }
  225. Y_NO_INLINE TString TDecoratorComputationNodeBase::DebugStringImpl(const TString& typeName) const {
  226. return typeName + "(" + Node->DebugString() + ")";
  227. }
  228. Y_NO_INLINE TBinaryComputationNodeBase::TBinaryComputationNodeBase(IComputationNode* left, IComputationNode* right, EValueRepresentation kind)
  229. : Left(left)
  230. , Right(right)
  231. , Kind(kind)
  232. {}
  233. Y_NO_INLINE ui32 TBinaryComputationNodeBase::GetIndexImpl() const {
  234. THROW yexception() << "Can't get index from decorator node.";
  235. }
  236. Y_NO_INLINE TString TBinaryComputationNodeBase::DebugStringImpl(const TString& typeName) const {
  237. return typeName + "(" + Left->DebugString() + "," + Right->DebugString() + ")";
  238. }
  239. void TExternalComputationNode::CollectDependentIndexes(const IComputationNode*, TIndexesMap& map) const {
  240. map.emplace(ValueIndex, RepresentationKind);
  241. }
  242. TExternalComputationNode::TExternalComputationNode(TComputationMutables& mutables, EValueRepresentation kind)
  243. : TStatefulComputationNode(mutables, kind)
  244. {
  245. mutables.CachedValues.push_back(ValueIndex);
  246. }
  247. NUdf::TUnboxedValue TExternalComputationNode::GetValue(TComputationContext& ctx) const {
  248. return Getter ? Getter(ctx) : ValueRef(ctx);
  249. }
  250. NUdf::TUnboxedValue& TExternalComputationNode::RefValue(TComputationContext& ctx) const {
  251. InvalidateValue(ctx);
  252. return ValueRef(ctx);
  253. }
  254. void TExternalComputationNode::SetValue(TComputationContext& ctx, NUdf::TUnboxedValue&& value) const {
  255. ValueRef(ctx) = std::move(value);
  256. InvalidateValue(ctx);
  257. }
  258. TString TExternalComputationNode::DebugString() const {
  259. return "External";
  260. }
  261. void TExternalComputationNode::RegisterDependencies() const {}
  262. void TExternalComputationNode::SetOwner(const IComputationNode* owner) {
  263. Y_DEBUG_ABORT_UNLESS(!Owner);
  264. Owner = owner;
  265. }
  266. void TExternalComputationNode::PrepareStageOne() {
  267. std::sort(Dependencies.begin(), Dependencies.end());
  268. Dependencies.erase(std::unique(Dependencies.begin(), Dependencies.end()), Dependencies.cend());
  269. if (const auto it = std::find(Dependencies.cbegin(), Dependencies.cend(), Owner); Dependencies.cend() != it)
  270. Dependencies.erase(it);
  271. }
  272. void TExternalComputationNode::PrepareStageTwo() {
  273. TIndexesMap dependencies;
  274. std::for_each(Dependencies.cbegin(), Dependencies.cend(),
  275. std::bind(&IComputationNode::CollectDependentIndexes, std::placeholders::_1, Owner, std::ref(dependencies)));
  276. InvalidationSet.assign(dependencies.cbegin(), dependencies.cend());
  277. }
  278. const IComputationNode* TExternalComputationNode::GetSource() const { return nullptr; }
  279. ui32 TExternalComputationNode::GetDependencyWeight() const { return 0U; }
  280. bool TExternalComputationNode::IsTemporaryValue() const {
  281. return bool(Getter);
  282. }
  283. void TExternalComputationNode::SetGetter(TGetter&& getter) {
  284. Getter = std::move(getter);
  285. }
  286. void TExternalComputationNode::InvalidateValue(TComputationContext& ctx) const {
  287. for (const auto& index : InvalidationSet) {
  288. ctx.MutableValues[index.first] = NUdf::TUnboxedValuePod::Invalid();
  289. }
  290. }
  291. bool TComputationValueBaseNotSupportedStub::HasFastListLength() const {
  292. ThrowNotSupported(__func__);
  293. return false;
  294. }
  295. ui64 TComputationValueBaseNotSupportedStub::GetListLength() const {
  296. ThrowNotSupported(__func__);
  297. return 0;
  298. }
  299. ui64 TComputationValueBaseNotSupportedStub::GetEstimatedListLength() const {
  300. ThrowNotSupported(__func__);
  301. return 0;
  302. }
  303. bool TComputationValueBaseNotSupportedStub::HasListItems() const {
  304. ThrowNotSupported(__func__);
  305. return false;
  306. }
  307. const NUdf::TOpaqueListRepresentation* TComputationValueBaseNotSupportedStub::GetListRepresentation() const {
  308. return nullptr;
  309. }
  310. NUdf::IBoxedValuePtr TComputationValueBaseNotSupportedStub::ReverseListImpl(const NUdf::IValueBuilder& builder) const {
  311. Y_UNUSED(builder);
  312. return nullptr;
  313. }
  314. NUdf::IBoxedValuePtr TComputationValueBaseNotSupportedStub::SkipListImpl(const NUdf::IValueBuilder& builder, ui64 count) const {
  315. Y_UNUSED(builder);
  316. Y_UNUSED(count);
  317. return nullptr;
  318. }
  319. NUdf::IBoxedValuePtr TComputationValueBaseNotSupportedStub::TakeListImpl(const NUdf::IValueBuilder& builder, ui64 count) const {
  320. Y_UNUSED(builder);
  321. Y_UNUSED(count);
  322. return nullptr;
  323. }
  324. NUdf::IBoxedValuePtr TComputationValueBaseNotSupportedStub::ToIndexDictImpl(const NUdf::IValueBuilder& builder) const {
  325. Y_UNUSED(builder);
  326. return nullptr;
  327. }
  328. ui64 TComputationValueBaseNotSupportedStub::GetDictLength() const {
  329. ThrowNotSupported(__func__);
  330. return 0;
  331. }
  332. bool TComputationValueBaseNotSupportedStub::HasDictItems() const {
  333. ThrowNotSupported(__func__);
  334. return false;
  335. }
  336. NUdf::TStringRef TComputationValueBaseNotSupportedStub::GetResourceTag() const {
  337. ThrowNotSupported(__func__);
  338. return NUdf::TStringRef();
  339. }
  340. void* TComputationValueBaseNotSupportedStub::GetResource() {
  341. ThrowNotSupported(__func__);
  342. return nullptr;
  343. }
  344. void TComputationValueBaseNotSupportedStub::Apply(NUdf::IApplyContext& applyCtx) const {
  345. Y_UNUSED(applyCtx);
  346. }
  347. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetListIterator() const {
  348. ThrowNotSupported(__func__);
  349. return {};
  350. }
  351. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetDictIterator() const {
  352. ThrowNotSupported(__func__);
  353. return {};
  354. }
  355. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetKeysIterator() const {
  356. ThrowNotSupported(__func__);
  357. return {};
  358. }
  359. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetPayloadsIterator() const {
  360. ThrowNotSupported(__func__);
  361. return {};
  362. }
  363. bool TComputationValueBaseNotSupportedStub::Contains(const NUdf::TUnboxedValuePod& key) const {
  364. Y_UNUSED(key);
  365. ThrowNotSupported(__func__);
  366. return false;
  367. }
  368. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::Lookup(const NUdf::TUnboxedValuePod& key) const {
  369. Y_UNUSED(key);
  370. ThrowNotSupported(__func__);
  371. return NUdf::TUnboxedValuePod();
  372. }
  373. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetElement(ui32 index) const {
  374. Y_UNUSED(index);
  375. ThrowNotSupported(__func__);
  376. return {};
  377. }
  378. const NUdf::TUnboxedValue* TComputationValueBaseNotSupportedStub::GetElements() const {
  379. return nullptr;
  380. }
  381. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::Run(
  382. const NUdf::IValueBuilder* valueBuilder,
  383. const NUdf::TUnboxedValuePod* args) const
  384. {
  385. Y_UNUSED(valueBuilder);
  386. Y_UNUSED(args);
  387. ThrowNotSupported(__func__);
  388. return {};
  389. }
  390. bool TComputationValueBaseNotSupportedStub::Skip() {
  391. NUdf::TUnboxedValue stub;
  392. return Next(stub);
  393. }
  394. bool TComputationValueBaseNotSupportedStub::Next(NUdf::TUnboxedValue&) {
  395. ThrowNotSupported(__func__);
  396. return false;
  397. }
  398. bool TComputationValueBaseNotSupportedStub::NextPair(NUdf::TUnboxedValue&, NUdf::TUnboxedValue&) {
  399. ThrowNotSupported(__func__);
  400. return false;
  401. }
  402. ui32 TComputationValueBaseNotSupportedStub::GetVariantIndex() const {
  403. ThrowNotSupported(__func__);
  404. return 0;
  405. }
  406. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetVariantItem() const {
  407. ThrowNotSupported(__func__);
  408. return {};
  409. }
  410. NUdf::EFetchStatus TComputationValueBaseNotSupportedStub::Fetch(NUdf::TUnboxedValue& result) {
  411. Y_UNUSED(result);
  412. ThrowNotSupported(__func__);
  413. return NUdf::EFetchStatus::Finish;
  414. }
  415. ui32 TComputationValueBaseNotSupportedStub::GetTraverseCount() const {
  416. ThrowNotSupported(__func__);
  417. return 0;
  418. }
  419. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::GetTraverseItem(ui32 index) const {
  420. Y_UNUSED(index);
  421. ThrowNotSupported(__func__);
  422. return {};
  423. }
  424. NUdf::TUnboxedValue TComputationValueBaseNotSupportedStub::Save() const {
  425. ThrowNotSupported(__func__);
  426. return NUdf::TUnboxedValue::Zero();
  427. }
  428. void TComputationValueBaseNotSupportedStub::Load(const NUdf::TStringRef& state) {
  429. Y_UNUSED(state);
  430. ThrowNotSupported(__func__);
  431. }
  432. bool TComputationValueBaseNotSupportedStub::Load2(const NUdf::TUnboxedValue& state) {
  433. Y_UNUSED(state);
  434. ThrowNotSupported(__func__);
  435. return false;
  436. }
  437. void TComputationValueBaseNotSupportedStub::Push(const NUdf::TUnboxedValuePod& value) {
  438. Y_UNUSED(value);
  439. ThrowNotSupported(__func__);
  440. }
  441. bool TComputationValueBaseNotSupportedStub::IsSortedDict() const {
  442. ThrowNotSupported(__func__);
  443. return false;
  444. }
  445. void TComputationValueBaseNotSupportedStub::Unused1() {
  446. ThrowNotSupported(__func__);
  447. }
  448. void TComputationValueBaseNotSupportedStub::Unused2() {
  449. ThrowNotSupported(__func__);
  450. }
  451. void TComputationValueBaseNotSupportedStub::Unused3() {
  452. ThrowNotSupported(__func__);
  453. }
  454. void TComputationValueBaseNotSupportedStub::Unused4() {
  455. ThrowNotSupported(__func__);
  456. }
  457. void TComputationValueBaseNotSupportedStub::Unused5() {
  458. ThrowNotSupported(__func__);
  459. }
  460. void TComputationValueBaseNotSupportedStub::Unused6() {
  461. ThrowNotSupported(__func__);
  462. }
  463. NUdf::EFetchStatus TComputationValueBaseNotSupportedStub::WideFetch(NUdf::TUnboxedValue* result, ui32 width) {
  464. Y_UNUSED(result);
  465. Y_UNUSED(width);
  466. ThrowNotSupported(__func__);
  467. return NUdf::EFetchStatus::Finish;
  468. }
  469. TString TWideFlowProxyComputationNode::DebugString() const { return "WideFlowArg"; }
  470. EValueRepresentation TWideFlowProxyComputationNode::GetRepresentation() const {
  471. THROW yexception() << "Failed to get representation kind.";
  472. }
  473. NUdf::TUnboxedValue TWideFlowProxyComputationNode::GetValue(TComputationContext&) const {
  474. THROW yexception() << "Failed to get value from wide flow node.";
  475. }
  476. ui32 TWideFlowProxyComputationNode::GetIndex() const {
  477. THROW yexception() << "Failed to get proxy node index.";
  478. }
  479. ui32 TWideFlowProxyComputationNode::GetDependencyWeight() const {
  480. THROW yexception() << "Failed to get dependency weight.";
  481. }
  482. ui32 TWideFlowProxyComputationNode::GetDependencesCount() const {
  483. return Dependence ? 1U : 0U;
  484. }
  485. IComputationNode* TWideFlowProxyComputationNode::AddDependence(const IComputationNode* node) {
  486. Y_DEBUG_ABORT_UNLESS(!Dependence);
  487. Dependence = node;
  488. return this;
  489. }
  490. const IComputationNode* TWideFlowProxyComputationNode::GetSource() const { return nullptr; }
  491. bool TWideFlowProxyComputationNode::IsTemporaryValue() const { return true; }
  492. void TWideFlowProxyComputationNode::RegisterDependencies() const {}
  493. void TWideFlowProxyComputationNode::PrepareStageOne() {}
  494. void TWideFlowProxyComputationNode::PrepareStageTwo() {
  495. if (Dependence) {
  496. TIndexesMap dependencies;
  497. Dependence->CollectDependentIndexes(Owner, dependencies);
  498. InvalidationSet.assign(dependencies.cbegin(), dependencies.cend());
  499. }
  500. }
  501. void TWideFlowProxyComputationNode::SetOwner(const IComputationNode* owner) {
  502. Y_DEBUG_ABORT_UNLESS(!Owner);
  503. Owner = owner;
  504. }
  505. void TWideFlowProxyComputationNode::CollectDependentIndexes(const IComputationNode*, TIndexesMap&) const {
  506. THROW yexception() << "Failed to collect dependent indexes.";
  507. }
  508. void TWideFlowProxyComputationNode::InvalidateValue(TComputationContext& ctx) const {
  509. for (const auto& index : InvalidationSet) {
  510. ctx.MutableValues[index.first] = NUdf::TUnboxedValuePod::Invalid();
  511. }
  512. }
  513. void TWideFlowProxyComputationNode::SetFetcher(TFetcher&& fetcher) {
  514. Fetcher = std::move(fetcher);
  515. }
  516. EFetchResult TWideFlowProxyComputationNode::FetchValues(TComputationContext& ctx, NUdf::TUnboxedValue*const* values) const {
  517. return Fetcher(ctx, values);
  518. }
  519. IComputationNode* LocateNode(const TNodeLocator& nodeLocator, TCallable& callable, ui32 index, bool pop) {
  520. return nodeLocator(callable.GetInput(index).GetNode(), pop);
  521. }
  522. IComputationNode* LocateNode(const TNodeLocator& nodeLocator, TNode& node, bool pop) {
  523. return nodeLocator(&node, pop);
  524. }
  525. IComputationExternalNode* LocateExternalNode(const TNodeLocator& nodeLocator, TCallable& callable, ui32 index, bool pop) {
  526. return dynamic_cast<IComputationExternalNode*>(LocateNode(nodeLocator, callable, index, pop));
  527. }
  528. template<class TContainerOne, class TContainerTwo>
  529. TPasstroughtMap GetPasstroughtMap(const TContainerOne& from, const TContainerTwo& to) {
  530. TPasstroughtMap map(from.size());
  531. for (size_t i = 0U; i < map.size(); ++i) {
  532. for (size_t j = 0U; j < to.size(); ++j) {
  533. if (from[i] == to[j]) {
  534. map[i].emplace(j);
  535. break;
  536. }
  537. }
  538. }
  539. return map;
  540. }
  541. template<class TContainerOne, class TContainerTwo>
  542. TPasstroughtMap GetPasstroughtMapOneToOne(const TContainerOne& from, const TContainerTwo& to) {
  543. TPasstroughtMap map(from.size());
  544. std::unordered_map<typename TContainerOne::value_type, size_t> unique(map.size());
  545. for (size_t i = 0U; i < map.size(); ++i) {
  546. if (const auto ins = unique.emplace(from[i], i); ins.second) {
  547. for (size_t j = 0U; j < to.size(); ++j) {
  548. if (from[i] == to[j]) {
  549. if (auto& item = map[i]) {
  550. item.reset();
  551. break;
  552. } else
  553. item.emplace(j);
  554. }
  555. }
  556. } else
  557. map[ins.first->second].reset();
  558. }
  559. return map;
  560. }
  561. template TPasstroughtMap GetPasstroughtMap(const TComputationExternalNodePtrVector& from, const TComputationNodePtrVector& to);
  562. template TPasstroughtMap GetPasstroughtMap(const TComputationNodePtrVector& from, const TComputationExternalNodePtrVector& to);
  563. template TPasstroughtMap GetPasstroughtMapOneToOne(const TComputationExternalNodePtrVector& from, const TComputationNodePtrVector& to);
  564. template TPasstroughtMap GetPasstroughtMapOneToOne(const TComputationNodePtrVector& from, const TComputationExternalNodePtrVector& to);
  565. std::optional<size_t> IsPasstrought(const IComputationNode* root, const TComputationExternalNodePtrVector& args) {
  566. for (size_t i = 0U; i < args.size(); ++i)
  567. if (root == args[i])
  568. return {i};
  569. return std::nullopt;
  570. }
  571. TPasstroughtMap MergePasstroughtMaps(const TPasstroughtMap& lhs, const TPasstroughtMap& rhs) {
  572. TPasstroughtMap map(std::min(lhs.size(), rhs.size()));
  573. auto i = 0U;
  574. for (auto& item : map) {
  575. if (const auto l = lhs[i], r = rhs[i]; l && r && *l == *r) {
  576. item.emplace(*l);
  577. }
  578. ++i;
  579. }
  580. return map;
  581. }
  582. void ApplyChanges(const NUdf::TUnboxedValue& list, NUdf::IApplyContext& applyCtx) {
  583. TThresher<false>::DoForEachItem(list,
  584. [&applyCtx] (const NUdf::TUnboxedValue& item) {
  585. if (item.IsBoxed())
  586. item.Apply(applyCtx);
  587. }
  588. );
  589. }
  590. const IComputationNode* GetCommonSource(const IComputationNode* first, const IComputationNode* second, const IComputationNode* common) {
  591. const auto f = first->GetSource();
  592. const auto s = second->GetSource();
  593. if (f && s) {
  594. return f == s ? f : common;
  595. } else if (f) {
  596. return f;
  597. } else if (s) {
  598. return s;
  599. }
  600. return common;
  601. }
  602. void CleanupCurrentContext() {
  603. auto& allocState = *TlsAllocState;
  604. if (allocState.CurrentContext) {
  605. TAllocState::CleanupPAllocList(allocState.CurrentPAllocList);
  606. }
  607. }
  608. }
  609. }