123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010 |
- #include "mkql_map_join.h"
- #include <yql/essentials/minikql/computation/mkql_computation_node_codegen.h> // Y_IGNORE
- #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h>
- #include <yql/essentials/minikql/computation/mkql_computation_node_holders_codegen.h>
- #include <yql/essentials/minikql/mkql_node_cast.h>
- #include <yql/essentials/minikql/mkql_program_builder.h>
- #include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h>
- #include <util/string/cast.h>
- namespace NKikimr {
- namespace NMiniKQL {
- namespace {
- template<bool IsTuple>
- class TWideMapJoinBase {
- protected:
- TWideMapJoinBase(TComputationMutables& mutables, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames,
- IComputationWideFlowNode* flow, IComputationNode* dict, ui32 inputWidth)
- : LeftKeyConverters(std::move(leftKeyConverters))
- , DictType(dictType)
- , OutputRepresentations(std::move(outputRepresentations))
- , LeftKeyColumns(std::move(leftKeyColumns))
- , LeftRenames(std::move(leftRenames))
- , RightRenames(std::move(rightRenames))
- , UsedInputs(GetUsedInputs())
- , Flow(flow)
- , Dict(dict)
- , KeyTuple(mutables)
- , InputsIndex(mutables.CurValueIndex)
- , WideFieldsIndex(mutables.CurWideFieldsIndex)
- {
- mutables.DeferWideFieldsInit(inputWidth, UsedInputs);
- }
- #ifndef MKQL_DISABLE_CODEGEN
- Value* GenMakeKeysTuple(Value* keysPtr, const ICodegeneratorInlineWideNode::TGettersList& getters, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto zero = ConstantInt::get(Type::getInt128Ty(context), 0);
- const auto keys = getters[LeftKeyColumns.front()](ctx, block);
- new StoreInst(keys, keysPtr, block);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, keys, zero, "check", block);
- return check;
- }
- Value* GenMakeKeysTuple(Value* keysPtr, const ICodegeneratorInlineWideNode::TGettersList& getters, Value* itemsPtr, Type* keysType, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto idxType = Type::getInt32Ty(context);
- const auto zero = ConstantInt::get(Type::getInt128Ty(context), 0);
- const auto keys = KeyTuple.GenNewArray(LeftKeyColumns.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(PointerType::getUnqual(keysType), itemsPtr, "items", block);
- const auto done = BasicBlock::Create(context, "done", ctx.Func);
- const auto result = PHINode::Create(Type::getInt1Ty(context), (LeftKeyColumns.size() + 1U) << 1U , "result", done);
- const auto keyType = AS_TYPE(TTupleType, DictType->GetKeyType());
- for (ui32 i = 0; i < LeftKeyColumns.size(); ++i) {
- const auto index = ConstantInt::get(idxType, i);
- const auto ptr = GetElementPtrInst::CreateInBounds(keysType, items, {ConstantInt::get(idxType, 0), index}, (TString("ptr_") += ToString(i)).c_str(), block);
- const auto elem = getters[LeftKeyColumns[i]](ctx, block);
- const auto converter = reinterpret_cast<TGeneratorPtr>(LeftKeyConverters[i].Generator);
- const auto conv = converter ? converter(reinterpret_cast<Value *const *>(&elem), ctx, block) : elem;
- result->addIncoming(ConstantInt::getTrue(context), block);
- const auto next = BasicBlock::Create(context, (TString("next_") += ToString(i)).c_str(), ctx.Func);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, conv, zero, "check", block);
- BranchInst::Create(done, next, check, block);
- block = next;
- new StoreInst(conv, ptr, block);
- ValueAddRef(GetValueRepresentation(keyType->GetElementType(i)), conv, ctx, block);
- }
- result->addIncoming(ConstantInt::getFalse(context), block);
- BranchInst::Create(done, block);
- block = done;
- new StoreInst(keys, keysPtr, block);
- return result;
- }
- void GenFillLeftStruct(const std::vector<Value*>& pointers, ICodegeneratorInlineWideNode::TGettersList& output) const {
- for (auto i = 0U; i < pointers.size(); ++i) {
- output[LeftRenames[(i << 1U) + 1U]] = [p = pointers[i]](const TCodegenContext& ctx, BasicBlock*& block) {
- auto& context = ctx.Codegen.GetContext();
- return new LoadInst(Type::getInt128Ty(context), p, "value", block);
- };
- }
- }
- void GenFillLeftStruct(const ICodegeneratorInlineWideNode::TGettersList& input, ICodegeneratorInlineWideNode::TGettersList& output) const {
- for (auto i = 0U; i < LeftRenames.size(); ++i) {
- const auto& src = input[LeftRenames[i]];
- output[LeftRenames[++i]] = src;
- }
- }
- bool IsUnusedInput(const ui32 index) const {
- for (auto i = 0U; i < LeftRenames.size(); ++++i)
- if (LeftRenames[i] == index)
- return false;
- return true;
- }
- template<class TLeftSideSource>
- std::array<Value*, 2U> GenFillOutput(ui32 idx, const TCodegenContext& ctx, const TLeftSideSource& input, ICodegeneratorInlineWideNode::TGettersList& output) const {
- GenFillLeftStruct(input, output);
- auto& context = ctx.Codegen.GetContext();
- const auto valueType = Type::getInt128Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- auto width = 0U;
- for (auto i = 0U; i < RightRenames.size(); ++++i) {
- width = std::max(width, RightRenames[i]);
- }
- const auto arrayType = ArrayType::get(valueType, ++width);
- const auto atTop = &ctx.Func->getEntryBlock().back();
- const auto stub = new AllocaInst(arrayType, 0U, "stub", atTop);
- Value* zeros = UndefValue::get(arrayType);
- for (auto i = 0U; i < width; ++i) {
- zeros = InsertValueInst::Create(zeros, zero, {i}, (TString("zero_") += ToString(i)).c_str(), atTop);
- }
- new StoreInst(zeros, stub, atTop);
- const auto item = new AllocaInst(valueType, 0U, "item", atTop);
- const auto placeholder = new AllocaInst(stub->getType(), 0U, "placeholder", atTop);
- const auto pointer = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), idx)}, "pointer", atTop);
- for (auto i = 0U; i < RightRenames.size(); ++i) {
- const auto from = RightRenames[i];
- const auto to = RightRenames[++i];
- const auto kind = OutputRepresentations[to];
- output[to] = [from, kind, item, pointer, placeholder, arrayType, valueType](const TCodegenContext& ctx, BasicBlock*& block) {
- auto& context = ctx.Codegen.GetContext();
- const auto index = ConstantInt::get(Type::getInt32Ty(context), from);
- const auto pointerType = PointerType::getUnqual(arrayType);
- const auto elements = new LoadInst(pointerType, placeholder, "elements", block);
- const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elements, ConstantPointerNull::get(pointerType), "null", block);
- const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
- const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
- const auto done = BasicBlock::Create(context, "done", ctx.Func);
- const auto out = PHINode::Create(pointer->getType(), 2U, "out", done);
- BranchInst::Create(slow, fast, null, block);
- block = fast;
- const auto ptr = GetElementPtrInst::CreateInBounds(arrayType, elements, {ConstantInt::get(Type::getInt32Ty(context), 0), index}, "ptr", block);
- out->addIncoming(ptr, block);
- BranchInst::Create(done, block);
- block = slow;
- const auto value = new LoadInst(valueType, pointer, "value", block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(item, value, ctx.Codegen, block, index);
- ValueRelease(kind, item, ctx, block);
- out->addIncoming(item, block);
- BranchInst::Create(done, block);
- block = done;
- const auto load = new LoadInst(valueType, out, "load", block);
- return load;
- };
- }
- return {{placeholder, stub}};
- }
- #endif
- NUdf::TUnboxedValue MakeKeysTuple(TComputationContext& ctx, NUdf::TUnboxedValue** fields) const {
- if constexpr (IsTuple) {
- NUdf::TUnboxedValue* items = nullptr;
- const auto keys = KeyTuple.NewArray(ctx, LeftKeyColumns.size(), items);
- if (!LeftKeyColumns.empty()) {
- Y_ABORT_UNLESS(items);
- for (auto i = 0U; i < LeftKeyColumns.size(); ++i) {
- const auto value = fields[LeftKeyColumns[i]];
- const auto converter = LeftKeyConverters[i].Function;
- if (!(*items++ = converter ? converter(value) : *value))
- return NUdf::TUnboxedValuePod();
- }
- }
- return keys;
- } else {
- const auto value = fields[LeftKeyColumns.front()];
- const auto converter = LeftKeyConverters.front().Function;
- return converter ? converter(value) : *value;
- }
- }
- void FillLeftStruct(NUdf::TUnboxedValue*const* output, NUdf::TUnboxedValue** fields) const {
- for (auto i = 0U; i < LeftRenames.size(); ++i) {
- const auto prevIndex = LeftRenames[i];
- const auto newIndex = LeftRenames[++i];
- if (const auto out = output[newIndex])
- *out = *fields[prevIndex];
- }
- }
- void FillRightStruct(const NUdf::TUnboxedValue& structObj, NUdf::TUnboxedValue*const* output) const {
- if (const auto ptr = structObj.GetElements()) {
- for (auto i = 0U; i < RightRenames.size(); ++i) {
- const auto prevIndex = RightRenames[i];
- const auto newIndex = RightRenames[++i];
- if (const auto out = output[newIndex])
- *out = ptr[prevIndex];
- }
- } else {
- for (auto i = 0U; i < RightRenames.size(); ++i) {
- const auto prevIndex = RightRenames[i];
- const auto newIndex = RightRenames[++i];
- if (const auto out = output[newIndex])
- *out = structObj.GetElement(prevIndex);
- }
- }
- }
- void NullRightStruct(NUdf::TUnboxedValue*const* output) const {
- for (auto i = 0U; i < RightRenames.size(); ++i) {
- const auto newIndex = RightRenames[++i];
- if (const auto out = output[newIndex])
- *out = NUdf::TUnboxedValuePod();
- }
- }
- std::set<ui32> GetUsedInputs() const {
- std::set<ui32> unique;
- for (auto i = 0U; i < LeftKeyColumns.size(); ++i)
- unique.emplace(LeftKeyColumns[i]);
- for (auto i = 0U; i < LeftRenames.size(); i += 2U)
- unique.emplace(LeftRenames[i]);
- return unique;
- }
- const std::vector<TFunctionDescriptor> LeftKeyConverters;
- TDictType* const DictType;
- const std::vector<EValueRepresentation> OutputRepresentations;
- const std::vector<ui32> LeftKeyColumns;
- const std::vector<ui32> LeftRenames;
- const std::vector<ui32> RightRenames;
- const std::set<ui32> UsedInputs;
- IComputationWideFlowNode* const Flow;
- IComputationNode* const Dict;
- const TContainerCacheOnContext KeyTuple;
- ui32 InputsIndex;
- ui32 WideFieldsIndex;
- };
- template<bool WithoutRight, bool RightRequired, bool IsTuple>
- class TWideMapJoinWrapper : public TWideMapJoinBase<IsTuple>, public TStatefulWideFlowCodegeneratorNode<TWideMapJoinWrapper<WithoutRight, RightRequired, IsTuple>> {
- using TBaseComputation = TStatefulWideFlowCodegeneratorNode<TWideMapJoinWrapper<WithoutRight, RightRequired, IsTuple>>;
- public:
- TWideMapJoinWrapper(TComputationMutables& mutables, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames,
- IComputationWideFlowNode* flow, IComputationNode* dict, ui32 inputWidth)
- : TWideMapJoinBase<IsTuple>(mutables, std::move(leftKeyConverters), dictType, std::move(outputRepresentations)
- , std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), flow, dict, inputWidth)
- , TBaseComputation(mutables, flow, EValueRepresentation::Boxed)
- {}
- EFetchResult DoCalculate(NUdf::TUnboxedValue& lookup, TComputationContext& ctx, NUdf::TUnboxedValue*const* output) const {
- auto** fields = ctx.WideFields.data() + this->WideFieldsIndex;
- const auto dict = this->Dict->GetValue(ctx);
- do {
- if (const auto res = this->Flow->FetchValues(ctx, fields); EFetchResult::One != res)
- return res;
- const auto keys = this->MakeKeysTuple(ctx, fields);
- if constexpr (WithoutRight) {
- if ((keys && dict.Contains(keys)) == RightRequired)
- break;
- else
- continue;
- } else if (keys) {
- if (lookup = dict.Lookup(keys)) {
- this->FillLeftStruct(output, fields);
- this->FillRightStruct(lookup, output);
- return EFetchResult::One;
- }
- }
- } while (RightRequired || WithoutRight);
- this->FillLeftStruct(output, fields);
- this->NullRightStruct(output);
- return EFetchResult::One;
- }
- #ifndef MKQL_DISABLE_CODEGEN
- ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* lookupPtr, BasicBlock*& block) const {
- MKQL_ENSURE(!this->Dict->IsTemporaryValue(), "Dict can't be temporary");
- auto& context = ctx.Codegen.GetContext();
- const auto valueType = Type::getInt128Ty(context);
- const auto resultType = Type::getInt32Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", &ctx.Func->getEntryBlock().back()) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", &ctx.Func->getEntryBlock().back());
- new StoreInst(zero, keysPtr, block);
- const auto dict = GetNodeValue(this->Dict, ctx, block);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
- const auto result = PHINode::Create(resultType, RightRequired ? 2U : 3U, "result", stop);
- BranchInst::Create(loop, block);
- block = loop;
- const auto current = GetNodeValues(this->Flow, ctx, block);
- const auto special = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLE, current.first, ConstantInt::get(resultType, 0), "special", block);
- result->addIncoming(current.first, block);
- BranchInst::Create(stop, next, special, block);
- block = next;
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current.second, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current.second, ctx, block);
- ICodegeneratorInlineWideNode::TGettersList getters(this->OutputRepresentations.size());
- if constexpr (WithoutRight) {
- this->GenFillLeftStruct(current.second, getters);
- if constexpr (RightRequired) {
- BranchInst::Create(loop, step, none, block);
- } else {
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- BranchInst::Create(stop, step, none, block);
- }
- block = step;
- const auto cont = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Contains>(Type::getInt1Ty(context), dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueCleanup(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- if constexpr (RightRequired) {
- BranchInst::Create(stop, loop, cont, block);
- } else {
- BranchInst::Create(loop, stop, cont, block);
- }
- } else {
- const auto output = this->GenFillOutput(static_cast<const IComputationNode*>(this)->GetIndex(), ctx, current.second, getters);
- const auto left = RightRequired ? nullptr : BasicBlock::Create(context, "left", ctx.Func);
- if constexpr (RightRequired)
- BranchInst::Create(loop, step, none, block);
- else
- BranchInst::Create(left, step, none, block);
- block = step;
- ValueUnRef(EValueRepresentation::Boxed, lookupPtr, ctx, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(lookupPtr, dict, ctx.Codegen, block, keysPtr);
- const auto lookup = new LoadInst(valueType, lookupPtr, "lookup", block);
- if constexpr (!IsTuple) {
- ValueCleanup(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- if constexpr (RightRequired)
- BranchInst::Create(full, loop, ok, block);
- else {
- BranchInst::Create(full, left, ok, block);
- block = left;
- new StoreInst(std::get<1U>(output), std::get<0U>(output), block);
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- BranchInst::Create(stop, block);
- }
- {
- block = full;
- const auto elements = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(std::get<1U>(output)->getType(), lookup, ctx.Codegen, block);
- new StoreInst(elements, std::get<0U>(output), block);
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- BranchInst::Create(stop, block);
- }
- }
- block = stop;
- return {result, std::move(getters)};
- }
- #endif
- private:
- void RegisterDependencies() const final {
- if (const auto flow = this->FlowDependsOn(this->Flow))
- this->DependsOn(flow, this->Dict);
- }
- };
- template<bool RightRequired, bool IsTuple>
- class TWideMultiMapJoinWrapper : public TWideMapJoinBase<IsTuple>, public TPairStateWideFlowCodegeneratorNode<TWideMultiMapJoinWrapper<RightRequired, IsTuple>> {
- using TBaseComputation = TPairStateWideFlowCodegeneratorNode<TWideMultiMapJoinWrapper<RightRequired, IsTuple>>;
- using TBase = TWideMapJoinBase<IsTuple>;
- public:
- TWideMultiMapJoinWrapper(TComputationMutables& mutables, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames,
- IComputationWideFlowNode* flow, IComputationNode* dict, ui32 inputWidth)
- : TWideMapJoinBase<IsTuple>(mutables, std::move(leftKeyConverters), dictType, std::move(outputRepresentations)
- , std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), flow, dict, inputWidth)
- , TBaseComputation(mutables, flow, EValueRepresentation::Boxed, EValueRepresentation::Boxed)
- {
- if (!TBase::LeftRenames.empty()) {
- LeftRenamesStorageIndex = mutables.CurValueIndex;
- mutables.CurValueIndex += TBase::LeftRenames.size() >> 1U;
- }
- }
- EFetchResult DoCalculate(NUdf::TUnboxedValue& iter, NUdf::TUnboxedValue& item, TComputationContext& ctx, NUdf::TUnboxedValue*const* output) const {
- auto** fields = ctx.WideFields.data() + this->WideFieldsIndex;
- for (auto iterator = std::move(iter);;) {
- if (iterator.HasValue()) {
- if (iterator.Next(item)) {
- this->FillLeftStruct(output, fields);
- this->FillRightStruct(item, output);
- iter = std::move(iterator);
- return EFetchResult::One;
- }
- }
- for (const auto& dict = this->Dict->GetValue(ctx);;) {
- if (const auto res = this->Flow->FetchValues(ctx, fields); EFetchResult::One != res)
- return res;
- if (const auto keys = this->MakeKeysTuple(ctx, fields)) {
- if (const auto lookup = dict.Lookup(keys)) {
- iterator = lookup.GetListIterator();
- break;
- }
- }
- if constexpr (!RightRequired) {
- this->FillLeftStruct(output, fields);
- this->NullRightStruct(output);
- return EFetchResult::One;
- }
- }
- }
- }
- #ifndef MKQL_DISABLE_CODEGEN
- ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, Value* iteraratorPtr, Value* itemPtr, BasicBlock*& block) const {
- MKQL_ENSURE(!this->Dict->IsTemporaryValue(), "Dict can't be temporary");
- auto& context = ctx.Codegen.GetContext();
- const auto resultType = Type::getInt32Ty(context);
- const auto valueType = Type::getInt128Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", &ctx.Func->getEntryBlock().back()) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", &ctx.Func->getEntryBlock().back());
- std::vector<Value*> leftStoragePointers;
- leftStoragePointers.reserve(TBase::LeftRenames.size() >> 1U);
- auto i = 0U;
- const auto values = ctx.GetMutables();
- std::generate_n(std::back_inserter(leftStoragePointers), TBase::LeftRenames.size() >> 1U,
- [&](){ return GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(resultType, LeftRenamesStorageIndex + i++)}, (TString("left_out_") += ToString(i)).c_str(), &ctx.Func->getEntryBlock().back()); });
- const auto work = BasicBlock::Create(context, "work", ctx.Func);
- BranchInst::Create(work, block);
- block = work;
- const auto subiter = new LoadInst(valueType, iteraratorPtr, "subiter", block);
- const auto hasi = BasicBlock::Create(context, "hasi", ctx.Func);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- const auto skip = BasicBlock::Create(context, "loop", ctx.Func);
- const auto part = BasicBlock::Create(context, "part", ctx.Func);
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto fill = BasicBlock::Create(context, "fill", ctx.Func);
- const auto left = RightRequired ? nullptr : BasicBlock::Create(context, "left", ctx.Func);
- const auto exit = BasicBlock::Create(context, "exit", ctx.Func);
- const auto result = PHINode::Create(resultType, RightRequired ? 2U : 3U, "result", exit);
- ICodegeneratorInlineWideNode::TGettersList getters(this->OutputRepresentations.size());
- BranchInst::Create(hasi, part, HasValue(subiter, block, context), block);
- block = part;
- for (const auto ptr : leftStoragePointers) {
- new StoreInst(GetInvalid(context), ptr, block);
- }
- const auto dict = GetNodeValue(this->Dict, ctx, block);
- BranchInst::Create(loop, block);
- block = loop;
- const auto current = GetNodeValues(this->Flow, ctx, block);
- const auto output = this->GenFillOutput(static_cast<const IComputationNode*>(this)->GetIndex() + 1U, ctx, leftStoragePointers, getters);
- const auto special = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLE, current.first, ConstantInt::get(resultType, 0), "special", block);
- result->addIncoming(current.first, block);
- BranchInst::Create(exit, next, special, block);
- block = hasi;
- const auto status = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Next>(Type::getInt1Ty(context), subiter, ctx.Codegen, block, itemPtr);
- BranchInst::Create(full, skip, status, block);
- {
- block = full;
- const auto item = new LoadInst(valueType, itemPtr, "item", block);
- const auto elements = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(std::get<1U>(output)->getType(), item, ctx.Codegen, block);
- new StoreInst(elements, std::get<0U>(output), block);
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- BranchInst::Create(exit, block);
- }
- {
- block = skip;
- UnRefBoxed(subiter, ctx, block);
- new StoreInst(zero, iteraratorPtr, block);
- for (auto i = 0U; i < leftStoragePointers.size(); ++i) {
- const auto ptr = leftStoragePointers[i];
- ValueUnRef(TBase::OutputRepresentations[TBase::LeftRenames[(i << 1U) + 1U]], ptr, ctx, block);
- new StoreInst(GetInvalid(context), ptr, block);
- }
- BranchInst::Create(part, block);
- }
- block = next;
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current.second, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current.second, ctx, block);
- if constexpr (RightRequired)
- BranchInst::Create(loop, step, none, block);
- else
- BranchInst::Create(left, step, none, block);
- block = step;
- ValueUnRef(EValueRepresentation::Boxed, itemPtr, ctx, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(itemPtr, dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- if (this->IsUnusedInput(this->LeftKeyColumns.front())) {
- ValueCleanup(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- }
- const auto lookup = new LoadInst(valueType, itemPtr, "lookup", block);
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- if constexpr (RightRequired)
- BranchInst::Create(fill, loop, ok, block);
- else {
- BranchInst::Create(fill, left, ok, block);
- block = left;
- for (auto i = 0U; i < leftStoragePointers.size(); ++i) {
- const auto item = current.second[TBase::LeftRenames[i << 1U]](ctx, block);
- new StoreInst(item, leftStoragePointers[i], block);
- }
- new StoreInst(std::get<1U>(output), std::get<0U>(output), block);
- result->addIncoming(ConstantInt::get(resultType, i32(EFetchResult::One)), block);
- BranchInst::Create(exit, block);
- }
- {
- block = fill;
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetListIterator>(iteraratorPtr, lookup, ctx.Codegen, block);
- for (auto i = 0U; i < leftStoragePointers.size(); ++i) {
- const auto item = current.second[TBase::LeftRenames[i << 1U]](ctx, block);
- ValueAddRef(TBase::OutputRepresentations[TBase::LeftRenames[(i << 1U) + 1U]], item, ctx, block);
- new StoreInst(item, leftStoragePointers[i], block);
- }
- BranchInst::Create(work, block);
- }
- block = exit;
- return {result, std::move(getters)};
- }
- #endif
- private:
- void RegisterDependencies() const final {
- if (const auto flow = this->FlowDependsOn(this->Flow))
- this->DependsOn(flow, this->Dict);
- }
- ui32 LeftRenamesStorageIndex = 0U;
- };
- template<bool IsTuple>
- class TMapJoinCoreWrapperBase {
- protected:
- TMapJoinCoreWrapperBase(TComputationMutables& mutables, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames, IComputationNode* stream, IComputationNode* dict)
- : LeftKeyConverters(std::move(leftKeyConverters))
- , DictType(dictType)
- , OutputRepresentations(std::move(outputRepresentations))
- , LeftKeyColumns(std::move(leftKeyColumns))
- , LeftRenames(std::move(leftRenames))
- , RightRenames(std::move(rightRenames))
- , Stream(stream)
- , Dict(dict)
- , ResStruct(mutables)
- , KeyTuple(mutables)
- {}
- static void FillStruct(const NUdf::TUnboxedValue& structObj, NUdf::TUnboxedValue* items, const std::vector<ui32>& renames) {
- if (renames.empty()) {
- return;
- }
- Y_ABORT_UNLESS(items);
- if (const auto ptr = structObj.GetElements()) {
- for (auto i = 0U; i < renames.size();) {
- const auto prevIndex = renames[i++];
- const auto newIndex = renames[i++];
- items[newIndex] = ptr[prevIndex];
- }
- } else {
- for (auto i = 0U; i < renames.size();) {
- const auto prevIndex = renames[i++];
- const auto newIndex = renames[i++];
- items[newIndex] = structObj.GetElement(prevIndex);
- }
- }
- }
- void FillLeftStruct(const NUdf::TUnboxedValue& structObj, NUdf::TUnboxedValue* items) const {
- FillStruct(structObj, items, LeftRenames);
- }
- void FillRightStruct(const NUdf::TUnboxedValue& structObj, NUdf::TUnboxedValue* items) const {
- FillStruct(structObj, items, RightRenames);
- }
- NUdf::TUnboxedValue MakeKeysTuple(TComputationContext& ctx, const NUdf::TUnboxedValuePod& structObj) const {
- if (IsTuple) {
- NUdf::TUnboxedValue* items = nullptr;
- const auto keys = KeyTuple.NewArray(ctx, LeftKeyColumns.size(), items);
- if (!LeftKeyColumns.empty()) {
- Y_ABORT_UNLESS(items);
- const auto ptr = structObj.GetElements();
- for (auto i = 0U; i < LeftKeyColumns.size(); ++i) {
- auto value = ptr ? ptr[LeftKeyColumns[i]] : structObj.GetElement(LeftKeyColumns[i]);
- const auto converter = LeftKeyConverters[i].Function;
- if (!(*items++ = converter ? NUdf::TUnboxedValue(converter(&value)) : std::move(value)))
- return NUdf::TUnboxedValuePod();
- }
- }
- return keys;
- } else {
- const auto value = structObj.GetElement(LeftKeyColumns.front());
- const auto converter = LeftKeyConverters.front().Function;
- return converter ? NUdf::TUnboxedValue(converter(&value)) : value;
- }
- }
- #ifndef MKQL_DISABLE_CODEGEN
- void GenFillLeftStruct(Value* left, Value* items, Type* arrayType, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto idxType = Type::getInt32Ty(context);
- const auto valType = Type::getInt128Ty(context);
- const auto ptrType = PointerType::getUnqual(valType);
- const auto elements = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(ptrType, left, ctx.Codegen, block);
- const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elements, ConstantPointerNull::get(ptrType), "null", block);
- const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
- const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
- const auto done = BasicBlock::Create(context, "done", ctx.Func);
- BranchInst::Create(slow, fast, null, block);
- {
- block = fast;
- for (auto i = 0U; i < LeftRenames.size();) {
- const auto oldI = LeftRenames[i++];
- const auto newI = LeftRenames[i++];
- const auto oldIndex = ConstantInt::get(idxType, oldI);
- const auto newIndex = ConstantInt::get(idxType, newI);
- const auto oldPtr = GetElementPtrInst::CreateInBounds(valType, elements, {oldIndex}, "old", block);
- const auto newPtr = GetElementPtrInst::CreateInBounds(arrayType, items, {ConstantInt::get(idxType, 0), newIndex}, "new", block);
- const auto item = new LoadInst(valType, oldPtr, "item", block);
- new StoreInst(item, newPtr, block);
- ValueAddRef(OutputRepresentations[newI], newPtr, ctx, block);
- }
- BranchInst::Create(done, block);
- }
- {
- block = slow;
- for (auto i = 0U; i < LeftRenames.size();) {
- const auto oldI = LeftRenames[i++];
- const auto newI = LeftRenames[i++];
- const auto oldIndex = ConstantInt::get(idxType, oldI);
- const auto newIndex = ConstantInt::get(idxType, newI);
- const auto item = GetElementPtrInst::CreateInBounds(arrayType, items, {ConstantInt::get(idxType, 0), newIndex}, "item", block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(item, left, ctx.Codegen, block, oldIndex);
- }
- BranchInst::Create(done, block);
- }
- block = done;
- }
- void GenFillRightStruct(Value* right, Value* items, Type* arrayType, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto idxType = Type::getInt32Ty(context);
- const auto valType = Type::getInt128Ty(context);
- const auto ptrType = PointerType::getUnqual(valType);
- const auto elements = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(ptrType, right, ctx.Codegen, block);
- const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elements, ConstantPointerNull::get(ptrType), "null", block);
- const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
- const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
- const auto done = BasicBlock::Create(context, "done", ctx.Func);
- BranchInst::Create(slow, fast, null, block);
- {
- block = fast;
- for (auto i = 0U; i < RightRenames.size();) {
- const auto oldI = RightRenames[i++];
- const auto newI = RightRenames[i++];
- const auto oldIndex = ConstantInt::get(idxType, oldI);
- const auto newIndex = ConstantInt::get(idxType, newI);
- const auto oldPtr = GetElementPtrInst::CreateInBounds(valType, elements, {oldIndex}, "old", block);
- const auto newPtr = GetElementPtrInst::CreateInBounds(arrayType, items, {ConstantInt::get(idxType, 0), newIndex}, "new", block);
- const auto elem = new LoadInst(valType, oldPtr, "elem", block);
- new StoreInst(elem, newPtr, block);
- ValueAddRef(OutputRepresentations[newI], newPtr, ctx, block);
- }
- BranchInst::Create(done, block);
- }
- {
- block = slow;
- for (auto i = 0U; i < RightRenames.size();) {
- const auto oldI = RightRenames[i++];
- const auto newI = RightRenames[i++];
- const auto oldIndex = ConstantInt::get(idxType, oldI);
- const auto newIndex = ConstantInt::get(idxType, newI);
- const auto item = GetElementPtrInst::CreateInBounds(arrayType, items, {ConstantInt::get(idxType, 0), newIndex}, "item", block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(item, right, ctx.Codegen, block, oldIndex);
- }
- BranchInst::Create(done, block);
- }
- block = done;
- }
- Value* GenMakeKeysTuple(Value* keysPtr, Value* current, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto idxType = Type::getInt32Ty(context);
- const auto zero = ConstantInt::get(Type::getInt128Ty(context), 0);
- const auto index = ConstantInt::get(idxType, LeftKeyColumns.front());
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(keysPtr, current, ctx.Codegen, block, index);
- if (const auto converter = reinterpret_cast<TGeneratorPtr>(LeftKeyConverters.front().Generator)) {
- Value *const elem = new LoadInst(Type::getInt128Ty(context), keysPtr, "elem", block);
- const auto conv = converter(&elem, ctx, block);
- new StoreInst(conv, keysPtr, block);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, conv, zero, "check", block);
- return check;
- } else {
- const auto keys = new LoadInst(Type::getInt128Ty(context), keysPtr, "keys", block);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, keys, zero, "check", block);
- return check;
- }
- }
- Value* GenMakeKeysTuple(Value* keysPtr, Value* current, Value* itemsPtr, Type* keysType, const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto idxType = Type::getInt32Ty(context);
- const auto valueType = Type::getInt128Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto keys = KeyTuple.GenNewArray(LeftKeyColumns.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(PointerType::getUnqual(keysType), itemsPtr, "items", block);
- const auto ptrType = PointerType::getUnqual(valueType);
- const auto elements = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(ptrType, current, ctx.Codegen, block);
- const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elements, ConstantPointerNull::get(ptrType), "null", block);
- const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
- const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
- const auto done = BasicBlock::Create(context, "done", ctx.Func);
- const auto result = PHINode::Create(Type::getInt1Ty(context), (LeftKeyColumns.size() + 1U) << 1U , "result", done);
- BranchInst::Create(slow, fast, null, block);
- {
- block = fast;
- const auto keyType = AS_TYPE(TTupleType, DictType->GetKeyType());
- for (ui32 i = 0; i < LeftKeyColumns.size(); ++i) {
- const auto oldIndex = ConstantInt::get(idxType, LeftKeyColumns[i]);
- const auto newIndex = ConstantInt::get(idxType, i);
- const auto oldPtr = GetElementPtrInst::CreateInBounds(valueType, elements, {oldIndex}, "old", block);
- const auto newPtr = GetElementPtrInst::CreateInBounds(keysType, items, {ConstantInt::get(idxType, 0), newIndex}, "new", block);
- const auto elem = new LoadInst(valueType, oldPtr, "elem", block);
- const auto converter = reinterpret_cast<TGeneratorPtr>(LeftKeyConverters[i].Generator);
- const auto conv = converter ? converter(reinterpret_cast<Value *const *>(&elem), ctx, block) : elem;
- result->addIncoming(ConstantInt::getTrue(context), block);
- const auto next = BasicBlock::Create(context, (TString("next_") += ToString(i)).c_str(), ctx.Func);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, conv, zero, "check", block);
- BranchInst::Create(done, next, check, block);
- block = next;
- new StoreInst(conv, newPtr, block);
- ValueAddRef(GetValueRepresentation(keyType->GetElementType(i)), newPtr, ctx, block);
- }
- result->addIncoming(ConstantInt::getFalse(context), block);
- BranchInst::Create(done, block);
- }
- {
- block = slow;
- for (ui32 i = 0; i < LeftKeyColumns.size(); ++i) {
- const auto item = GetElementPtrInst::CreateInBounds(keysType, items, {ConstantInt::get(idxType, 0), ConstantInt::get(idxType, i)}, "item", block);
- const auto index = ConstantInt::get(idxType, LeftKeyColumns[i]);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(item, current, ctx.Codegen, block, index);
- const auto next = BasicBlock::Create(context, (TString("next_") += ToString(i)).c_str(), ctx.Func);
- const auto elem = new LoadInst(valueType, item, "elem", block);
- if (const auto converter = reinterpret_cast<TGeneratorPtr>(LeftKeyConverters[i].Generator)) {
- const auto conv = converter(reinterpret_cast<Value *const *>(&elem), ctx, block);
- new StoreInst(conv, item, block);
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, conv, zero, "check", block);
- result->addIncoming(ConstantInt::getTrue(context), block);
- BranchInst::Create(done, next, check, block);
- } else {
- const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elem, zero, "check", block);
- result->addIncoming(ConstantInt::getTrue(context), block);
- BranchInst::Create(done, next, check, block);
- }
- block = next;
- }
- result->addIncoming(ConstantInt::getFalse(context), block);
- BranchInst::Create(done, block);
- }
- block = done;
- new StoreInst(keys, keysPtr, block);
- return result;
- }
- #endif
- const std::vector<TFunctionDescriptor> LeftKeyConverters;
- TDictType* const DictType;
- const std::vector<EValueRepresentation> OutputRepresentations;
- const std::vector<ui32> LeftKeyColumns;
- const std::vector<ui32> LeftRenames;
- const std::vector<ui32> RightRenames;
- IComputationNode* const Stream;
- IComputationNode* const Dict;
- const TContainerCacheOnContext ResStruct;
- const TContainerCacheOnContext KeyTuple;
- };
- enum class ERightKind {
- None = 0,
- Once,
- Many
- };
- template<ERightKind RightKind, bool RightRequired, bool IsTuple>
- class TMapJoinCoreFlowWrapper : public TMapJoinCoreWrapperBase<IsTuple>, public std::conditional_t<ERightKind::Many != RightKind,
- TStatelessFlowCodegeneratorNode<TMapJoinCoreFlowWrapper<RightKind, RightRequired, IsTuple>>,
- TPairStateFlowCodegeneratorNode<TMapJoinCoreFlowWrapper<RightKind, RightRequired, IsTuple>>> {
- typedef std::conditional_t<ERightKind::Many != RightKind,
- TStatelessFlowCodegeneratorNode<TMapJoinCoreFlowWrapper<RightKind, RightRequired, IsTuple>>,
- TPairStateFlowCodegeneratorNode<TMapJoinCoreFlowWrapper<RightKind, RightRequired, IsTuple>>> TBaseComputation;
- public:
- TMapJoinCoreFlowWrapper(TComputationMutables& mutables, EValueRepresentation kind, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames,
- IComputationNode* flow, IComputationNode* dict)
- : TMapJoinCoreWrapperBase<IsTuple>(mutables, std::move(leftKeyConverters), dictType, std::move(outputRepresentations),
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), flow, dict), TBaseComputation(mutables, flow, kind)
- {}
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- for (const auto dict = this->Dict->GetValue(ctx);;) {
- auto item = this->Stream->GetValue(ctx);
- if (item.IsSpecial()) {
- return item.Release();
- }
- const auto keys = this->MakeKeysTuple(ctx, item);
- switch (RightKind) {
- case ERightKind::Once:
- if (keys) {
- if (const auto lookup = dict.Lookup(keys)) {
- NUdf::TUnboxedValue* items = nullptr;
- const auto result = this->ResStruct.NewArray(ctx, this->OutputRepresentations.size(), items);
- this->FillLeftStruct(item, items);
- this->FillRightStruct(lookup, items);
- return result;
- }
- }
- if constexpr (RightRequired)
- continue;
- else
- break;
- case ERightKind::None:
- if constexpr (RightRequired) {
- if (keys && dict.Contains(keys))
- break;
- else
- continue;
- } else {
- if (keys && dict.Contains(keys))
- continue;
- else
- break;
- }
- default:
- Y_ABORT("Unreachable");
- }
- NUdf::TUnboxedValue* items = nullptr;
- const auto result = this->ResStruct.NewArray(ctx, this->OutputRepresentations.size(), items);
- this->FillLeftStruct(item, items);
- return result;
- }
- }
- NUdf::TUnboxedValuePod DoCalculate(NUdf::TUnboxedValue& curr, NUdf::TUnboxedValue& iter, TComputationContext& ctx) const {
- for (auto iterator = std::move(iter);;) {
- if (iterator.HasValue() && curr.HasValue()) {
- if (NUdf::TUnboxedValue item; iterator.Next(item)) {
- NUdf::TUnboxedValue* items = nullptr;
- const auto result = this->ResStruct.NewArray(ctx, this->OutputRepresentations.size(), items);
- this->FillLeftStruct(curr, items);
- this->FillRightStruct(item, items);
- iter = std::move(iterator);
- return result;
- }
- }
- const auto& dict = this->Dict->GetValue(ctx);
- for (auto current = std::move(curr);;) {
- current = this->Stream->GetValue(ctx);
- if (current.IsSpecial()) {
- return current.Release();
- }
- if (const auto keys = this->MakeKeysTuple(ctx, current)) {
- if (const auto lookup = dict.Lookup(keys)) {
- iterator = lookup.GetListIterator();
- curr = std::move(current);
- break;
- }
- }
- if constexpr (!RightRequired) {
- NUdf::TUnboxedValue* items = nullptr;
- const auto result = this->ResStruct.NewArray(ctx, this->OutputRepresentations.size(), items);
- this->FillLeftStruct(current, items);
- return result;
- }
- }
- }
- }
- #ifndef MKQL_DISABLE_CODEGEN
- Value* DoGenerateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto valueType = Type::getInt128Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto arrayType = ArrayType::get(valueType, this->OutputRepresentations.size());
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto itemsType = PointerType::getUnqual(arrayType);
- const auto itemsPtr = new AllocaInst(itemsType, 0U, "items_ptr", &ctx.Func->getEntryBlock().back());
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", &ctx.Func->getEntryBlock().back()) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", &ctx.Func->getEntryBlock().back());
- new StoreInst(zero, keysPtr, block);
- const auto itemPtr = new AllocaInst(valueType, 0U, "item_ptr", &ctx.Func->getEntryBlock().back());
- new StoreInst(zero, itemPtr, block);
- const auto dict = GetNodeValue(this->Dict, ctx, block);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
- const auto result = PHINode::Create(valueType, 3U, "result", stop);
- BranchInst::Create(loop, block);
- block = loop;
- const auto current = GetNodeValue(this->Stream, ctx, block);
- result->addIncoming(current, block);
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- BranchInst::Create(stop, next, IsSpecial(current, block, context), block);
- block = next;
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current, ctx, block);
- const auto skip = BasicBlock::Create(context, "skip", ctx.Func);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto half = BasicBlock::Create(context, "half", ctx.Func);
- BranchInst::Create(RightRequired ? skip : half, step, none, block);
- block = step;
- switch (RightKind) {
- case ERightKind::None: {
- const auto cont = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Contains>(Type::getInt1Ty(context), dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- if constexpr (RightRequired) {
- BranchInst::Create(half, skip, cont, block);
- } else {
- BranchInst::Create(skip, half, cont, block);
- }
- break;
- }
- case ERightKind::Once: {
- new StoreInst(zero, itemPtr, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(itemPtr, dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- const auto lookup = new LoadInst(valueType, itemPtr, "lookup", block);
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- BranchInst::Create(full, RightRequired ? skip : half, ok, block);
- {
- block = full;
- const auto out = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- this->GenFillRightStruct(lookup, items, arrayType, ctx, block);
- UnRefBoxed(lookup, ctx, block);
- ValueCleanup(static_cast<const IComputationNode*>(this)->GetRepresentation(), current, ctx, block);
- result->addIncoming(out, block);
- BranchInst::Create(stop, block);
- }
- break;
- }
- case ERightKind::Many:
- Y_ABORT("Wrong case");
- }
- {
- block = half;
- const auto out = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- ValueCleanup(static_cast<const IComputationNode*>(this)->GetRepresentation(), current, ctx, block);
- result->addIncoming(out, block);
- BranchInst::Create(stop, block);
- }
- {
- block = skip;
- ValueCleanup(static_cast<const IComputationNode*>(this)->GetRepresentation(), current, ctx, block);
- BranchInst::Create(loop, block);
- }
- block = stop;
- if (this->Dict->IsTemporaryValue())
- CleanupBoxed(dict, ctx, block);
- return result;
- }
- Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* currentPtr, Value* iteraratorPtr, BasicBlock*& block) const {
- auto& context = ctx.Codegen.GetContext();
- const auto valueType = Type::getInt128Ty(context);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto arrayType = ArrayType::get(valueType, this->OutputRepresentations.size());
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto itemsType = PointerType::getUnqual(arrayType);
- const auto itemsPtr = new AllocaInst(itemsType, 0U, "items_ptr", &ctx.Func->getEntryBlock().back());
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", &ctx.Func->getEntryBlock().back()) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", &ctx.Func->getEntryBlock().back());
- const auto itemPtr = new AllocaInst(valueType, 0U, "item_ptr", &ctx.Func->getEntryBlock().back());
- new StoreInst(zero, itemPtr, block);
- const auto work = BasicBlock::Create(context, "work", ctx.Func);
- BranchInst::Create(work, block);
- block = work;
- const auto subiter = new LoadInst(valueType, iteraratorPtr, "subiter", block);
- const auto hasi = BasicBlock::Create(context, "hasi", ctx.Func);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- const auto skip = BasicBlock::Create(context, "loop", ctx.Func);
- const auto part = BasicBlock::Create(context, "part", ctx.Func);
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
- const auto exit = BasicBlock::Create(context, "exit", ctx.Func);
- const auto result = PHINode::Create(valueType, 3U, "result", exit);
- BranchInst::Create(hasi, part, HasValue(subiter, block, context), block);
- block = hasi;
- const auto curr = new LoadInst(valueType, currentPtr, "curr", block);
- const auto status = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Next>(Type::getInt1Ty(context), subiter, ctx.Codegen, block, itemPtr);
- BranchInst::Create(full, skip, status, block);
- {
- block = full;
- const auto out = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- const auto item = new LoadInst(valueType, itemPtr, "item", block);
- this->GenFillLeftStruct(curr, items, arrayType, ctx, block);
- this->GenFillRightStruct(item, items, arrayType, ctx, block);
- UnRefBoxed(item, ctx, block);
- result->addIncoming(out, block);
- BranchInst::Create(exit, block);
- }
- {
- block = skip;
- UnRefBoxed(curr, ctx, block);
- UnRefBoxed(subiter, ctx, block);
- new StoreInst(zero, currentPtr, block);
- new StoreInst(zero, iteraratorPtr, block);
- BranchInst::Create(part, block);
- }
- block = part;
- const auto dict = GetNodeValue(this->Dict, ctx, block);
- BranchInst::Create(loop, block);
- block = loop;
- GetNodeValue(currentPtr, this->Stream, ctx, block);
- const auto current = new LoadInst(valueType, currentPtr, "current", block);
- BranchInst::Create(stop, next, IsSpecial(current, block, context), block);
- block = stop;
- if (this->Dict->IsTemporaryValue())
- CleanupBoxed(dict, ctx, block);
- result->addIncoming(current, block);
- BranchInst::Create(exit, block);
- block = next;
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current, ctx, block);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto hsnt = BasicBlock::Create(context, "half", ctx.Func);
- BranchInst::Create(hsnt, step, none, block);
- block = step;
- new StoreInst(zero, itemPtr, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(itemPtr, dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- const auto lookup = new LoadInst(valueType, itemPtr, "lookup", block);
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- const auto fill = BasicBlock::Create(context, "fill", ctx.Func);
- BranchInst::Create(fill, hsnt, ok, block);
- block = hsnt;
- if constexpr (RightRequired) {
- UnRefBoxed(current, ctx, block);
- new StoreInst(zero, currentPtr, block);
- BranchInst::Create(loop, block);
- } else {
- const auto out = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- UnRefBoxed(current, ctx, block);
- new StoreInst(zero, currentPtr, block);
- if (this->Dict->IsTemporaryValue())
- CleanupBoxed(dict, ctx, block);
- result->addIncoming(out, block);
- BranchInst::Create(exit, block);
- }
- {
- block = fill;
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetListIterator>(iteraratorPtr, lookup, ctx.Codegen, block);
- CleanupBoxed(lookup, ctx, block);
- BranchInst::Create(work, block);
- }
- block = exit;
- return result;
- }
- #endif
- private:
- void RegisterDependencies() const final {
- if (const auto flow = this->FlowDependsOn(this->Stream))
- this->DependsOn(flow, this->Dict);
- }
- };
- template<ERightKind RightKind, bool RightRequired, bool IsTuple>
- class TMapJoinCoreWrapper : public TMapJoinCoreWrapperBase<IsTuple>, public TCustomValueCodegeneratorNode<TMapJoinCoreWrapper<RightKind, RightRequired, IsTuple>> {
- private:
- typedef TCustomValueCodegeneratorNode<TMapJoinCoreWrapper<RightKind, RightRequired, IsTuple>> TBaseComputation;
- class TCodegenValue : public TComputationValue<TCodegenValue> {
- public:
- using TBase = TComputationValue<TCodegenValue>;
- using TFetchPtr = NUdf::EFetchStatus (*)(TComputationContext*, NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod&);
- TCodegenValue(TMemoryUsageInfo* memInfo, TFetchPtr fetch, TComputationContext* ctx, NUdf::TUnboxedValue&& stream, NUdf::TUnboxedValue&& dict)
- : TBase(memInfo)
- , FetchFunc(fetch)
- , Ctx(ctx)
- , Stream(std::move(stream))
- , Dict(std::move(dict))
- {}
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
- return FetchFunc(Ctx, static_cast<const NUdf::TUnboxedValuePod&>(Stream), static_cast<const NUdf::TUnboxedValuePod&>(Dict), result);
- }
- const TFetchPtr FetchFunc;
- TComputationContext* const Ctx;
- const NUdf::TUnboxedValue Stream;
- const NUdf::TUnboxedValue Dict;
- };
- class TCodegenStatefulValue : public TComputationValue<TCodegenStatefulValue> {
- public:
- using TBase = TComputationValue<TCodegenStatefulValue>;
- using TFetchPtr = NUdf::EFetchStatus (*)(TComputationContext*, NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod&, NUdf::TUnboxedValuePod&, NUdf::TUnboxedValuePod&);
- TCodegenStatefulValue(TMemoryUsageInfo* memInfo, TFetchPtr fetch, TComputationContext* ctx, NUdf::TUnboxedValue&& stream, NUdf::TUnboxedValue&& dict)
- : TBase(memInfo)
- , FetchFunc(fetch)
- , Ctx(ctx)
- , Stream(std::move(stream))
- , Dict(std::move(dict))
- {}
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
- return FetchFunc(Ctx, static_cast<const NUdf::TUnboxedValuePod&>(Stream), static_cast<const NUdf::TUnboxedValuePod&>(Dict), Current, Iterator, result);
- }
- const TFetchPtr FetchFunc;
- TComputationContext* const Ctx;
- const NUdf::TUnboxedValue Stream;
- const NUdf::TUnboxedValue Dict;
- NUdf::TUnboxedValue Current;
- NUdf::TUnboxedValue Iterator;
- };
- using TSelf = TMapJoinCoreWrapper<RightKind, RightRequired, IsTuple>;
- using TBase = TCustomValueCodegeneratorNode<TSelf>;
- class TValue : public TComputationValue<TValue> {
- public:
- using TBase = TComputationValue<TValue>;
- TValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& stream,
- NUdf::TUnboxedValue&& dict, TComputationContext& ctx, const TSelf* self)
- : TBase(memInfo)
- , Stream(std::move(stream))
- , Dict(std::move(dict))
- , Ctx(ctx)
- , Self(self)
- {}
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- for (NUdf::TUnboxedValue current;;) {
- if (const auto status = Stream.Fetch(current); status != NUdf::EFetchStatus::Ok) {
- return status;
- }
- const auto keys = Self->MakeKeysTuple(Ctx, current);
- switch (RightKind) {
- case ERightKind::Once:
- if (keys) {
- if (const auto lookup = Dict.Lookup(keys)) {
- NUdf::TUnboxedValue* items = nullptr;
- result = Self->ResStruct.NewArray(Ctx, Self->OutputRepresentations.size(), items);
- Self->FillLeftStruct(current, items);
- Self->FillRightStruct(lookup, items);
- return NUdf::EFetchStatus::Ok;
- }
- }
- if constexpr (RightRequired)
- continue;
- else
- break;
- case ERightKind::None:
- if constexpr (RightRequired) {
- if (keys && Dict.Contains(keys))
- break;
- else
- continue;
- } else {
- if (keys && Dict.Contains(keys))
- continue;
- else
- break;
- }
- default:
- Y_ABORT("Unreachable");
- }
- NUdf::TUnboxedValue* items = nullptr;
- result = Self->ResStruct.NewArray(Ctx, Self->OutputRepresentations.size(), items);
- Self->FillLeftStruct(current, items);
- return NUdf::EFetchStatus::Ok;
- }
- }
- private:
- NUdf::TUnboxedValue Stream;
- NUdf::TUnboxedValue Dict;
- TComputationContext& Ctx;
- const TSelf* const Self;
- };
- class TMultiRowValue : public TComputationValue<TMultiRowValue> {
- public:
- using TBase = TComputationValue<TMultiRowValue>;
- TMultiRowValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& stream,
- NUdf::TUnboxedValue&& dict, TComputationContext& ctx, const TSelf* self)
- : TBase(memInfo)
- , Stream(std::move(stream))
- , Dict(std::move(dict))
- , Ctx(ctx)
- , Self(self)
- {}
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- for (auto iterator = std::move(Iterator);;) {
- if (iterator && Current) {
- if (NUdf::TUnboxedValue item; iterator.Next(item)) {
- NUdf::TUnboxedValue* items = nullptr;
- result = Self->ResStruct.NewArray(Ctx, Self->OutputRepresentations.size(), items);
- Self->FillLeftStruct(Current, items);
- Self->FillRightStruct(item, items);
- Iterator = std::move(iterator);
- return NUdf::EFetchStatus::Ok;
- }
- }
- for (auto current = std::move(Current);;) {
- if (const auto status = Stream.Fetch(current); NUdf::EFetchStatus::Ok != status) {
- return status;
- }
- if (const auto keys = Self->MakeKeysTuple(Ctx, current)) {
- if (const auto lookup = Dict.Lookup(keys)) {
- iterator = lookup.GetListIterator();
- Current = std::move(current);
- break;
- }
- }
- if (!RightRequired) {
- NUdf::TUnboxedValue* items = nullptr;
- result = Self->ResStruct.NewArray(Ctx, Self->OutputRepresentations.size(), items);
- Self->FillLeftStruct(current, items);
- return NUdf::EFetchStatus::Ok;
- }
- }
- }
- }
- private:
- NUdf::TUnboxedValue Stream;
- NUdf::TUnboxedValue Dict;
- TComputationContext& Ctx;
- const TSelf* const Self;
- NUdf::TUnboxedValue Current;
- NUdf::TUnboxedValue Iterator;
- };
- using TMyCodegenValue = std::conditional_t<ERightKind::Many == RightKind, TCodegenStatefulValue, TCodegenValue>;
- public:
- TMapJoinCoreWrapper(TComputationMutables& mutables, std::vector<TFunctionDescriptor>&& leftKeyConverters,
- TDictType* dictType, std::vector<EValueRepresentation>&& outputRepresentations, std::vector<ui32>&& leftKeyColumns,
- std::vector<ui32>&& leftRenames, std::vector<ui32>&& rightRenames,
- IComputationNode* stream, IComputationNode* dict)
- : TMapJoinCoreWrapperBase<IsTuple>(mutables, std::move(leftKeyConverters), dictType, std::move(outputRepresentations),
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), stream, dict), TBaseComputation(mutables)
- {}
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- #ifndef MKQL_DISABLE_CODEGEN
- if (ctx.ExecuteLLVM && MapJoin)
- return ctx.HolderFactory.Create<TMyCodegenValue>(MapJoin, &ctx, this->Stream->GetValue(ctx), this->Dict->GetValue(ctx));
- #endif
- return ctx.HolderFactory.Create<std::conditional_t<ERightKind::Many == RightKind, TMultiRowValue, TValue>>(this->Stream->GetValue(ctx), this->Dict->GetValue(ctx), ctx, this);
- }
- private:
- void RegisterDependencies() const final {
- this->DependsOn(this->Stream);
- this->DependsOn(this->Dict);
- }
- #ifndef MKQL_DISABLE_CODEGEN
- void GenerateFunctions(NYql::NCodegen::ICodegen& codegen) final {
- MapJoinFunc = RightKind == ERightKind::Many ? GenerateStatefulMapper(codegen) : GenerateMapper(codegen);
- codegen.ExportSymbol(MapJoinFunc);
- }
- void FinalizeFunctions(NYql::NCodegen::ICodegen& codegen) final {
- if (MapJoinFunc)
- MapJoin = reinterpret_cast<TMapJoinPtr>(codegen.GetPointerToFunction(MapJoinFunc));
- }
- Function* GenerateMapper(NYql::NCodegen::ICodegen& codegen) const {
- auto& module = codegen.GetModule();
- auto& context = codegen.GetContext();
- const auto& name = TBaseComputation::MakeName("Fetch");
- if (const auto f = module.getFunction(name.c_str()))
- return f;
- const auto valueType = Type::getInt128Ty(context);
- const auto arrayType = ArrayType::get(valueType, this->OutputRepresentations.size());
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto containerType = static_cast<Type*>(valueType);
- const auto contextType = GetCompContextType(context);
- const auto statusType = Type::getInt32Ty(context);
- const auto funcType = FunctionType::get(statusType, {PointerType::getUnqual(contextType), containerType, containerType, PointerType::getUnqual(valueType)}, false);
- TCodegenContext ctx(codegen);
- ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
- DISubprogramAnnotator annotator(ctx, ctx.Func);
- auto args = ctx.Func->arg_begin();
- ctx.Ctx = &*args;
- const auto streamArg = &*++args;
- const auto dictArg = &*++args;
- const auto valuePtr = &*++args;
- const auto main = BasicBlock::Create(context, "main", ctx.Func);
- auto block = main;
- const auto stream = static_cast<Value*>(streamArg);
- const auto dict = static_cast<Value*>(dictArg);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto fsok = ConstantInt::get(statusType, static_cast<ui32>(NUdf::EFetchStatus::Ok));
- const auto itemsType = PointerType::getUnqual(arrayType);
- const auto itemsPtr = new AllocaInst(itemsType, 0U, "items_ptr", block);
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", block) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", block);
- new StoreInst(zero, keysPtr, block);
- const auto itemPtr = new AllocaInst(valueType, 0U, "item_ptr", block);
- new StoreInst(zero, itemPtr, block);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
- BranchInst::Create(loop, block);
- block = loop;
- const auto status = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Fetch>(statusType, stream, codegen, block, itemPtr);
- ReturnInst::Create(context, status, stop);
- const auto stat = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, status, fsok, "stat", block);
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- BranchInst::Create(stop, next, stat, block);
- block = next;
- const auto current = new LoadInst(valueType, itemPtr, "current", block);
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current, ctx, block);
- const auto skip = BasicBlock::Create(context, "skip", ctx.Func);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto half = BasicBlock::Create(context, "half", ctx.Func);
- BranchInst::Create(RightRequired ? skip : half, step, none, block);
- block = step;
- switch (RightKind) {
- case ERightKind::None: {
- const auto cont = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Contains>(Type::getInt1Ty(context), dict, codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- if constexpr (RightRequired) {
- BranchInst::Create(half, skip, cont, block);
- } else {
- BranchInst::Create(skip, half, cont, block);
- }
- break;
- }
- case ERightKind::Once: {
- new StoreInst(zero, itemPtr, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(itemPtr, dict, codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- const auto lookup = new LoadInst(valueType, itemPtr, "lookup", block);
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- BranchInst::Create(full, RightRequired ? skip : half, ok, block);
- {
- block = full;
- const auto result = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- this->GenFillRightStruct(lookup, items, arrayType, ctx, block);
- SafeUnRefUnboxedOne(valuePtr, ctx, block);
- AddRefBoxed(result, ctx, block);
- new StoreInst(result, valuePtr, block);
- UnRefBoxed(current, ctx, block);
- UnRefBoxed(lookup, ctx, block);
- ReturnInst::Create(context, fsok, block);
- }
- break;
- }
- case ERightKind::Many:
- Y_ABORT("Wrong case");
- }
- {
- block = half;
- const auto result = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- SafeUnRefUnboxedOne(valuePtr, ctx, block);
- AddRefBoxed(result, ctx, block);
- new StoreInst(result, valuePtr, block);
- UnRefBoxed(current, ctx, block);
- ReturnInst::Create(context, fsok, block);
- }
- {
- block = skip;
- UnRefBoxed(current, ctx, block);
- new StoreInst(zero, itemPtr, block);
- BranchInst::Create(loop, block);
- }
- return ctx.Func;
- }
- Function* GenerateStatefulMapper(NYql::NCodegen::ICodegen& codegen) const {
- auto& module = codegen.GetModule();
- auto& context = codegen.GetContext();
- const auto& name = TBaseComputation::MakeName("Fetch");
- if (const auto f = module.getFunction(name.c_str()))
- return f;
- const auto valueType = Type::getInt128Ty(context);
- const auto arrayType = ArrayType::get(valueType, this->OutputRepresentations.size());
- const auto keysType = IsTuple ? ArrayType::get(valueType, this->LeftKeyColumns.size()) : nullptr;
- const auto containerType = static_cast<Type*>(valueType);
- const auto contextType = GetCompContextType(context);
- const auto statusType = Type::getInt32Ty(context);
- const auto funcType = FunctionType::get(statusType, {PointerType::getUnqual(contextType), containerType, containerType, PointerType::getUnqual(valueType), PointerType::getUnqual(valueType), PointerType::getUnqual(valueType)}, false);
- TCodegenContext ctx(codegen);
- ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
- DISubprogramAnnotator annotator(ctx, ctx.Func);
- auto args = ctx.Func->arg_begin();
- ctx.Ctx = &*args;
- const auto streamArg = &*++args;
- const auto dictArg = &*++args;
- const auto currentArg = &*++args;
- const auto iteratorArg = &*++args;
- const auto valuePtr = &*++args;
- const auto main = BasicBlock::Create(context, "main", ctx.Func);
- auto block = main;
- const auto stream = static_cast<Value*>(streamArg);
- const auto dict = static_cast<Value*>(dictArg);
- const auto zero = ConstantInt::get(valueType, 0);
- const auto fsok = ConstantInt::get(statusType, static_cast<ui32>(NUdf::EFetchStatus::Ok));
- const auto itemsType = PointerType::getUnqual(arrayType);
- const auto itemsPtr = new AllocaInst(itemsType, 0U, "items_ptr", block);
- const auto kitmsPtr = IsTuple ? new AllocaInst(PointerType::getUnqual(keysType), 0U, "kitms_ptr", block) : nullptr;
- const auto keysPtr = new AllocaInst(valueType, 0U, "keys_ptr", block);
- const auto itemPtr = new AllocaInst(valueType, 0U, "item_ptr", block);
- new StoreInst(zero, itemPtr, block);
- const auto work = BasicBlock::Create(context, "work", ctx.Func);
- BranchInst::Create(work, block);
- block = work;
- const auto subiter = new LoadInst(valueType, iteratorArg, "subiter", block);
- const auto hasi = BasicBlock::Create(context, "hasi", ctx.Func);
- const auto loop = BasicBlock::Create(context, "loop", ctx.Func);
- const auto full = BasicBlock::Create(context, "full", ctx.Func);
- const auto skip = BasicBlock::Create(context, "loop", ctx.Func);
- const auto ichk = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, subiter, zero, "ichk", block);
- BranchInst::Create(hasi, loop, ichk, block);
- {
- block = hasi;
- const auto status = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Next>(Type::getInt1Ty(context), subiter, codegen, block, itemPtr);
- BranchInst::Create(full, skip, status, block);
- }
- {
- block = skip;
- UnRefBoxed(subiter, ctx, block);
- new StoreInst(zero, iteratorArg, block);
- BranchInst::Create(loop, block);
- }
- {
- block = full;
- const auto result = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- const auto curr = new LoadInst(valueType, currentArg, "curr", block);
- const auto item = new LoadInst(valueType, itemPtr, "item", block);
- this->GenFillLeftStruct(curr, items, arrayType, ctx, block);
- this->GenFillRightStruct(item, items, arrayType, ctx, block);
- SafeUnRefUnboxedOne(valuePtr, ctx, block);
- AddRefBoxed(result, ctx, block);
- new StoreInst(result, valuePtr, block);
- UnRefBoxed(item, ctx, block);
- ReturnInst::Create(context, fsok, block);
- }
- const auto next = BasicBlock::Create(context, "next", ctx.Func);
- const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
- {
- block = loop;
- const auto status = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Fetch>(statusType, stream, codegen, block, currentArg);
- ReturnInst::Create(context, status, stop);
- const auto stat = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, status, fsok, "stat", block);
- BranchInst::Create(stop, next, stat, block);
- }
- {
- block = next;
- const auto current = new LoadInst(valueType, currentArg, "current", block);
- const auto none = IsTuple ?
- this->GenMakeKeysTuple(keysPtr, current, kitmsPtr, keysType, ctx, block):
- this->GenMakeKeysTuple(keysPtr, current, ctx, block);
- const auto step = BasicBlock::Create(context, "step", ctx.Func);
- const auto hsnt = RightRequired ? loop : BasicBlock::Create(context, "half", ctx.Func);
- BranchInst::Create(hsnt, step, none, block);
- block = step;
- new StoreInst(zero, itemPtr, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Lookup>(itemPtr, dict, ctx.Codegen, block, keysPtr);
- if constexpr (!IsTuple) {
- ValueUnRef(GetValueRepresentation(this->DictType->GetKeyType()), keysPtr, ctx, block);
- }
- const auto lookup = new LoadInst(valueType, itemPtr, "lookup", block);
- const auto ok = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, lookup, zero, "ok", block);
- const auto fill = BasicBlock::Create(context, "fill", ctx.Func);
- BranchInst::Create(fill, hsnt, ok, block);
- if constexpr (!RightRequired) {
- block = hsnt;
- const auto result = this->ResStruct.GenNewArray(this->OutputRepresentations.size(), itemsPtr, ctx, block);
- const auto items = new LoadInst(itemsType, itemsPtr, "items", block);
- this->GenFillLeftStruct(current, items, arrayType, ctx, block);
- SafeUnRefUnboxedOne(valuePtr, ctx, block);
- AddRefBoxed(result, ctx, block);
- new StoreInst(result, valuePtr, block);
- ReturnInst::Create(context, fsok, block);
- }
- {
- block = fill;
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetListIterator>(iteratorArg, lookup, codegen, block);
- CleanupBoxed(lookup, ctx, block);
- BranchInst::Create(work, block);
- }
- }
- return ctx.Func;
- }
- using TMapJoinPtr = typename TMyCodegenValue::TFetchPtr;
- Function* MapJoinFunc = nullptr;
- TMapJoinPtr MapJoin = nullptr;
- #endif
- };
- }
- IComputationNode* WrapMapJoinCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 6, "Expected 6 args");
- const auto type = callable.GetType()->GetReturnType();
- const auto leftStreamNode = callable.GetInput(0);
- const auto leftItemType = leftStreamNode.GetStaticType()->IsFlow() ?
- AS_TYPE(TFlowType, leftStreamNode)->GetItemType():
- AS_TYPE(TStreamType, leftStreamNode)->GetItemType();
- const auto dictNode = callable.GetInput(1);
- const auto dictType = AS_TYPE(TDictType, dictNode);
- const auto dictKeyType = dictType->GetKeyType();
- const auto joinKindNode = callable.GetInput(2);
- const auto rawKind = AS_VALUE(TDataLiteral, joinKindNode)->AsValue().Get<ui32>();
- const auto kind = GetJoinKind(rawKind);
- const bool isMany = dictType->GetPayloadType()->IsList();
- const bool boolWithoutRight = EJoinKind::LeftOnly == kind || EJoinKind::LeftSemi == kind;
- const auto returnItemType = type->IsFlow() ?
- AS_TYPE(TFlowType, callable.GetType()->GetReturnType())->GetItemType():
- AS_TYPE(TStreamType, callable.GetType()->GetReturnType())->GetItemType();
- const auto leftKeyColumnsNode = AS_VALUE(TTupleLiteral, callable.GetInput(3));
- const bool isTupleKey = leftKeyColumnsNode->GetValuesCount() > 1;
- const auto leftRenamesNode = AS_VALUE(TTupleLiteral, callable.GetInput(4));
- const auto rightRenamesNode = AS_VALUE(TTupleLiteral, callable.GetInput(5));
- std::vector<ui32> leftKeyColumns, leftRenames, rightRenames;
- leftKeyColumns.reserve(leftKeyColumnsNode->GetValuesCount());
- for (ui32 i = 0; i < leftKeyColumnsNode->GetValuesCount(); ++i) {
- leftKeyColumns.emplace_back(AS_VALUE(TDataLiteral, leftKeyColumnsNode->GetValue(i))->AsValue().Get<ui32>());
- }
- leftRenames.reserve(leftRenamesNode->GetValuesCount());
- for (ui32 i = 0; i < leftRenamesNode->GetValuesCount(); ++i) {
- leftRenames.emplace_back(AS_VALUE(TDataLiteral, leftRenamesNode->GetValue(i))->AsValue().Get<ui32>());
- }
- rightRenames.reserve(rightRenamesNode->GetValuesCount());
- for (ui32 i = 0; i < rightRenamesNode->GetValuesCount(); ++i) {
- rightRenames.emplace_back(AS_VALUE(TDataLiteral, rightRenamesNode->GetValue(i))->AsValue().Get<ui32>());
- }
- std::vector<TFunctionDescriptor> leftKeyConverters;
- leftKeyConverters.resize(leftKeyColumns.size());
- for (ui32 i = 0; i < leftKeyColumns.size(); ++i) {
- const auto leftColumnType = leftItemType->IsTuple() ?
- AS_TYPE(TTupleType, leftItemType)->GetElementType(leftKeyColumns[i]):
- (leftItemType->IsMulti() ?
- AS_TYPE(TMultiType, leftItemType)->GetElementType(leftKeyColumns[i]):
- AS_TYPE(TStructType, leftItemType)->GetMemberType(leftKeyColumns[i]));
- const auto rightType = isTupleKey ? AS_TYPE(TTupleType, dictKeyType)->GetElementType(i) : dictKeyType;
- bool isOptional;
- if (UnpackOptional(leftColumnType, isOptional)->IsSameType(*rightType)) {
- continue;
- }
- bool isLeftOptional;
- const auto leftDataType = UnpackOptionalData(leftColumnType, isLeftOptional);
- bool isRightOptional;
- const auto rightDataType = UnpackOptionalData(rightType, isRightOptional);
- if (leftDataType->GetSchemeType() != rightDataType->GetSchemeType()) {
- // find a converter
- const std::array<TArgType, 2U> argsTypes = {{{rightDataType->GetSchemeType(), isLeftOptional}, {leftDataType->GetSchemeType(), isLeftOptional}}};
- leftKeyConverters[i] = ctx.FunctionRegistry.GetBuiltins()->GetBuiltin("Convert", argsTypes.data(), 2U);
- }
- }
- std::vector<EValueRepresentation> outputRepresentations;
- if (returnItemType->IsTuple()) {
- const auto tupleType = AS_TYPE(TTupleType, returnItemType);
- outputRepresentations.reserve(tupleType->GetElementsCount());
- for (ui32 i = 0U; i < tupleType->GetElementsCount(); ++i)
- outputRepresentations.emplace_back(GetValueRepresentation(tupleType->GetElementType(i)));
- } else if (returnItemType->IsMulti()) {
- const auto multiType = AS_TYPE(TMultiType, returnItemType);
- outputRepresentations.reserve(multiType->GetElementsCount());
- for (ui32 i = 0U; i < multiType->GetElementsCount(); ++i)
- outputRepresentations.emplace_back(GetValueRepresentation(multiType->GetElementType(i)));
- } else if (returnItemType->IsStruct()) {
- const auto structType = AS_TYPE(TStructType, returnItemType);
- outputRepresentations.reserve(structType->GetMembersCount());
- for (ui32 i = 0U; i < structType->GetMembersCount(); ++i)
- outputRepresentations.emplace_back(GetValueRepresentation(structType->GetMemberType(i)));
- }
- const auto flow = LocateNode(ctx.NodeLocator, callable, 0);
- const auto dict = LocateNode(ctx.NodeLocator, callable, 1);
- #define NEW_WRAPPER(KIND, RIGHT_REQ, IS_TUPLE) \
- if (type->IsFlow()) { \
- if (const auto wide = dynamic_cast<IComputationWideFlowNode*>(flow)) { \
- const auto width = GetWideComponentsCount(AS_TYPE(TFlowType, callable.GetInput(0U).GetStaticType())); \
- if (boolWithoutRight) \
- return new TWideMapJoinWrapper<true, RIGHT_REQ, IS_TUPLE>(ctx.Mutables, \
- std::move(leftKeyConverters), dictType, std::move(outputRepresentations), \
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), wide, dict, width); \
- else if (isMany) \
- return new TWideMultiMapJoinWrapper<RIGHT_REQ, IS_TUPLE>(ctx.Mutables, \
- std::move(leftKeyConverters), dictType, std::move(outputRepresentations), \
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), wide, dict, width); \
- else \
- return new TWideMapJoinWrapper<false, RIGHT_REQ, IS_TUPLE>(ctx.Mutables, \
- std::move(leftKeyConverters), dictType, std::move(outputRepresentations), \
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), wide, dict, width); \
- } else \
- return new TMapJoinCoreFlowWrapper<KIND, RIGHT_REQ, IS_TUPLE>(ctx.Mutables, GetValueRepresentation(type), \
- std::move(leftKeyConverters), dictType, std::move(outputRepresentations), \
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), flow, dict); \
- } else { \
- return new TMapJoinCoreWrapper<KIND, RIGHT_REQ, IS_TUPLE>(ctx.Mutables, \
- std::move(leftKeyConverters), dictType, std::move(outputRepresentations), \
- std::move(leftKeyColumns), std::move(leftRenames), std::move(rightRenames), flow, dict); \
- } \
- #define MAKE_WRAPPER(IS_TUPLE) \
- switch (kind) { \
- case EJoinKind::Inner: \
- if (isMany) { \
- NEW_WRAPPER(ERightKind::Many, true, IS_TUPLE); \
- } else { \
- NEW_WRAPPER(ERightKind::Once, true, IS_TUPLE); \
- } \
- case EJoinKind::Left: \
- if (isMany) { \
- NEW_WRAPPER(ERightKind::Many, false, IS_TUPLE); \
- } else { \
- NEW_WRAPPER(ERightKind::Once, false, IS_TUPLE); \
- } \
- case EJoinKind::LeftOnly: \
- NEW_WRAPPER(ERightKind::None, false, IS_TUPLE); \
- case EJoinKind::LeftSemi: \
- NEW_WRAPPER(ERightKind::None, true, IS_TUPLE); \
- default: \
- ythrow yexception() << "Unsupported join kind"; \
- } \
- if (isTupleKey) {
- MAKE_WRAPPER(true)
- } else {
- MAKE_WRAPPER(false)
- }
- #undef MAKE_WRAPPER
- #undef NEW_WRAPPER
- }
- }
- }
|