mkql_computation_node_pack_ut.cpp 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952
  1. #include "mkql_computation_node_pack.h"
  2. #include "mkql_computation_node_holders.h"
  3. #include "mkql_block_builder.h"
  4. #include "mkql_block_reader.h"
  5. #include <yql/essentials/minikql/mkql_function_registry.h>
  6. #include <yql/essentials/minikql/mkql_node.h>
  7. #include <yql/essentials/minikql/mkql_program_builder.h>
  8. #include <yql/essentials/minikql/mkql_string_util.h>
  9. #include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h>
  10. #include <yql/essentials/public/udf/arrow/memory_pool.h>
  11. #include <yql/essentials/public/udf/arrow/util.h>
  12. #include <library/cpp/random_provider/random_provider.h>
  13. #include <yql/essentials/minikql/aligned_page_pool.h>
  14. #include <yql/essentials/public/udf/udf_value.h>
  15. #include <library/cpp/testing/unittest/registar.h>
  16. #include <util/generic/ylimits.h>
  17. #include <util/generic/xrange.h>
  18. #include <util/generic/maybe.h>
  19. #include <util/string/cast.h>
  20. #include <util/string/builder.h>
  21. #include <util/system/hp_timer.h>
  22. #include <util/system/sanitizers.h>
  23. namespace NKikimr {
  24. namespace NMiniKQL {
  25. using NYql::TChunkedBuffer;
  26. #ifdef WITH_VALGRIND
  27. constexpr static size_t PERFORMANCE_COUNT = 0x1000;
  28. #elif defined(NDEBUG)
  29. constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x4000000, 0x1000);
  30. #else
  31. constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x1000000, 0x1000);
  32. #endif
  33. template<bool Fast, bool Transport>
  34. struct TPackerTraits;
  35. template<bool Fast>
  36. struct TPackerTraits<Fast, false> {
  37. using TPackerType = TValuePackerGeneric<Fast>;
  38. };
  39. template<bool Fast>
  40. struct TPackerTraits<Fast, true> {
  41. using TPackerType = TValuePackerTransport<Fast>;
  42. };
  43. using NDetails::TChunkedInputBuffer;
  44. template<bool Fast, bool Transport>
  45. class TMiniKQLComputationNodePackTest: public TTestBase {
  46. using TValuePackerType = typename TPackerTraits<Fast, Transport>::TPackerType;
  47. protected:
  48. TMiniKQLComputationNodePackTest()
  49. : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry()))
  50. , RandomProvider(CreateDefaultRandomProvider())
  51. , Alloc(__LOCATION__)
  52. , Env(Alloc)
  53. , PgmBuilder(Env, *FunctionRegistry)
  54. , MemInfo("Memory")
  55. , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
  56. , ArrowPool_(NYql::NUdf::GetYqlMemoryPool())
  57. {
  58. }
  59. void TestNumericTypes() {
  60. TestNumericType<bool>(NUdf::TDataType<bool>::Id);
  61. TestNumericType<ui8>(NUdf::TDataType<ui8>::Id);
  62. TestNumericType<i32>(NUdf::TDataType<i32>::Id);
  63. TestNumericType<i64>(NUdf::TDataType<i64>::Id);
  64. TestNumericType<ui32>(NUdf::TDataType<ui32>::Id);
  65. TestNumericType<ui64>(NUdf::TDataType<ui64>::Id);
  66. TestNumericType<float>(NUdf::TDataType<float>::Id);
  67. TestNumericType<double>(NUdf::TDataType<double>::Id);
  68. }
  69. void TestOptionalNumericTypes() {
  70. TestOptionalNumericType<bool>(NUdf::TDataType<bool>::Id);
  71. TestOptionalNumericType<ui8>(NUdf::TDataType<ui8>::Id);
  72. TestOptionalNumericType<i32>(NUdf::TDataType<i32>::Id);
  73. TestOptionalNumericType<i64>(NUdf::TDataType<i64>::Id);
  74. TestOptionalNumericType<ui32>(NUdf::TDataType<ui32>::Id);
  75. TestOptionalNumericType<ui64>(NUdf::TDataType<ui64>::Id);
  76. TestOptionalNumericType<float>(NUdf::TDataType<float>::Id);
  77. TestOptionalNumericType<double>(NUdf::TDataType<double>::Id);
  78. }
  79. void TestStringTypes() {
  80. TestStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
  81. TestStringType(NUdf::TDataType<char*>::Id);
  82. TestStringType(NUdf::TDataType<NUdf::TYson>::Id);
  83. TestStringType(NUdf::TDataType<NUdf::TJson>::Id);
  84. }
  85. void TestOptionalStringTypes() {
  86. TestOptionalStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
  87. TestOptionalStringType(NUdf::TDataType<char*>::Id);
  88. TestOptionalStringType(NUdf::TDataType<NUdf::TYson>::Id);
  89. TestOptionalStringType(NUdf::TDataType<NUdf::TJson>::Id);
  90. }
  91. void TestListType() {
  92. TDefaultListRepresentation listValues;
  93. for (ui32 val: xrange(0, 10)) {
  94. listValues = listValues.Append(NUdf::TUnboxedValuePod(val));
  95. }
  96. TType* listType = PgmBuilder.NewListType(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
  97. const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
  98. const auto uValue = TestPackUnpack(listType, value, "Type:List(ui32)");
  99. UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
  100. ui32 i = 0;
  101. const auto iter = uValue.GetListIterator();
  102. for (NUdf::TUnboxedValue item; iter.Next(item); ++i) {
  103. UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i);
  104. }
  105. UNIT_ASSERT_VALUES_EQUAL(i, 10);
  106. }
  107. void TestListOfOptionalsType() {
  108. TDefaultListRepresentation listValues;
  109. for (ui32 i: xrange(0, 10)) {
  110. NUdf::TUnboxedValue uVal = NUdf::TUnboxedValuePod();
  111. if (i % 2) {
  112. uVal = MakeString(TString(i * 2, '0' + i));
  113. }
  114. listValues = listValues.Append(std::move(uVal));
  115. }
  116. TType* listType = PgmBuilder.NewListType(PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)));
  117. const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
  118. const auto uValue = TestPackUnpack(listType, value, "Type:List(Optional(utf8))");
  119. UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
  120. ui32 i = 0;
  121. const auto iter = uValue.GetListIterator();
  122. for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) {
  123. if (i % 2) {
  124. UNIT_ASSERT(uVal);
  125. UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), TString(i * 2, '0' + i));
  126. } else {
  127. UNIT_ASSERT(!uVal);
  128. }
  129. }
  130. UNIT_ASSERT_VALUES_EQUAL(i, 10);
  131. }
  132. void TestTupleType() {
  133. std::vector<TType*> tupleElemenTypes;
  134. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
  135. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
  136. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
  137. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
  138. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
  139. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
  140. TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
  141. TUnboxedValueVector tupleElemens;
  142. tupleElemens.push_back(MakeString("01234567890123456789"));
  143. tupleElemens.push_back(MakeString("01234567890"));
  144. tupleElemens.push_back(NUdf::TUnboxedValuePod());
  145. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  146. tupleElemens.push_back(NUdf::TUnboxedValuePod());
  147. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  148. const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens);
  149. const auto uValue = TestPackUnpack(tupleType, value, "Type:Tuple");
  150. {
  151. auto e = uValue.GetElement(0);
  152. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
  153. }
  154. {
  155. auto e = uValue.GetElement(1);
  156. UNIT_ASSERT(e);
  157. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
  158. }
  159. {
  160. auto e = uValue.GetElement(2);
  161. UNIT_ASSERT(!e);
  162. }
  163. {
  164. auto e = uValue.GetElement(3);
  165. UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
  166. }
  167. {
  168. auto e = uValue.GetElement(4);
  169. UNIT_ASSERT(!e);
  170. }
  171. {
  172. auto e = uValue.GetElement(5);
  173. UNIT_ASSERT(e);
  174. UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
  175. }
  176. }
  177. void TestStructType() {
  178. const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
  179. {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
  180. {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
  181. {"c", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
  182. {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)},
  183. {"e", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)},
  184. {"f", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)}
  185. };
  186. TType* structType = PgmBuilder.NewStructType(structElemenTypes);
  187. TUnboxedValueVector structElemens;
  188. structElemens.push_back(MakeString("01234567890123456789"));
  189. structElemens.push_back(MakeString("01234567890"));
  190. structElemens.push_back(NUdf::TUnboxedValuePod());
  191. structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  192. structElemens.push_back(NUdf::TUnboxedValuePod());
  193. structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  194. const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(structElemens);
  195. const auto uValue = TestPackUnpack(structType, value, "Type:Struct");
  196. {
  197. auto e = uValue.GetElement(0);
  198. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
  199. }
  200. {
  201. auto e = uValue.GetElement(1);
  202. UNIT_ASSERT(e);
  203. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
  204. }
  205. {
  206. auto e = uValue.GetElement(2);
  207. UNIT_ASSERT(!e);
  208. }
  209. {
  210. auto e = uValue.GetElement(3);
  211. UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
  212. }
  213. {
  214. auto e = uValue.GetElement(4);
  215. UNIT_ASSERT(!e);
  216. }
  217. {
  218. auto e = uValue.GetElement(5);
  219. UNIT_ASSERT(e);
  220. UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
  221. }
  222. }
  223. void TestOptionalType() {
  224. TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
  225. type = PgmBuilder.NewOptionalType(type);
  226. type = PgmBuilder.NewOptionalType(type);
  227. type = PgmBuilder.NewOptionalType(type);
  228. NUdf::TUnboxedValue uValue = NUdf::TUnboxedValuePod();
  229. uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:null");
  230. UNIT_ASSERT(!uValue);
  231. uValue = NUdf::TUnboxedValuePod(ui64(123));
  232. uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(opt(123)))");
  233. UNIT_ASSERT_VALUES_EQUAL(uValue.Get<ui64>(), 123);
  234. uValue = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional();
  235. uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(null))");
  236. UNIT_ASSERT(uValue);
  237. uValue = uValue.GetOptionalValue();
  238. UNIT_ASSERT(uValue);
  239. uValue = uValue.GetOptionalValue();
  240. UNIT_ASSERT(!uValue);
  241. uValue = NUdf::TUnboxedValuePod().MakeOptional();
  242. uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(null)");
  243. UNIT_ASSERT(uValue);
  244. uValue = uValue.GetOptionalValue();
  245. UNIT_ASSERT(!uValue);
  246. }
  247. void TestDictType() {
  248. TType* keyType = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
  249. TType* payloadType = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
  250. TType* dictType = PgmBuilder.NewDictType(keyType, payloadType, false);
  251. TValuesDictHashSingleFixedMap<ui32> map;
  252. map[4] = NUdf::TUnboxedValuePod::Embedded("4");
  253. map[10] = NUdf::TUnboxedValuePod::Embedded("10");
  254. map[1] = NUdf::TUnboxedValuePod::Embedded("1");
  255. const NUdf::TUnboxedValue value = HolderFactory.CreateDirectHashedSingleFixedMapHolder<ui32, false>(std::move(map), std::nullopt);
  256. const auto uValue = TestPackUnpack(dictType, value, "Type:Dict");
  257. UNIT_ASSERT_VALUES_EQUAL(uValue.GetDictLength(), 3);
  258. const auto it = uValue.GetDictIterator();
  259. for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) {
  260. UNIT_ASSERT_VALUES_EQUAL(ToString(key.template Get<ui32>()), std::string_view(payload.AsStringRef()));
  261. }
  262. }
  263. void TestVariantTypeOverStruct() {
  264. const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
  265. {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
  266. {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
  267. {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}
  268. };
  269. TType* structType = PgmBuilder.NewStructType(structElemenTypes);
  270. TestVariantTypeImpl(PgmBuilder.NewVariantType(structType));
  271. }
  272. void TestVariantTypeOverTuple() {
  273. std::vector<TType*> tupleElemenTypes;
  274. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
  275. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
  276. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
  277. TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
  278. TestVariantTypeImpl(PgmBuilder.NewVariantType(tupleType));
  279. }
  280. void ValidateEmbeddedLength(TChunkedBuffer buf, const TString& info) {
  281. size_t size = buf.Size();
  282. TChunkedInputBuffer chunked(std::move(buf));
  283. return ValidateEmbeddedLength(chunked, size, info);
  284. }
  285. void ValidateEmbeddedLength(TStringBuf buf, const TString& info) {
  286. TChunkedInputBuffer chunked(buf);
  287. return ValidateEmbeddedLength(chunked, buf.size(), info);
  288. }
  289. void ValidateEmbeddedLength(TChunkedInputBuffer& buf, size_t totalSize, const TString& info) {
  290. if constexpr (!Fast) {
  291. if (totalSize > 8) {
  292. ui32 len = NDetails::GetRawData<ui32>(buf);
  293. UNIT_ASSERT_VALUES_EQUAL_C(len + 4, totalSize, info);
  294. } else {
  295. ui32 len = NDetails::GetRawData<ui8>(buf);
  296. UNIT_ASSERT_VALUES_EQUAL_C(((len & 0x0f) >> 1) + 1, totalSize, info);
  297. }
  298. }
  299. }
  300. void TestPackPerformance(TType* type, const NUdf::TUnboxedValuePod& uValue)
  301. {
  302. TValuePackerType packer(false, type);
  303. const THPTimer timer;
  304. for (size_t i = 0U; i < PERFORMANCE_COUNT; ++i)
  305. packer.Pack(uValue);
  306. Cerr << timer.Passed() << Endl;
  307. }
  308. NUdf::TUnboxedValue TestPackUnpack(TValuePackerType& packer, const NUdf::TUnboxedValuePod& uValue,
  309. const TString& additionalMsg, const std::optional<ui32>& expectedLength = {})
  310. {
  311. auto packedValue = packer.Pack(uValue);
  312. if constexpr (Transport) {
  313. if (expectedLength) {
  314. UNIT_ASSERT_VALUES_EQUAL_C(packedValue.Size(), *expectedLength, additionalMsg);
  315. }
  316. ValidateEmbeddedLength(packedValue, additionalMsg);
  317. return packer.Unpack(std::move(packedValue), HolderFactory);
  318. } else {
  319. if (expectedLength) {
  320. UNIT_ASSERT_VALUES_EQUAL_C(packedValue.Size(), *expectedLength, additionalMsg);
  321. }
  322. ValidateEmbeddedLength(packedValue, additionalMsg);
  323. return packer.Unpack(packedValue, HolderFactory);
  324. }
  325. }
  326. NUdf::TUnboxedValue TestPackUnpack(TType* type, const NUdf::TUnboxedValuePod& uValue, const TString& additionalMsg,
  327. const std::optional<ui32>& expectedLength = {})
  328. {
  329. TValuePackerType packer(false, type);
  330. return TestPackUnpack(packer, uValue, additionalMsg, expectedLength);
  331. }
  332. template <typename T>
  333. void TestNumericValue(T value, TValuePackerType& packer, const TString& typeDesc) {
  334. TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
  335. auto uValue = TestPackUnpack(packer, NUdf::TUnboxedValuePod(value), additionalMsg);
  336. UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), value, additionalMsg);
  337. }
  338. template <typename T>
  339. void TestNumericType(NUdf::TDataTypeId schemeType) {
  340. TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
  341. TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
  342. TestNumericValue<T>(Max<T>(), packer, typeDesc);
  343. TestNumericValue<T>(Min<T>(), packer, typeDesc);
  344. TestNumericValue<T>(T(0), packer, typeDesc);
  345. TestNumericValue<T>(T(1), packer, typeDesc);
  346. }
  347. template <typename T>
  348. void TestOptionalNumericValue(std::optional<T> value, TValuePackerType& packer, const TString& typeDesc,
  349. const std::optional<ui32>& expectedLength = {})
  350. {
  351. TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? ToString(*value) : TString("null"));
  352. const auto v = value ? NUdf::TUnboxedValuePod(*value) : NUdf::TUnboxedValuePod();
  353. const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
  354. if (value) {
  355. UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), *value, additionalMsg);
  356. } else {
  357. UNIT_ASSERT_C(!uValue, additionalMsg);
  358. }
  359. }
  360. template <typename T>
  361. void TestOptionalNumericType(NUdf::TDataTypeId schemeType) {
  362. TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
  363. TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)));
  364. TestOptionalNumericValue<T>(std::optional<T>(Max<T>()), packer, typeDesc);
  365. TestOptionalNumericValue<T>(std::optional<T>(Min<T>()), packer, typeDesc);
  366. TestOptionalNumericValue<T>(std::optional<T>(), packer, typeDesc, 1);
  367. TestOptionalNumericValue<T>(std::optional<T>(0), packer, typeDesc);
  368. TestOptionalNumericValue<T>(std::optional<T>(1), packer, typeDesc);
  369. }
  370. void TestStringValue(const std::string_view& value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) {
  371. TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
  372. const auto v = NUdf::TUnboxedValue(MakeString(value));
  373. const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
  374. UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), value, additionalMsg);
  375. }
  376. void TestStringType(NUdf::TDataTypeId schemeType) {
  377. TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
  378. TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
  379. TestStringValue("0123456789012345678901234567890123456789", packer, typeDesc, 40 + 4);
  380. TestStringValue("[]", packer, typeDesc, Fast ? (2 + 4) : (2 + 1));
  381. TestStringValue("1234567", packer, typeDesc, Fast ? (7 + 4) : (7 + 1));
  382. TestStringValue("", packer, typeDesc, Fast ? (0 + 4) : (0 + 1));
  383. TestStringValue("12345678", packer, typeDesc, 8 + 4);
  384. TString hugeString(12345678, 'X');
  385. TestStringValue(hugeString, packer, typeDesc, hugeString.size() + 4);
  386. }
  387. void TestUuidType() {
  388. auto schemeType = NUdf::TDataType<NUdf::TUuid>::Id;
  389. TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
  390. TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
  391. TestStringValue("0123456789abcdef", packer, typeDesc, Fast ? 16 : (16 + 4));
  392. }
  393. void TestOptionalStringValue(std::optional<std::string_view> value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) {
  394. TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? *value : TString("null"));
  395. const auto v = value ? NUdf::TUnboxedValue(MakeString(*value)) : NUdf::TUnboxedValue();
  396. const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
  397. if (value) {
  398. UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), *value, additionalMsg);
  399. } else {
  400. UNIT_ASSERT_C(!uValue, additionalMsg);
  401. }
  402. }
  403. void TestOptionalStringType(NUdf::TDataTypeId schemeType) {
  404. TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
  405. TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)));
  406. TestOptionalStringValue("0123456789012345678901234567890123456789", packer, typeDesc, Fast ? (40 + 4 + 1) : (40 + 4));
  407. TestOptionalStringValue(std::nullopt, packer, typeDesc, 1);
  408. TestOptionalStringValue("[]", packer, typeDesc, Fast ? (2 + 4 + 1) : (2 + 1));
  409. TestOptionalStringValue("1234567", packer, typeDesc, Fast ? (7 + 4 + 1) : (7 + 1));
  410. TestOptionalStringValue("", packer, typeDesc, Fast ? (0 + 4 + 1) : 1);
  411. TestOptionalStringValue("12345678", packer, typeDesc, Fast ? (8 + 4 + 1) : (8 + 4));
  412. }
  413. void TestVariantTypeImpl(TType* variantType) {
  414. TString descr = TStringBuilder() << "Type:Variant("
  415. << static_cast<TVariantType*>(variantType)->GetUnderlyingType()->GetKindAsStr() << ')';
  416. {
  417. const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(MakeString("01234567890123456789"), 0);
  418. const auto uValue = TestPackUnpack(variantType, value, descr);
  419. UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 0);
  420. auto e = uValue.GetVariantItem();
  421. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
  422. }
  423. {
  424. const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(), 1);
  425. const auto uValue = TestPackUnpack(variantType, value, descr);
  426. UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 1);
  427. auto e = uValue.GetVariantItem();
  428. UNIT_ASSERT(!e);
  429. }
  430. {
  431. const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui64(12345)), 2);
  432. const auto uValue = TestPackUnpack(variantType, value, descr);
  433. UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 2);
  434. auto e = uValue.GetVariantItem();
  435. UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345ull);
  436. }
  437. }
  438. NUdf::TUnboxedValue MakeTupleValue(TType*& tupleType, bool forPerf = false) {
  439. std::vector<TType*> tupleElemenTypes;
  440. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
  441. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
  442. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
  443. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
  444. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
  445. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
  446. if (!forPerf) {
  447. tupleElemenTypes.push_back(PgmBuilder.NewDecimalType(16, 8));
  448. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(PgmBuilder.NewDecimalType(22, 3)));
  449. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(PgmBuilder.NewDecimalType(35, 2)));
  450. tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(PgmBuilder.NewDecimalType(29, 0)));
  451. }
  452. tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
  453. auto inf = NYql::NDecimal::FromString("inf", 16, 8);
  454. auto dec1 = NYql::NDecimal::FromString("12345.673", 22, 3);
  455. auto dec2 = NYql::NDecimal::FromString("-9781555555.99", 35, 2);
  456. TUnboxedValueVector tupleElemens;
  457. tupleElemens.push_back(MakeString("01234567890123456789"));
  458. tupleElemens.push_back(MakeString("01234567890"));
  459. tupleElemens.push_back(NUdf::TUnboxedValuePod());
  460. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  461. tupleElemens.push_back(NUdf::TUnboxedValuePod());
  462. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
  463. if (!forPerf) {
  464. tupleElemens.push_back(NUdf::TUnboxedValuePod(inf));
  465. tupleElemens.push_back(NUdf::TUnboxedValuePod(dec1));
  466. tupleElemens.push_back(NUdf::TUnboxedValuePod(dec2));
  467. tupleElemens.push_back(NUdf::TUnboxedValuePod());
  468. }
  469. return HolderFactory.VectorAsArray(tupleElemens);
  470. }
  471. void ValidateTupleValue(const NUdf::TUnboxedValue& value, bool forPerf = false) {
  472. using NYql::NUdf::TStringValue;
  473. UNIT_ASSERT(value.IsBoxed());
  474. auto e0 = value.GetElement(0);
  475. auto e1 = value.GetElement(1);
  476. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e0.AsStringRef()), "01234567890123456789");
  477. UNIT_ASSERT_VALUES_EQUAL(std::string_view(e1.AsStringRef()), "01234567890");
  478. UNIT_ASSERT(!value.GetElement(2).HasValue());
  479. UNIT_ASSERT_VALUES_EQUAL(value.GetElement(3).Get<ui64>(), 12345);
  480. UNIT_ASSERT(!value.GetElement(4).HasValue());
  481. UNIT_ASSERT_VALUES_EQUAL(value.GetElement(5).Get<ui64>(), 12345);
  482. if (!forPerf) {
  483. UNIT_ASSERT_VALUES_EQUAL(std::string_view(NYql::NDecimal::ToString(value.GetElement(6).GetInt128(), 16, 8)), "inf");
  484. UNIT_ASSERT_VALUES_EQUAL(std::string_view(NYql::NDecimal::ToString(value.GetElement(7).GetInt128(), 22, 3)), "12345.673");
  485. UNIT_ASSERT_VALUES_EQUAL(std::string_view(NYql::NDecimal::ToString(value.GetElement(8).GetInt128(), 35, 2)), "-9781555555.99");
  486. UNIT_ASSERT(!value.GetElement(9).HasValue());
  487. }
  488. }
  489. void TestTuplePackPerformance() {
  490. TType* tupleType;
  491. const auto value = MakeTupleValue(tupleType, true);
  492. TestPackPerformance(tupleType, value);
  493. }
  494. void TestPairPackPerformance() {
  495. std::vector<TType*> tupleElemenTypes;
  496. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
  497. tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
  498. TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
  499. TUnboxedValueVector tupleElemens;
  500. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(12345)));
  501. tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(67890)));
  502. const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens);
  503. TestPackPerformance(tupleType, value);
  504. }
  505. void TestShortStringPackPerformance() {
  506. const auto v = NUdf::TUnboxedValuePod::Embedded("01234");
  507. TType* type = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
  508. TestPackPerformance(type, v);
  509. }
  510. void TestIntegerPackPerformance() {
  511. const auto& v = NUdf::TUnboxedValuePod(ui64("123456789ULL"));
  512. TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
  513. TestPackPerformance(type, v);
  514. }
  515. void TestRopeSplit() {
  516. if constexpr (Transport) {
  517. TType* tupleType;
  518. const auto value = MakeTupleValue(tupleType);
  519. TValuePackerType packer(false, tupleType);
  520. auto buffer = packer.Pack(value);
  521. TString packed;
  522. TStringOutput sout(packed);
  523. buffer.CopyTo(sout);
  524. TStringBuf packedBuf(packed);
  525. if constexpr (Fast) {
  526. UNIT_ASSERT_VALUES_EQUAL(packed.size(), 73);
  527. } else {
  528. UNIT_ASSERT_VALUES_EQUAL(packed.size(), 54);
  529. }
  530. for (size_t chunk = 1; chunk < packed.size(); ++chunk) {
  531. TStringBuf first = packedBuf.substr(0, chunk);
  532. TStringBuf second = packedBuf.substr(chunk);
  533. TChunkedBuffer result(first, {});
  534. result.Append(second, {});
  535. UNIT_ASSERT_VALUES_EQUAL(result.Size(), packed.size());
  536. UNIT_ASSERT(result.Size() != result.ContigousSize());
  537. ValidateTupleValue(packer.Unpack(std::move(result), HolderFactory));
  538. }
  539. }
  540. }
  541. void TestIncrementalPacking() {
  542. if constexpr (Transport) {
  543. auto itemType = PgmBuilder.NewDataType(NUdf::TDataType<char *>::Id);
  544. auto listType = PgmBuilder.NewListType(itemType);
  545. TValuePackerType packer(false, itemType);
  546. TValuePackerType listPacker(false, listType);
  547. TStringBuf str = "01234567890ABCDEFG";
  548. size_t count = 500000;
  549. for (size_t i = 0; i < count; ++i) {
  550. NUdf::TUnboxedValue item(MakeString(str));
  551. packer.AddItem(item);
  552. }
  553. auto serialized = packer.Finish();
  554. auto listObj = listPacker.Unpack(TChunkedBuffer(serialized), HolderFactory);
  555. UNIT_ASSERT_VALUES_EQUAL(listObj.GetListLength(), count);
  556. const auto iter = listObj.GetListIterator();
  557. for (NUdf::TUnboxedValue uVal; iter.Next(uVal);) {
  558. UNIT_ASSERT(uVal);
  559. UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), str);
  560. }
  561. TUnboxedValueBatch items;
  562. packer.UnpackBatch(std::move(serialized), HolderFactory, items);
  563. UNIT_ASSERT_VALUES_EQUAL(items.RowCount(), count);
  564. items.ForEachRow([&](const NUdf::TUnboxedValue& value) {
  565. UNIT_ASSERT(value);
  566. UNIT_ASSERT_VALUES_EQUAL(std::string_view(value.AsStringRef()), str);
  567. });
  568. }
  569. }
  570. void DoTestBlockPacking(ui64 offset, ui64 len, bool legacyStruct) {
  571. if constexpr (Transport) {
  572. auto strType = PgmBuilder.NewDataType(NUdf::TDataType<char*>::Id);
  573. auto ui32Type = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
  574. auto ui64Type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
  575. auto optStrType = PgmBuilder.NewOptionalType(strType);
  576. auto optUi32Type = PgmBuilder.NewOptionalType(ui32Type);
  577. auto tupleOptUi32StrType = PgmBuilder.NewTupleType({ optUi32Type, strType });
  578. auto optTupleOptUi32StrType = PgmBuilder.NewOptionalType(tupleOptUi32StrType);
  579. auto blockUi32Type = PgmBuilder.NewBlockType(ui32Type, TBlockType::EShape::Many);
  580. auto blockOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Many);
  581. auto scalarOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Scalar);
  582. auto blockOptTupleOptUi32StrType = PgmBuilder.NewBlockType(optTupleOptUi32StrType, TBlockType::EShape::Many);
  583. auto scalarUi64Type = PgmBuilder.NewBlockType(ui64Type, TBlockType::EShape::Scalar);
  584. auto tzDateType = PgmBuilder.NewDataType(NUdf::EDataSlot::TzDate);
  585. auto blockTzDateType = PgmBuilder.NewBlockType(tzDateType, TBlockType::EShape::Many);
  586. auto rowType =
  587. legacyStruct
  588. ? PgmBuilder.NewStructType({
  589. {"A", blockUi32Type},
  590. {"B", blockOptStrType},
  591. {"_yql_block_length", scalarUi64Type},
  592. {"a", scalarOptStrType},
  593. {"b", blockOptTupleOptUi32StrType},
  594. {"c", blockTzDateType}
  595. })
  596. : PgmBuilder.NewMultiType(
  597. {blockUi32Type, blockOptStrType, scalarOptStrType,
  598. blockOptTupleOptUi32StrType, blockTzDateType, scalarUi64Type});
  599. ui64 blockLen = 1000;
  600. UNIT_ASSERT_LE(offset + len, blockLen);
  601. auto builder1 = MakeArrayBuilder(TTypeInfoHelper(), ui32Type, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(ui32Type)), nullptr);
  602. auto builder2 = MakeArrayBuilder(TTypeInfoHelper(), optStrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optStrType)), nullptr);
  603. auto builder3 = MakeArrayBuilder(TTypeInfoHelper(), optTupleOptUi32StrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optTupleOptUi32StrType)), nullptr);
  604. auto builder4 = MakeArrayBuilder(TTypeInfoHelper(), tzDateType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(tzDateType)), nullptr);
  605. for (ui32 i = 0; i < blockLen; ++i) {
  606. TBlockItem b1(i);
  607. builder1->Add(b1);
  608. TString a = "a string " + ToString(i);
  609. TBlockItem b2 = (i % 2) ? TBlockItem(a) : TBlockItem();
  610. builder2->Add(b2);
  611. TBlockItem b3items[] = { (i % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(a) };
  612. TBlockItem b3 = (i % 7) ? TBlockItem(b3items) : TBlockItem();
  613. builder3->Add(b3);
  614. TBlockItem tzDate {i};
  615. tzDate.SetTimezoneId(i % 100);
  616. builder4->Add(tzDate);
  617. }
  618. std::string_view testScalarString = "foobar";
  619. auto strbuf = std::make_shared<arrow::Buffer>((const ui8*)testScalarString.data(), testScalarString.size());
  620. TVector<arrow::Datum> datums;
  621. if (legacyStruct) {
  622. datums.emplace_back(builder1->Build(true));
  623. datums.emplace_back(builder2->Build(true));
  624. datums.emplace_back(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(blockLen)));
  625. datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf)));
  626. datums.emplace_back(builder3->Build(true));
  627. datums.emplace_back(builder4->Build(true));
  628. } else {
  629. datums.emplace_back(builder1->Build(true));
  630. datums.emplace_back(builder2->Build(true));
  631. datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf)));
  632. datums.emplace_back(builder3->Build(true));
  633. datums.emplace_back(builder4->Build(true));
  634. datums.emplace_back(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(blockLen)));
  635. }
  636. if (offset != 0 || len != blockLen) {
  637. for (auto& datum : datums) {
  638. if (datum.is_array()) {
  639. datum = NYql::NUdf::DeepSlice(datum.array(), offset, len);
  640. }
  641. }
  642. }
  643. TUnboxedValueVector columns;
  644. for (auto& datum : datums) {
  645. columns.emplace_back(HolderFactory.CreateArrowBlock(std::move(datum)));
  646. }
  647. TValuePackerType packer(false, rowType, ArrowPool_);
  648. if (legacyStruct) {
  649. TUnboxedValueVector columnsCopy = columns;
  650. NUdf::TUnboxedValue row = HolderFactory.VectorAsArray(columnsCopy);
  651. packer.AddItem(row);
  652. } else {
  653. packer.AddWideItem(columns.data(), columns.size());
  654. }
  655. TChunkedBuffer packed = packer.Finish();
  656. TUnboxedValueBatch unpacked(rowType);
  657. packer.UnpackBatch(std::move(packed), HolderFactory, unpacked);
  658. UNIT_ASSERT_VALUES_EQUAL(unpacked.RowCount(), 1);
  659. TUnboxedValueVector unpackedColumns;
  660. if (legacyStruct) {
  661. auto elements = unpacked.Head()->GetElements();
  662. unpackedColumns.insert(unpackedColumns.end(), elements, elements + columns.size());
  663. } else {
  664. unpacked.ForEachRowWide([&](const NYql::NUdf::TUnboxedValue* values, ui32 count) {
  665. unpackedColumns.insert(unpackedColumns.end(), values, values + count);
  666. });
  667. }
  668. UNIT_ASSERT_VALUES_EQUAL(unpackedColumns.size(), columns.size());
  669. if (legacyStruct) {
  670. UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns[2]).GetDatum().scalar_as<arrow::UInt64Scalar>().value, blockLen);
  671. UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns[3]).GetDatum().scalar_as<arrow::BinaryScalar>().value->ToString(), testScalarString);
  672. } else {
  673. UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns.back()).GetDatum().scalar_as<arrow::UInt64Scalar>().value, blockLen);
  674. UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns[2]).GetDatum().scalar_as<arrow::BinaryScalar>().value->ToString(), testScalarString);
  675. }
  676. auto reader1 = MakeBlockReader(TTypeInfoHelper(), ui32Type);
  677. auto reader2 = MakeBlockReader(TTypeInfoHelper(), optStrType);
  678. auto reader3 = MakeBlockReader(TTypeInfoHelper(), optTupleOptUi32StrType);
  679. auto reader4 = MakeBlockReader(TTypeInfoHelper(), tzDateType);
  680. for (ui32 i = offset; i < len; ++i) {
  681. TBlockItem b1 = reader1->GetItem(*TArrowBlock::From(unpackedColumns[0]).GetDatum().array(), i - offset);
  682. UNIT_ASSERT_VALUES_EQUAL(b1.As<ui32>(), i);
  683. TString a = "a string " + ToString(i);
  684. TBlockItem b2 = reader2->GetItem(*TArrowBlock::From(unpackedColumns[1]).GetDatum().array(), i - offset);
  685. if (i % 2) {
  686. UNIT_ASSERT_VALUES_EQUAL(std::string_view(b2.AsStringRef()), a);
  687. } else {
  688. UNIT_ASSERT(!b2);
  689. }
  690. TBlockItem b3 = reader3->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 4 : 3]).GetDatum().array(), i - offset);
  691. if (i % 7) {
  692. auto elements = b3.GetElements();
  693. if (i % 2) {
  694. UNIT_ASSERT_VALUES_EQUAL(elements[0].As<ui32>(), i);
  695. } else {
  696. UNIT_ASSERT(!elements[0]);
  697. }
  698. UNIT_ASSERT_VALUES_EQUAL(std::string_view(elements[1].AsStringRef()), a);
  699. } else {
  700. UNIT_ASSERT(!b3);
  701. }
  702. TBlockItem b4 = reader4->GetItem(*TArrowBlock::From(unpackedColumns[legacyStruct ? 5 : 4]).GetDatum().array(), i - offset);
  703. UNIT_ASSERT(b4.Get<ui16>() == i);
  704. UNIT_ASSERT(b4.GetTimezoneId() == (i % 100));
  705. }
  706. }
  707. }
  708. void TestBlockPacking() {
  709. DoTestBlockPacking(0, 1000, false);
  710. }
  711. void TestBlockPackingSliced() {
  712. DoTestBlockPacking(19, 623, false);
  713. }
  714. void TestLegacyBlockPacking() {
  715. DoTestBlockPacking(0, 1000, true);
  716. }
  717. void TestLegacyBlockPackingSliced() {
  718. DoTestBlockPacking(19, 623, true);
  719. }
  720. private:
  721. TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry;
  722. TIntrusivePtr<IRandomProvider> RandomProvider;
  723. TScopedAlloc Alloc;
  724. TTypeEnvironment Env;
  725. TProgramBuilder PgmBuilder;
  726. TMemoryUsageInfo MemInfo;
  727. THolderFactory HolderFactory;
  728. arrow::MemoryPool* const ArrowPool_;
  729. };
  730. class TMiniKQLComputationNodeGenericPackTest: public TMiniKQLComputationNodePackTest<false, false> {
  731. UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericPackTest);
  732. UNIT_TEST(TestNumericTypes);
  733. UNIT_TEST(TestOptionalNumericTypes);
  734. UNIT_TEST(TestStringTypes);
  735. UNIT_TEST(TestUuidType);
  736. UNIT_TEST(TestOptionalStringTypes);
  737. UNIT_TEST(TestListType);
  738. UNIT_TEST(TestListOfOptionalsType);
  739. UNIT_TEST(TestTupleType);
  740. UNIT_TEST(TestStructType);
  741. UNIT_TEST(TestOptionalType);
  742. UNIT_TEST(TestDictType);
  743. UNIT_TEST(TestVariantTypeOverStruct);
  744. UNIT_TEST(TestVariantTypeOverTuple);
  745. UNIT_TEST(TestIntegerPackPerformance);
  746. UNIT_TEST(TestShortStringPackPerformance);
  747. UNIT_TEST(TestPairPackPerformance);
  748. UNIT_TEST(TestTuplePackPerformance);
  749. UNIT_TEST_SUITE_END();
  750. };
  751. class TMiniKQLComputationNodeGenericFastPackTest: public TMiniKQLComputationNodePackTest<true, false> {
  752. UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericFastPackTest);
  753. UNIT_TEST(TestNumericTypes);
  754. UNIT_TEST(TestOptionalNumericTypes);
  755. UNIT_TEST(TestStringTypes);
  756. UNIT_TEST(TestUuidType);
  757. UNIT_TEST(TestOptionalStringTypes);
  758. UNIT_TEST(TestListType);
  759. UNIT_TEST(TestListOfOptionalsType);
  760. UNIT_TEST(TestTupleType);
  761. UNIT_TEST(TestStructType);
  762. UNIT_TEST(TestOptionalType);
  763. UNIT_TEST(TestDictType);
  764. UNIT_TEST(TestVariantTypeOverStruct);
  765. UNIT_TEST(TestVariantTypeOverTuple);
  766. UNIT_TEST(TestIntegerPackPerformance);
  767. UNIT_TEST(TestShortStringPackPerformance);
  768. UNIT_TEST(TestPairPackPerformance);
  769. UNIT_TEST(TestTuplePackPerformance);
  770. UNIT_TEST_SUITE_END();
  771. };
  772. class TMiniKQLComputationNodeTransportPackTest: public TMiniKQLComputationNodePackTest<false, true> {
  773. UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportPackTest);
  774. UNIT_TEST(TestNumericTypes);
  775. UNIT_TEST(TestOptionalNumericTypes);
  776. UNIT_TEST(TestStringTypes);
  777. UNIT_TEST(TestUuidType);
  778. UNIT_TEST(TestOptionalStringTypes);
  779. UNIT_TEST(TestListType);
  780. UNIT_TEST(TestListOfOptionalsType);
  781. UNIT_TEST(TestTupleType);
  782. UNIT_TEST(TestStructType);
  783. UNIT_TEST(TestOptionalType);
  784. UNIT_TEST(TestDictType);
  785. UNIT_TEST(TestVariantTypeOverStruct);
  786. UNIT_TEST(TestVariantTypeOverTuple);
  787. UNIT_TEST(TestIntegerPackPerformance);
  788. UNIT_TEST(TestShortStringPackPerformance);
  789. UNIT_TEST(TestPairPackPerformance);
  790. UNIT_TEST(TestTuplePackPerformance);
  791. UNIT_TEST(TestRopeSplit);
  792. UNIT_TEST(TestIncrementalPacking);
  793. UNIT_TEST(TestBlockPacking);
  794. UNIT_TEST(TestBlockPackingSliced);
  795. UNIT_TEST(TestLegacyBlockPacking);
  796. UNIT_TEST(TestLegacyBlockPackingSliced);
  797. UNIT_TEST_SUITE_END();
  798. };
  799. class TMiniKQLComputationNodeTransportFastPackTest: public TMiniKQLComputationNodePackTest<true, true> {
  800. UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportFastPackTest);
  801. UNIT_TEST(TestNumericTypes);
  802. UNIT_TEST(TestOptionalNumericTypes);
  803. UNIT_TEST(TestStringTypes);
  804. UNIT_TEST(TestUuidType);
  805. UNIT_TEST(TestOptionalStringTypes);
  806. UNIT_TEST(TestListType);
  807. UNIT_TEST(TestListOfOptionalsType);
  808. UNIT_TEST(TestTupleType);
  809. UNIT_TEST(TestStructType);
  810. UNIT_TEST(TestOptionalType);
  811. UNIT_TEST(TestDictType);
  812. UNIT_TEST(TestVariantTypeOverStruct);
  813. UNIT_TEST(TestVariantTypeOverTuple);
  814. UNIT_TEST(TestIntegerPackPerformance);
  815. UNIT_TEST(TestShortStringPackPerformance);
  816. UNIT_TEST(TestPairPackPerformance);
  817. UNIT_TEST(TestTuplePackPerformance);
  818. UNIT_TEST(TestRopeSplit);
  819. UNIT_TEST(TestIncrementalPacking);
  820. UNIT_TEST(TestBlockPacking);
  821. UNIT_TEST(TestBlockPackingSliced);
  822. UNIT_TEST(TestLegacyBlockPacking);
  823. UNIT_TEST(TestLegacyBlockPackingSliced);
  824. UNIT_TEST_SUITE_END();
  825. };
  826. UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericPackTest);
  827. UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericFastPackTest);
  828. UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportPackTest);
  829. UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportFastPackTest);
  830. }
  831. }