#include #include #include #include #include "chunked_helpers.h" /// Data for TChunkedHelpersTest::TestGeneralVector struct TPodStruct { int x; float y; TPodStruct(int _x = 0, float _y = 0) : x(_x) , y(_y) { } }; /// And its serialization template <> struct TSaveLoadVectorNonPodElement { typedef TPodStruct TItem; static inline void Save(IOutputStream* out, const TItem& item) { TSerializer::Save(out, item.x); TSerializer::Save(out, item.y); } static inline void Load(IInputStream* in, TItem& item, size_t elementSize) { Y_ASSERT(elementSize == sizeof(TItem)); TSerializer::Load(in, item.x); TSerializer::Load(in, item.y); } }; class TChunkedHelpersTest: public TTestBase { UNIT_TEST_SUITE(TChunkedHelpersTest); UNIT_TEST(TestHash) UNIT_TEST(TestGeneralVector) UNIT_TEST(TestStrings); UNIT_TEST(TestNamedChunkedData); UNIT_TEST_SUITE_END(); public: void TestHash() { { TBufferStream stream; { TPlainHashWriter writer; writer.Add(5, 7); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TPlainHash reader(temp); ui16 value = 0; UNIT_ASSERT(reader.Find(5, &value)); UNIT_ASSERT_EQUAL(7, value); UNIT_ASSERT(!reader.Find(6, &value)); } } { TBufferStream stream; int v = 1; wchar16 k = 'a'; { TPlainHashWriter writer; writer.Add(k, &v); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TPlainHash reader(temp); void* value = nullptr; UNIT_ASSERT(reader.Find(k, &value)); UNIT_ASSERT_EQUAL((int*)value, &v); } } } void TestGeneralVector() { { /// ui32 const size_t N = 3; TBufferStream stream; { TGeneralVectorWriter writer; for (size_t i = 0; i < N; ++i) writer.PushBack(i); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { ui32 value; reader.Get(i, value); UNIT_ASSERT_EQUAL(value, i); UNIT_ASSERT_EQUAL(reader.At(i), i); } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(ui32)); } } { /// TString const size_t N = 4; TBufferStream stream; { TGeneralVectorWriter writer; for (size_t i = 0; i < N; ++i) writer.PushBack(ToString(i)); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { TString value; reader.Get(i, value); UNIT_ASSERT_EQUAL(value, ToString(i)); UNIT_ASSERT_EQUAL(reader.Get(i), ToString(i)); } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * 2); } } { /// some other struct typedef TPodStruct TItem; const size_t N = 2; TBufferStream stream; { TGeneralVectorWriter writer; writer.PushBack(TItem(1, 2)); writer.PushBack(TItem(3, 4)); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); TItem value; reader.Get(0, value); UNIT_ASSERT(value.x == 1 && value.y == 2.0); reader.Get(1, value); UNIT_ASSERT(value.x == 3 && value.y == 4.0); UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * sizeof(TItem)); } } { /// pointer const size_t N = 3; TVector data_holder(N); int* a = &(data_holder[0]); TBufferStream stream; { TGeneralVectorWriter writer; for (size_t i = 0; i < N; ++i) { a[i] = i; writer.PushBack(a + i); } writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { int* value; reader.Get(i, value); UNIT_ASSERT_EQUAL(value, a + i); UNIT_ASSERT_EQUAL(reader.At(i), a + i); } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*)); } } { /// std::pair typedef std::pair TItem; const size_t N = 3; TBufferStream stream; { TGeneralVectorWriter writer; for (size_t i = 0; i < N; ++i) writer.PushBack(TItem(i, i)); writer.Save(stream); } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { TItem value; reader.Get(i, value); UNIT_ASSERT_EQUAL(value, TItem(i, i)); } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem)); } } } void TestStrings() { const TString FILENAME = "chunked_helpers_test.bin"; TTempFileHandle file(FILENAME.c_str()); { TFixedBufferFileOutput fOut(FILENAME); TStringsVectorWriter stringsWriter; stringsWriter.PushBack(""); stringsWriter.PushBack("test"); TChunkedDataWriter writer(fOut); WriteBlock(writer, stringsWriter); writer.WriteFooter(); } { TBlob fIn = TBlob::FromFileSingleThreaded(FILENAME); TStringsVector vct(GetBlock(fIn, 0)); UNIT_ASSERT_EQUAL(vct.Get(0), ""); UNIT_ASSERT_EQUAL(vct.Get(1), "test"); bool wasException = false; try { vct.Get(2); } catch (...) { wasException = true; } UNIT_ASSERT(wasException); } } void TestNamedChunkedData() { const TString filename = MakeTempName(nullptr, "named_chunked_data_test"); TTempFile file(filename); { TFixedBufferFileOutput fOut(filename); TNamedChunkedDataWriter writer(fOut); writer.NewBlock("alpha"); writer.Write("123456"); writer.NewBlock(); writer.Write("anonymous"); writer.NewBlock("omega"); writer.Write("12345678901234567"); writer.WriteFooter(); } { TBlob mf = TBlob::FromFileSingleThreaded(filename); TNamedChunkedDataReader reader(mf); UNIT_ASSERT(reader.GetBlocksCount() == 3); UNIT_ASSERT(reader.HasBlock("alpha")); UNIT_ASSERT(reader.HasBlock("omega")); UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(0), "alpha"); UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(1), ""); UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(2), "omega"); UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("alpha"), 6); // padding not included UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("omega"), 17); UNIT_ASSERT(memcmp(reader.GetBlockByName("alpha"), "123456", 6) == 0); UNIT_ASSERT(memcmp(reader.GetBlock(1), "anonymous", 9) == 0); UNIT_ASSERT(memcmp(reader.GetBlockByName("omega"), "12345678901234567", 17) == 0); } } }; UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest); class TChunkedDataTest: public TTestBase { private: UNIT_TEST_SUITE(TChunkedDataTest); UNIT_TEST(Test) UNIT_TEST(TestEmpty) UNIT_TEST_SUITE_END(); void Test() { TBuffer buffer; { TBufferOutput out(buffer); TChunkedDataWriter writer(out); writer.NewBlock(); writer << "test"; writer.NewBlock(); writer << 4; writer.NewBlock(); writer.NewBlock(); writer << 1; writer << 2; writer.WriteFooter(); } { TBlob blob = TBlob::FromBufferSingleThreaded(buffer); TChunkedDataReader data(blob); // printf("%d\n", (int)data.GetBlockLen(3)); UNIT_ASSERT_EQUAL(4, data.GetBlockLen(0)); UNIT_ASSERT_EQUAL(1, data.GetBlockLen(1)); UNIT_ASSERT_EQUAL(0, data.GetBlockLen(2)); UNIT_ASSERT_EQUAL(2, data.GetBlockLen(3)); } } void TestEmpty() { TBuffer buffer; { TBufferOutput out(buffer); TChunkedDataWriter writer(out); writer.NewBlock(); writer.NewBlock(); writer.WriteFooter(); } { TBlob blob = TBlob::FromBufferSingleThreaded(buffer); TChunkedDataReader data(blob); // printf("%d\n", (int)data.GetBlockLen(1)); UNIT_ASSERT_EQUAL(0, data.GetBlockLen(0)); UNIT_ASSERT_EQUAL(0, data.GetBlockLen(1)); } } }; UNIT_TEST_SUITE_REGISTRATION(TChunkedDataTest);