ydb/util/stream/ios_ut.cpp

#include "output.h"
#include "tokenizer.h"
#include "buffer.h"
#include "buffered.h"
#include "walk.h"

#include <library/cpp/testing/unittest/registar.h>

#include <util/string/cast.h>
#include <util/memory/tempbuf.h>
#include <util/charset/wide.h>

#include <string>
#include <iostream>

class TStreamsTest: public TTestBase {
    UNIT_TEST_SUITE(TStreamsTest);
    UNIT_TEST(TestGenericRead);
    UNIT_TEST(TestGenericWrite);
    UNIT_TEST(TestReadLine);
    UNIT_TEST(TestMemoryStream);
    UNIT_TEST(TestBufferedIO);
    UNIT_TEST(TestBufferStream);
    UNIT_TEST(TestStringStream);
    UNIT_TEST(TestWtrokaInput);
    UNIT_TEST(TestStrokaInput);
    UNIT_TEST(TestReadTo);
    UNIT_TEST(TestWtrokaOutput);
    UNIT_TEST(TestIStreamOperators);
    UNIT_TEST(TestWchar16Output);
    UNIT_TEST(TestWchar32Output);
    UNIT_TEST(TestUtf16StingOutputByChars);
    UNIT_TEST_SUITE_END();

public:
    void TestGenericRead();
    void TestGenericWrite();
    void TestReadLine();
    void TestMemoryStream();
    void TestBufferedIO();
    void TestBufferStream();
    void TestStringStream();
    void TestWtrokaInput();
    void TestStrokaInput();
    void TestWtrokaOutput();
    void TestIStreamOperators();
    void TestReadTo();
    void TestWchar16Output();
    void TestWchar32Output();
    void TestUtf16StingOutputByChars();
};

UNIT_TEST_SUITE_REGISTRATION(TStreamsTest);

void TStreamsTest::TestIStreamOperators() {
    TString data("first line\r\nsecond\t\xd1\x82\xd0\xb5\xd1\x81\xd1\x82 line\r\n 1 -4 59 4320000000009999999 c\n -1.5 1e-110");
    TStringInput si(data);

    TString l1;
    TString l2;
    TString l3;
    TUtf16String w1;
    TString l4;
    ui16 i1;
    i16 i2;
    i32 i3;
    ui64 i4;
    char c1;
    unsigned char c2;
    float f1;
    double f2;

    si >> l1 >> l2 >> l3 >> w1 >> l4 >> i1 >> i2 >> i3 >> i4 >> c1 >> c2 >> f1 >> f2;

    UNIT_ASSERT_EQUAL(l1, "first");
    UNIT_ASSERT_EQUAL(l2, "line");
    UNIT_ASSERT_EQUAL(l3, "second");
    UNIT_ASSERT_EQUAL(l4, "line");
    UNIT_ASSERT_EQUAL(i1, 1);
    UNIT_ASSERT_EQUAL(i2, -4);
    UNIT_ASSERT_EQUAL(i3, 59);
    UNIT_ASSERT_EQUAL(i4, 4320000000009999999ULL);
    UNIT_ASSERT_EQUAL(c1, 'c');
    UNIT_ASSERT_EQUAL(c2, '\n');
    UNIT_ASSERT_EQUAL(f1, -1.5);
    UNIT_ASSERT_EQUAL(f2, 1e-110);
}

void TStreamsTest::TestStringStream() {
    TStringStream s;

    s << "qw\r\n1234"
      << "\n"
      << 34;

    UNIT_ASSERT_EQUAL(s.ReadLine(), "qw");
    UNIT_ASSERT_EQUAL(s.ReadLine(), "1234");

    s << "\r\n"
      << 123.1;

    UNIT_ASSERT_EQUAL(s.ReadLine(), "34");
    UNIT_ASSERT_EQUAL(s.ReadLine(), "123.1");

    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1");

    // Test stream copying
    TStringStream sc = s;

    s << "-666-" << 13;
    sc << "-777-" << 0 << "JackPot";

    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
    UNIT_ASSERT_EQUAL(sc.Str(), "qw\r\n1234\n34\r\n123.1-777-0JackPot");

    TStringStream ss;
    ss = s;
    s << "... and some trash";
    UNIT_ASSERT_EQUAL(ss.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
}

void TStreamsTest::TestGenericRead() {
    TString s("1234567890");
    TStringInput si(s);
    char buf[1024];

    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 6);
    UNIT_ASSERT_EQUAL(memcmp(buf, "123456", 6), 0);
    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 4);
    UNIT_ASSERT_EQUAL(memcmp(buf, "7890", 4), 0);
}

void TStreamsTest::TestGenericWrite() {
    TString s;
    TStringOutput so(s);

    so.Write("123456", 6);
    so.Write("7890", 4);

    UNIT_ASSERT_EQUAL(s, "1234567890");
}

void TStreamsTest::TestReadLine() {
    TString data("1234\r\n5678\nqw");
    TStringInput si(data);

    UNIT_ASSERT_EQUAL(si.ReadLine(), "1234");
    UNIT_ASSERT_EQUAL(si.ReadLine(), "5678");
    UNIT_ASSERT_EQUAL(si.ReadLine(), "qw");
}

void TStreamsTest::TestMemoryStream() {
    char buf[1024];
    TMemoryOutput mo(buf, sizeof(buf));
    bool ehandled = false;

    try {
        for (size_t i = 0; i < sizeof(buf) + 1; ++i) {
            mo.Write(i % 127);
        }
    } catch (...) {
        ehandled = true;
    }

    UNIT_ASSERT_EQUAL(ehandled, true);

    for (size_t i = 0; i < sizeof(buf); ++i) {
        UNIT_ASSERT_EQUAL(buf[i], (char)(i % 127));
    }
}

class TMyStringOutput: public IOutputStream {
public:
    inline TMyStringOutput(TString& s, size_t buflen) noexcept
        : S_(s)
        , BufLen_(buflen)
    {
    }

    ~TMyStringOutput() override = default;

    void DoWrite(const void* data, size_t len) override {
        S_.Write(data, len);
        UNIT_ASSERT(len < BufLen_ || ((len % BufLen_) == 0));
    }

    void DoWriteV(const TPart* p, size_t count) override {
        TString s;

        for (size_t i = 0; i < count; ++i) {
            s.append((const char*)p[i].buf, p[i].len);
        }

        DoWrite(s.data(), s.size());
    }

private:
    TStringOutput S_;
    const size_t BufLen_;
};

void TStreamsTest::TestBufferedIO() {
    TString s;

    {
        const size_t buflen = 7;
        TBuffered<TMyStringOutput> bo(buflen, s, buflen);

        for (size_t i = 0; i < 1000; ++i) {
            TString str(" ");
            str += ToString(i % 10);

            bo.Write(str.data(), str.size());
        }

        bo.Finish();
    }

    UNIT_ASSERT_EQUAL(s.size(), 2000);

    {
        const size_t buflen = 11;
        TBuffered<TStringInput> bi(buflen, s);

        for (size_t i = 0; i < 1000; ++i) {
            TString str(" ");
            str += ToString(i % 10);

            char buf[3];

            UNIT_ASSERT_EQUAL(bi.Load(buf, 2), 2);

            buf[2] = 0;

            UNIT_ASSERT_EQUAL(str, buf);
        }
    }

    s.clear();

    {
        const size_t buflen = 13;
        TBuffered<TMyStringOutput> bo(buflen, s, buflen);
        TString f = "1234567890";

        for (size_t i = 0; i < 10; ++i) {
            f += f;
        }

        for (size_t i = 0; i < 1000; ++i) {
            bo.Write(f.data(), i);
        }

        bo.Finish();
    }
}

void TStreamsTest::TestBufferStream() {
    TBufferStream stream;
    TString s = "test";

    stream.Write(s.data(), s.size());
    char buf[5];
    size_t bytesRead = stream.Read(buf, 4);
    UNIT_ASSERT_EQUAL(4, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp(s.data(), buf, 4));

    stream.Write(s.data(), s.size());
    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(2, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp("te", buf, 2));

    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(2, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp("st", buf, 2));

    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(0, bytesRead);
}

namespace {
    class TStringListInput: public IWalkInput {
    public:
        TStringListInput(const TVector<TString>& data)
            : Data_(data)
            , Index_(0)
        {
        }

    protected:
        size_t DoUnboundedNext(const void** ptr) override {
            if (Index_ >= Data_.size()) {
                return 0;
            }

            const TString& string = Data_[Index_++];

            *ptr = string.data();
            return string.size();
        }

    private:
        const TVector<TString>& Data_;
        size_t Index_;
    };

    const char Text[] =
        // UTF8 encoded "one \ntwo\r\nthree\n\tfour\nfive\n" in russian and ...
        "один \n"
        "два\r\n"
        "три\n"
        "\tчетыре\n"
        "пять\n"
        // ... additional test cases
        "\r\n"
        "\n\r" // this char goes to the front of the next string
        "one two\n"
        "123\r\n"
        "\t\r ";

    const char* Expected[] = {
        // UTF8 encoded "one ", "two", "three", "\tfour", "five" in russian and ...
        "один ",
        "два",
        "три",
        "\tчетыре",
        "пять",
        // ... additional test cases
        "",
        "",
        "\rone two",
        "123",
        "\t\r "};
    void TestStreamReadTo1(IInputStream& input, const char* comment) {
        TString tmp;
        input.ReadTo(tmp, 'c');
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "111a222b333", comment);

        char tmp2;
        input.Read(&tmp2, 1);
        UNIT_ASSERT_VALUES_EQUAL_C(tmp2, '4', comment);

        input.ReadTo(tmp, '6');
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "44d555e", comment);

        tmp = input.ReadAll();
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "66f", comment);
    }

    void TestStreamReadTo2(IInputStream& input, const char* comment) {
        TString s;
        size_t i = 0;
        while (input.ReadLine(s)) {
            UNIT_ASSERT_C(i < Y_ARRAY_SIZE(Expected), comment);
            UNIT_ASSERT_VALUES_EQUAL_C(s, Expected[i], comment);
            ++i;
        }
    }

    void TestStreamReadTo3(IInputStream& input, const char* comment) {
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadLine(), "111a222b333c444d555e666f", comment);
    }

    void TestStreamReadTo4(IInputStream& input, const char* comment) {
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "one", comment);
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "two", comment);
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "three", comment);
    }

    void TestStrokaInput(IInputStream& input, const char* comment) {
        TString line;
        ui32 i = 0;
        TInstant start = Now();
        while (input.ReadLine(line)) {
            ++i;
        }
        Cout << comment << ":" << (Now() - start).SecondsFloat() << Endl;
        UNIT_ASSERT_VALUES_EQUAL(i, 100000);
    }

    template <class T>
    void TestStreamReadTo(const TString& text, T test) {
        TStringInput is(text);
        test(is, "TStringInput");
        TMemoryInput mi(text.data(), text.size());
        test(mi, "TMemoryInput");
        TBuffer b(text.data(), text.size());
        TBufferInput bi(b);
        test(bi, "TBufferInput");
        TStringInput slave(text);
        TBufferedInput bdi(&slave);
        test(bdi, "TBufferedInput");
        TVector<TString> lst(1, text);
        TStringListInput sli(lst);
        test(sli, "IWalkInput");
    }
}

void TStreamsTest::TestReadTo() {
    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo1);
    TestStreamReadTo(Text, TestStreamReadTo2);
    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo3);
    TString withZero = "one";
    withZero.append('\0').append("two").append('\0').append("three");
    TestStreamReadTo(withZero, TestStreamReadTo4);
}

void TStreamsTest::TestStrokaInput() {
    TString s;
    for (ui32 i = 0; i < 100000; ++i) {
        TVector<char> d(i % 1000, 'a');
        s.append(d.data(), d.size());
        s.append('\n');
    }
    TestStreamReadTo(s, ::TestStrokaInput);
}

void TStreamsTest::TestWtrokaInput() {
    const TString s(Text);
    TStringInput is(s);
    TUtf16String w;
    size_t i = 0;

    while (is.ReadLine(w)) {
        UNIT_ASSERT(i < Y_ARRAY_SIZE(Expected));
        UNIT_ASSERT_VALUES_EQUAL(w, UTF8ToWide(Expected[i]));

        ++i;
    }
}

void TStreamsTest::TestWtrokaOutput() {
    TString s;
    TStringOutput os(s);
    const size_t n = sizeof(Expected) / sizeof(Expected[0]);

    for (size_t i = 0; i < n; ++i) {
        TUtf16String w = UTF8ToWide(Expected[i]);

        os << w;

        if (i == 1 || i == 5 || i == 8) {
            os << '\r';
        }

        if (i < n - 1) {
            os << '\n';
        }
    }

    UNIT_ASSERT(s == Text);
}

void TStreamsTest::TestWchar16Output() {
    TString s;
    TStringOutput os(s);
    os << wchar16(97); // latin a
    os << u'\u044E';   // cyrillic ю
    os << u'я';
    os << wchar16(0xD801); // high surrogate is printed as replacement character U+FFFD
    os << u'b';

    UNIT_ASSERT_VALUES_EQUAL(s, "aюя"
                                "\xEF\xBF\xBD"
                                "b");
}

void TStreamsTest::TestWchar32Output() {
    TString s;
    TStringOutput os(s);
    os << wchar32(97); // latin a
    os << U'\u044E';   // cyrillic ю
    os << U'я';
    os << U'\U0001F600'; // grinning face
    os << u'b';

    UNIT_ASSERT_VALUES_EQUAL(s, "aюя"
                                "\xF0\x9F\x98\x80"
                                "b");
}

void TStreamsTest::TestUtf16StingOutputByChars() {
    TString s = "\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8";
    TUtf16String w = UTF8ToWide(s);

    UNIT_ASSERT_VALUES_EQUAL(w.size(), 10);

    TStringStream stream0;
    stream0 << w;
    UNIT_ASSERT_VALUES_EQUAL(stream0.Str(), s);

    TStringStream stream1;
    for (size_t i = 0; i < 10; i++) {
        stream1 << w[i];
    }
    UNIT_ASSERT_VALUES_EQUAL(stream1.Str(), s);
}