#include #include #include #include #include #include namespace { NYsonPull::TReader memory_reader(TStringBuf data, NYsonPull::EStreamType mode) { return NYsonPull::TReader( NYsonPull::NInput::FromMemory(data), mode); } template void expect_scalar(const NYsonPull::TScalar& scalar, T value) { UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{value}, scalar); } template <> void expect_scalar(const NYsonPull::TScalar& scalar, double value) { UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Float64, scalar.Type()); auto scalarValue = scalar.AsFloat64(); auto message = TStringBuilder() << "expected " << value << ", got " << scalarValue; if (std::isfinite(value)) { UNIT_ASSERT_C(std::isfinite(scalarValue), message); UNIT_ASSERT_DOUBLES_EQUAL(value, scalarValue, 1e-5); } else if (std::isnan(value)) { UNIT_ASSERT_C(std::isnan(scalarValue), message); } else if (value > 0) { UNIT_ASSERT_C(std::isinf(scalarValue) && (scalarValue > 0), message); } else { UNIT_ASSERT_C(std::isinf(scalarValue) && (scalarValue < 0), message); } } template void test_scalar(TStringBuf data, T value) { // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); auto reader = memory_reader(data, NYsonPull::EStreamType::Node); try { UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); { auto& event = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, event.Type()); expect_scalar(event.AsScalar(), value); } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); } catch (const std::exception& err) { UNIT_FAIL(err.what()); } } void consume(TStringBuf data, NYsonPull::EStreamType mode = NYsonPull::EStreamType::Node) { // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); auto input_range = NYsonPull::TStreamEventsRange( NYsonPull::NInput::FromMemory(data), mode); for (auto& event : input_range) { Y_UNUSED(event); } } #define ACCEPT(data) UNIT_ASSERT_NO_EXCEPTION(consume(data)) #define REJECT(data) UNIT_ASSERT_EXCEPTION(consume(data), NYsonPull::NException::TBadInput) #define ACCEPT2(data, mode) UNIT_ASSERT_NO_EXCEPTION(consume(data, mode)) #define REJECT2(data, mode) UNIT_ASSERT_EXCEPTION(consume(data, mode), NYsonPull::NException::TBadInput) } // anonymous namespace Y_UNIT_TEST_SUITE(Reader) { Y_UNIT_TEST(ScalarEntity) { test_scalar(TStringBuf("#"), NYsonPull::TScalar{}); } Y_UNIT_TEST(ScalarBoolean) { test_scalar(TStringBuf("%true"), true); test_scalar(TStringBuf("%false"), false); test_scalar(TStringBuf("\x05"sv), true); test_scalar(TStringBuf("\x04"sv), false); REJECT("%"); REJECT("%trueth"); REJECT("%tru"); REJECT("%falseth"); REJECT("%fals"); REJECT("%hithere"); } Y_UNIT_TEST(ScalarInt64) { test_scalar(TStringBuf("1"), i64{1}); test_scalar(TStringBuf("+1"), i64{1}); test_scalar(TStringBuf("100000"), i64{100000}); test_scalar(TStringBuf("+100000"), i64{100000}); test_scalar(TStringBuf("-100000"), i64{-100000}); test_scalar(TStringBuf("9223372036854775807"), i64{9223372036854775807}); test_scalar(TStringBuf("+9223372036854775807"), i64{9223372036854775807}); test_scalar(TStringBuf("\x02\x02"sv), i64{1}); test_scalar(TStringBuf("\x02\xc0\x9a\x0c"sv), i64{100000}); test_scalar(TStringBuf("\x02\xbf\x9a\x0c"sv), i64{-100000}); test_scalar(TStringBuf("\x02\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv), i64{9223372036854775807}); REJECT("1a2"); REJECT("1-1-1-1"); REJECT("1+0"); } Y_UNIT_TEST(SclarUInt64) { test_scalar(TStringBuf("1u"), ui64{1}); test_scalar(TStringBuf("+1u"), ui64{1}); test_scalar(TStringBuf("100000u"), ui64{100000}); test_scalar(TStringBuf("+100000u"), ui64{100000}); test_scalar(TStringBuf("9223372036854775807u"), ui64{9223372036854775807u}); test_scalar(TStringBuf("+9223372036854775807u"), ui64{9223372036854775807u}); test_scalar(TStringBuf("18446744073709551615u"), ui64{18446744073709551615u}); test_scalar(TStringBuf("+18446744073709551615u"), ui64{18446744073709551615u}); REJECT("1a2u"); REJECT("1-1-1-1u"); REJECT("1+0u"); // TODO: binary } Y_UNIT_TEST(ScalarFloat64) { test_scalar(TStringBuf("0.0"), double{0.0}); test_scalar(TStringBuf("+0.0"), double{0.0}); test_scalar(TStringBuf("+.0"), double{0.0}); test_scalar(TStringBuf("+.5"), double{0.5}); test_scalar(TStringBuf("-.5"), double{-0.5}); test_scalar(TStringBuf("1.0"), double{1.0}); test_scalar(TStringBuf("+1.0"), double{1.0}); test_scalar(TStringBuf("-1.0"), double{-1.0}); test_scalar(TStringBuf("1000.0"), double{1000.0}); test_scalar(TStringBuf("+1000.0"), double{1000.0}); test_scalar(TStringBuf("-1000.0"), double{-1000.0}); test_scalar(TStringBuf("1e12"), double{1e12}); test_scalar(TStringBuf("1e+12"), double{1e12}); test_scalar(TStringBuf("+1e+12"), double{1e12}); test_scalar(TStringBuf("-1e+12"), double{-1e12}); test_scalar(TStringBuf("1e-12"), double{1e-12}); test_scalar(TStringBuf("+1e-12"), double{1e-12}); test_scalar(TStringBuf("-1e-12"), double{-1e-12}); test_scalar(TStringBuf("\x03\x00\x00\x00\x00\x00\x00\x00\x00"sv), double{0.0}); test_scalar( TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf8\x7f"sv), double{std::numeric_limits::quiet_NaN()}); test_scalar( TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf0\x7f"sv), double{std::numeric_limits::infinity()}); test_scalar( TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf0\xff"sv), double{-std::numeric_limits::infinity()}); test_scalar( TStringBuf("%nan"), double{std::numeric_limits::quiet_NaN()}); test_scalar( TStringBuf("%inf"), double{std::numeric_limits::infinity()}); test_scalar( TStringBuf("%-inf"), double{-std::numeric_limits::infinity()}); REJECT("++0.0"); REJECT("++1.0"); REJECT("++.1"); REJECT("1.0.0"); //REJECT("1e+10000"); REJECT(TStringBuf("\x03\x00\x00\x00\x00\x00\x00\x00"sv)); // XXX: Questionable behaviour? ACCEPT("+.0"); ACCEPT("-.0"); // XXX: Rejected on Mac OS, accepted on Linux (?!) //REJECT(".0"); //REJECT(".5"); REJECT("%NaN"); REJECT("%+inf"); REJECT("%infinity"); REJECT("%na"); REJECT("%in"); REJECT("%-in"); } Y_UNIT_TEST(ScalarString) { test_scalar(TStringBuf(R"(foobar)"), TStringBuf("foobar")); test_scalar(TStringBuf(R"(foobar11)"), TStringBuf("foobar11")); test_scalar(TStringBuf(R"("foobar")"), TStringBuf("foobar")); // wat? "\x0cf" parsed as a single char? no way! test_scalar("\x01\x0c" "foobar"sv, TStringBuf("foobar")); REJECT(R"("foobar)"); REJECT("\x01\x0c" "fooba"sv); REJECT("\x01\x0d" "foobar"sv); // negative length } Y_UNIT_TEST(EmptyList) { auto reader = memory_reader("[]", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); REJECT("["); REJECT("]"); } Y_UNIT_TEST(EmptyMap) { auto reader = memory_reader("{}", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); REJECT("{"); REJECT("}"); } Y_UNIT_TEST(Sample) { auto reader = memory_reader( R"({"11"=11;"nothing"=#;"zero"=0.;"foo"="bar";"list"=[1;2;3]})", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("11"), e.AsString()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{11}}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("nothing"), e.AsString()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("zero"), e.AsString()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{0.0}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("foo"), e.AsString()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{TStringBuf("bar")}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("list"), e.AsString()); } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{1}}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{2}}, e.AsScalar()); } { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{3}}, e.AsScalar()); } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); } Y_UNIT_TEST(Accept) { ACCEPT("[]"); ACCEPT("{}"); ACCEPT("<>[]"); ACCEPT("<>{}"); ACCEPT("[{};{};{}]"); ACCEPT("[{};{};{};]"); ACCEPT("[<>{};<>{};<>{}]"); ACCEPT("[<>{};<>{};<>{};]"); ACCEPT("foo"); ACCEPT("[foo]"); ACCEPT("[foo;]"); ACCEPT("{foo=foo}"); ACCEPT("{foo=foo;}"); ACCEPT("<>{foo=foo}"); ACCEPT("{foo=foo}"); ACCEPT("{foo=foo}"); ACCEPT("{foo=[foo;foo]}"); } Y_UNIT_TEST(Reject) { REJECT("["); REJECT("{"); REJECT("<"); REJECT("[[}]"); REJECT("<>{]"); REJECT("[>]"); REJECT("<><>[]"); REJECT("[<>;<>]"); REJECT("{<>foo=foo}"); REJECT("{foo=<>}"); REJECT("{foo}"); REJECT(""); REJECT("<>"); REJECT("@"); } Y_UNIT_TEST(ReadPastEnd) { auto reader = memory_reader("#", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); UNIT_ASSERT_EXCEPTION(reader.NextEvent(), NYsonPull::NException::TBadInput); } Y_UNIT_TEST(BadInput) { // max_size < varint size < max_size auto t = TString("\x01\xff\xff\xff\xff\xff\xff\xff\xff"); auto reader = memory_reader(t, NYsonPull::EStreamType::Node); UNIT_ASSERT_EQUAL(reader.NextEvent().Type(), NYsonPull::EEventType::BeginStream); UNIT_ASSERT_EXCEPTION(reader.NextEvent(), NYsonPull::NException::TBadInput); } Y_UNIT_TEST(StreamType) { REJECT2("", NYsonPull::EStreamType::Node); ACCEPT2("", NYsonPull::EStreamType::ListFragment); ACCEPT2("", NYsonPull::EStreamType::MapFragment); ACCEPT2("[1]", NYsonPull::EStreamType::Node); ACCEPT2("[1]", NYsonPull::EStreamType::ListFragment); REJECT2("[1]", NYsonPull::EStreamType::MapFragment); ACCEPT2("[1]", NYsonPull::EStreamType::Node); ACCEPT2("[1]", NYsonPull::EStreamType::ListFragment); REJECT2("[1]", NYsonPull::EStreamType::MapFragment); ACCEPT2(" [1] \t \t ", NYsonPull::EStreamType::Node); ACCEPT2(" [1] \t \t ", NYsonPull::EStreamType::ListFragment); REJECT2(" [1] \t \t ", NYsonPull::EStreamType::MapFragment); REJECT2("[1];", NYsonPull::EStreamType::Node); ACCEPT2("[1];", NYsonPull::EStreamType::ListFragment); REJECT2("[1];", NYsonPull::EStreamType::MapFragment); REJECT2("[1]; foobar", NYsonPull::EStreamType::Node); ACCEPT2("[1]; foobar", NYsonPull::EStreamType::ListFragment); REJECT2("[1]; foobar", NYsonPull::EStreamType::MapFragment); REJECT2("a=[1]", NYsonPull::EStreamType::Node); REJECT2("a=[1]", NYsonPull::EStreamType::ListFragment); ACCEPT2("a=[1]", NYsonPull::EStreamType::MapFragment); REJECT2("a=[1]; ", NYsonPull::EStreamType::Node); REJECT2("a=[1]; ", NYsonPull::EStreamType::ListFragment); ACCEPT2("a=[1]; ", NYsonPull::EStreamType::MapFragment); REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::Node); REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::ListFragment); ACCEPT2("a=[1]; b=foobar", NYsonPull::EStreamType::MapFragment); } } // Y_UNIT_TEST_SUITE(Reader)