#include "skiff.h" #include "skiff_validator.h" #include #include #include namespace NSkiff { //////////////////////////////////////////////////////////////////////////////// bool operator==(TInt128 lhs, TInt128 rhs) { return lhs.Low == rhs.Low && lhs.High == rhs.High; } bool operator!=(TInt128 lhs, TInt128 rhs) { return !(lhs == rhs); } bool operator==(TUint128 lhs, TUint128 rhs) { return lhs.Low == rhs.Low && lhs.High == rhs.High; } bool operator!=(TUint128 lhs, TUint128 rhs) { return !(lhs == rhs); } //////////////////////////////////////////////////////////////////////////////// TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying) : Underlying_(underlying) , Buffer_(512 * 1024) { } TUncheckedSkiffParser::TUncheckedSkiffParser(const std::shared_ptr& /*schema*/, IZeroCopyInput* underlying) : TUncheckedSkiffParser(underlying) { } i8 TUncheckedSkiffParser::ParseInt8() { return ParseSimple(); } i16 TUncheckedSkiffParser::ParseInt16() { return ParseSimple(); } i32 TUncheckedSkiffParser::ParseInt32() { return ParseSimple(); } i64 TUncheckedSkiffParser::ParseInt64() { return ParseSimple(); } ui8 TUncheckedSkiffParser::ParseUint8() { return ParseSimple(); } ui16 TUncheckedSkiffParser::ParseUint16() { return ParseSimple(); } ui32 TUncheckedSkiffParser::ParseUint32() { return ParseSimple(); } ui64 TUncheckedSkiffParser::ParseUint64() { return ParseSimple(); } TInt128 TUncheckedSkiffParser::ParseInt128() { auto low = ParseSimple(); auto high = ParseSimple(); return {low, high}; } TUint128 TUncheckedSkiffParser::ParseUint128() { auto low = ParseSimple(); auto high = ParseSimple(); return {low, high}; } double TUncheckedSkiffParser::ParseDouble() { return ParseSimple(); } bool TUncheckedSkiffParser::ParseBoolean() { ui8 result = ParseSimple(); if (result > 1) { ythrow TSkiffException() << "Invalid boolean value \"" << result << "\""; } return result; } TStringBuf TUncheckedSkiffParser::ParseString32() { ui32 len = ParseSimple(); const void* data = GetData(len); return TStringBuf(static_cast(data), len); } TStringBuf TUncheckedSkiffParser::ParseYson32() { return ParseString32(); } ui8 TUncheckedSkiffParser::ParseVariant8Tag() { return ParseSimple(); } ui16 TUncheckedSkiffParser::ParseVariant16Tag() { return ParseSimple(); } template T TUncheckedSkiffParser::ParseSimple() { return ReadUnaligned(GetData(sizeof(T))); } const void* TUncheckedSkiffParser::GetData(size_t size) { if (RemainingBytes() >= size) { const void* result = Position_; Advance(size); return result; } return GetDataViaBuffer(size); } const void* TUncheckedSkiffParser::GetDataViaBuffer(size_t size) { Buffer_.Clear(); Buffer_.Reserve(size); while (Buffer_.Size() < size) { size_t toCopy = Min(size - Buffer_.Size(), RemainingBytes()); Buffer_.Append(Position_, toCopy); Advance(toCopy); if (RemainingBytes() == 0) { RefillBuffer(); if (Exhausted_ && Buffer_.Size() < size) { ythrow TSkiffException() << "Premature end of stream while parsing Skiff"; } } } return Buffer_.Data(); } size_t TUncheckedSkiffParser::RemainingBytes() const { Y_ASSERT(End_ >= Position_); return End_ - Position_; } void TUncheckedSkiffParser::Advance(size_t size) { Y_ASSERT(size <= RemainingBytes()); Position_ += size; ReadBytesCount_ += size; } void TUncheckedSkiffParser::RefillBuffer() { size_t bufferSize = Underlying_->Next(&Position_); End_ = Position_ + bufferSize; if (bufferSize == 0) { Exhausted_ = true; } } bool TUncheckedSkiffParser::HasMoreData() { if (RemainingBytes() == 0 && !Exhausted_) { RefillBuffer(); } return !(RemainingBytes() == 0 && Exhausted_); } void TUncheckedSkiffParser::ValidateFinished() { } ui64 TUncheckedSkiffParser::GetReadBytesCount() const { return ReadBytesCount_; } //////////////////////////////////////////////////////////////////////////////// TCheckedSkiffParser::TCheckedSkiffParser(const std::shared_ptr& schema, IZeroCopyInput* stream) : Parser_(stream) , Validator_(std::make_unique(schema)) { } TCheckedSkiffParser::~TCheckedSkiffParser() = default; i8 TCheckedSkiffParser::ParseInt8() { Validator_->OnSimpleType(EWireType::Int8); return Parser_.ParseInt8(); } i16 TCheckedSkiffParser::ParseInt16() { Validator_->OnSimpleType(EWireType::Int16); return Parser_.ParseInt16(); } i32 TCheckedSkiffParser::ParseInt32() { Validator_->OnSimpleType(EWireType::Int32); return Parser_.ParseInt32(); } i64 TCheckedSkiffParser::ParseInt64() { Validator_->OnSimpleType(EWireType::Int64); return Parser_.ParseInt64(); } ui8 TCheckedSkiffParser::ParseUint8() { Validator_->OnSimpleType(EWireType::Uint8); return Parser_.ParseUint8(); } ui16 TCheckedSkiffParser::ParseUint16() { Validator_->OnSimpleType(EWireType::Uint16); return Parser_.ParseUint16(); } ui32 TCheckedSkiffParser::ParseUint32() { Validator_->OnSimpleType(EWireType::Uint32); return Parser_.ParseUint32(); } ui64 TCheckedSkiffParser::ParseUint64() { Validator_->OnSimpleType(EWireType::Uint64); return Parser_.ParseUint64(); } TInt128 TCheckedSkiffParser::ParseInt128() { Validator_->OnSimpleType(EWireType::Int128); return Parser_.ParseInt128(); } TUint128 TCheckedSkiffParser::ParseUint128() { Validator_->OnSimpleType(EWireType::Uint128); return Parser_.ParseUint128(); } double TCheckedSkiffParser::ParseDouble() { Validator_->OnSimpleType(EWireType::Double); return Parser_.ParseDouble(); } bool TCheckedSkiffParser::ParseBoolean() { Validator_->OnSimpleType(EWireType::Boolean); return Parser_.ParseBoolean(); } TStringBuf TCheckedSkiffParser::ParseString32() { Validator_->OnSimpleType(EWireType::String32); return Parser_.ParseString32(); } TStringBuf TCheckedSkiffParser::ParseYson32() { Validator_->OnSimpleType(EWireType::Yson32); return Parser_.ParseYson32(); } ui8 TCheckedSkiffParser::ParseVariant8Tag() { Validator_->BeforeVariant8Tag(); auto result = Parser_.ParseVariant8Tag(); Validator_->OnVariant8Tag(result); return result; } ui16 TCheckedSkiffParser::ParseVariant16Tag() { Validator_->BeforeVariant16Tag(); auto result = Parser_.ParseVariant16Tag(); Validator_->OnVariant16Tag(result); return result; } bool TCheckedSkiffParser::HasMoreData() { return Parser_.HasMoreData(); } void TCheckedSkiffParser::ValidateFinished() { Validator_->ValidateFinished(); Parser_.ValidateFinished(); } ui64 TCheckedSkiffParser::GetReadBytesCount() const { return Parser_.GetReadBytesCount(); } //////////////////////////////////////////////////////////////////////////////// TUncheckedSkiffWriter::TUncheckedSkiffWriter(IZeroCopyOutput* underlying) : Underlying_(underlying) { } TUncheckedSkiffWriter::TUncheckedSkiffWriter(IOutputStream* underlying) : BufferedOutput_(MakeHolder(underlying)) , Underlying_(BufferedOutput_.Get()) { } TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr& /*schema*/, IZeroCopyOutput* underlying) : TUncheckedSkiffWriter(underlying) { } TUncheckedSkiffWriter::TUncheckedSkiffWriter(const std::shared_ptr& /*schema*/, IOutputStream* underlying) : TUncheckedSkiffWriter(underlying) { } TUncheckedSkiffWriter::~TUncheckedSkiffWriter() { try { Flush(); } catch (...) { } } void TUncheckedSkiffWriter::WriteInt8(i8 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteInt16(i16 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteInt32(i32 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteInt64(i64 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteInt128(TInt128 value) { WriteSimple(value.Low); WriteSimple(value.High); } void TUncheckedSkiffWriter::WriteUint128(TUint128 value) { WriteSimple(value.Low); WriteSimple(value.High); } void TUncheckedSkiffWriter::WriteUint8(ui8 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteUint16(ui16 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteUint32(ui32 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteUint64(ui64 value) { WriteSimple(value); } void TUncheckedSkiffWriter::WriteDouble(double value) { return WriteSimple(value); } void TUncheckedSkiffWriter::WriteBoolean(bool value) { return WriteSimple(value ? 1 : 0); } void TUncheckedSkiffWriter::WriteString32(TStringBuf value) { WriteSimple(value.size()); Underlying_.Write(value.data(), value.size()); } void TUncheckedSkiffWriter::WriteYson32(TStringBuf value) { WriteSimple(value.size()); Underlying_.Write(value.data(), value.size()); } void TUncheckedSkiffWriter::WriteVariant8Tag(ui8 tag) { WriteSimple(tag); } void TUncheckedSkiffWriter::WriteVariant16Tag(ui16 tag) { WriteSimple(tag); } void TUncheckedSkiffWriter::Flush() { Underlying_.UndoRemaining(); if (BufferedOutput_) { BufferedOutput_->Flush(); } } template Y_FORCE_INLINE void TUncheckedSkiffWriter::WriteSimple(T value) { if constexpr (std::is_integral_v) { value = HostToLittle(value); Underlying_.Write(&value, sizeof(T)); } else { Underlying_.Write(&value, sizeof(T)); } } void TUncheckedSkiffWriter::Finish() { Flush(); } //////////////////////////////////////////////////////////////////////////////// TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr& schema, IZeroCopyOutput* underlying) : Writer_(underlying) , Validator_(std::make_unique(schema)) { } TCheckedSkiffWriter::TCheckedSkiffWriter(const std::shared_ptr& schema, IOutputStream* underlying) : Writer_(underlying) , Validator_(std::make_unique(schema)) { } TCheckedSkiffWriter::~TCheckedSkiffWriter() = default; void TCheckedSkiffWriter::WriteDouble(double value) { Validator_->OnSimpleType(EWireType::Double); Writer_.WriteDouble(value); } void TCheckedSkiffWriter::WriteBoolean(bool value) { Validator_->OnSimpleType(EWireType::Boolean); Writer_.WriteBoolean(value); } void TCheckedSkiffWriter::WriteInt8(i8 value) { Validator_->OnSimpleType(EWireType::Int8); Writer_.WriteInt8(value); } void TCheckedSkiffWriter::WriteInt16(i16 value) { Validator_->OnSimpleType(EWireType::Int16); Writer_.WriteInt16(value); } void TCheckedSkiffWriter::WriteInt32(i32 value) { Validator_->OnSimpleType(EWireType::Int32); Writer_.WriteInt32(value); } void TCheckedSkiffWriter::WriteInt64(i64 value) { Validator_->OnSimpleType(EWireType::Int64); Writer_.WriteInt64(value); } void TCheckedSkiffWriter::WriteUint8(ui8 value) { Validator_->OnSimpleType(EWireType::Uint8); Writer_.WriteUint8(value); } void TCheckedSkiffWriter::WriteUint16(ui16 value) { Validator_->OnSimpleType(EWireType::Uint16); Writer_.WriteUint16(value); } void TCheckedSkiffWriter::WriteUint32(ui32 value) { Validator_->OnSimpleType(EWireType::Uint32); Writer_.WriteUint32(value); } void TCheckedSkiffWriter::WriteUint64(ui64 value) { Validator_->OnSimpleType(EWireType::Uint64); Writer_.WriteUint64(value); } void TCheckedSkiffWriter::WriteInt128(TInt128 value) { Validator_->OnSimpleType(EWireType::Int128); Writer_.WriteInt128(value); } void TCheckedSkiffWriter::WriteUint128(TUint128 value) { Validator_->OnSimpleType(EWireType::Uint128); Writer_.WriteUint128(value); } void TCheckedSkiffWriter::WriteString32(TStringBuf value) { Validator_->OnSimpleType(EWireType::String32); Writer_.WriteString32(value); } void TCheckedSkiffWriter::WriteYson32(TStringBuf value) { Validator_->OnSimpleType(EWireType::Yson32); Writer_.WriteYson32(value); } void TCheckedSkiffWriter::WriteVariant8Tag(ui8 tag) { Validator_->OnVariant8Tag(tag); Writer_.WriteVariant8Tag(tag); } void TCheckedSkiffWriter::WriteVariant16Tag(ui16 tag) { Validator_->OnVariant16Tag(tag); Writer_.WriteVariant16Tag(tag); } void TCheckedSkiffWriter::Flush() { Writer_.Flush(); } void TCheckedSkiffWriter::Finish() { Validator_->ValidateFinished(); Writer_.Finish(); } //////////////////////////////////////////////////////////////////// } // namespace NSkiff