#pragma once #include #include #include #include #include #include #include #include #include #ifndef __NVCC__ // cuda is compiled in C++14 mode at the time #include #include #endif template class TSerializeTypeTraits { public: /* * pointer types cannot be serialized as POD-type */ enum { IsSerializablePod = TTypeTraits::IsPod && !std::is_pointer::value }; }; struct TSerializeException: public yexception { }; struct TLoadEOF: public TSerializeException { }; template static inline void Save(IOutputStream* out, const T& t); template static inline void SaveArray(IOutputStream* out, const T* t, size_t len); template static inline void Load(IInputStream* in, T& t); template static inline void LoadArray(IInputStream* in, T* t, size_t len); template static inline void Load(IInputStream* in, T& t, TStorage& pool); template static inline void LoadArray(IInputStream* in, T* t, size_t len, TStorage& pool); template static inline void SavePodType(IOutputStream* rh, const T& t) { rh->Write(&t, sizeof(T)); } namespace NPrivate { [[noreturn]] void ThrowLoadEOFException(size_t typeSize, size_t realSize, TStringBuf structName); [[noreturn]] void ThrowUnexpectedVariantTagException(ui8 tagIndex); } template static inline void LoadPodType(IInputStream* rh, T& t) { const size_t res = rh->Load(&t, sizeof(T)); if (Y_UNLIKELY(res != sizeof(T))) { ::NPrivate::ThrowLoadEOFException(sizeof(T), res, TStringBuf("pod type")); } } template static inline void SavePodArray(IOutputStream* rh, const T* arr, size_t count) { rh->Write(arr, sizeof(T) * count); } template static inline void LoadPodArray(IInputStream* rh, T* arr, size_t count) { const size_t len = sizeof(T) * count; const size_t res = rh->Load(arr, len); if (Y_UNLIKELY(res != len)) { ::NPrivate::ThrowLoadEOFException(len, res, TStringBuf("pod array")); } } template static inline void SaveIterRange(IOutputStream* rh, It b, It e) { while (b != e) { ::Save(rh, *b++); } } template static inline void LoadIterRange(IInputStream* rh, It b, It e) { while (b != e) { ::Load(rh, *b++); } } template static inline void LoadIterRange(IInputStream* rh, It b, It e, TStorage& pool) { while (b != e) { ::Load(rh, *b++, pool); } } template struct TSerializerTakingIntoAccountThePodType { static inline void Save(IOutputStream* out, const T& t) { ::SavePodType(out, t); } static inline void Load(IInputStream* in, T& t) { ::LoadPodType(in, t); } template static inline void Load(IInputStream* in, T& t, TStorage& /*pool*/) { ::LoadPodType(in, t); } static inline void SaveArray(IOutputStream* out, const T* t, size_t len) { ::SavePodArray(out, t, len); } static inline void LoadArray(IInputStream* in, T* t, size_t len) { ::LoadPodArray(in, t, len); } }; namespace NHasSaveLoad { Y_HAS_MEMBER(SaveLoad); } template struct TSerializerMethodSelector; template struct TSerializerMethodSelector::value>> { static inline void Save(IOutputStream* out, const T& t) { //assume Save clause do not change t (const_cast(t)).SaveLoad(out); } static inline void Load(IInputStream* in, T& t) { t.SaveLoad(in); } template static inline void Load(IInputStream* in, T& t, TStorage& pool) { t.SaveLoad(in, pool); } }; template struct TSerializerMethodSelector::value>> { static inline void Save(IOutputStream* out, const T& t) { t.Save(out); } static inline void Load(IInputStream* in, T& t) { t.Load(in); } template static inline void Load(IInputStream* in, T& t, TStorage& pool) { t.Load(in, pool); } }; template struct TSerializerTakingIntoAccountThePodType: public TSerializerMethodSelector { static inline void SaveArray(IOutputStream* out, const T* t, size_t len) { ::SaveIterRange(out, t, t + len); } static inline void LoadArray(IInputStream* in, T* t, size_t len) { ::LoadIterRange(in, t, t + len); } template static inline void LoadArray(IInputStream* in, T* t, size_t len, TStorage& pool) { ::LoadIterRange(in, t, t + len, pool); } }; template struct TRangeSerialize { static inline void Save(IOutputStream* rh, It b, It e) { SaveArray(rh, b, e - b); } static inline void Load(IInputStream* rh, It b, It e) { LoadArray(rh, b, e - b); } template static inline void Load(IInputStream* rh, It b, It e, TStorage& pool) { LoadArray(rh, b, e - b, pool); } }; template struct TRangeSerialize { static inline void Save(IOutputStream* rh, It b, It e) { SaveIterRange(rh, b, e); } static inline void Load(IInputStream* rh, It b, It e) { LoadIterRange(rh, b, e); } template static inline void Load(IInputStream* rh, It b, It e, TStorage& pool) { LoadIterRange(rh, b, e, pool); } }; template static inline void SaveRange(IOutputStream* rh, It b, It e) { TRangeSerialize::value>::Save(rh, b, e); } template static inline void LoadRange(IInputStream* rh, It b, It e) { TRangeSerialize::value>::Load(rh, b, e); } template static inline void LoadRange(IInputStream* rh, It b, It e, TStorage& pool) { TRangeSerialize::value>::Load(rh, b, e, pool); } template class TSerializer: public TSerializerTakingIntoAccountThePodType::IsSerializablePod> { }; template class TArraySerializer: public TSerializerTakingIntoAccountThePodType::IsSerializablePod> { }; template static inline void Save(IOutputStream* out, const T& t) { TSerializer::Save(out, t); } template static inline void SaveArray(IOutputStream* out, const T* t, size_t len) { TArraySerializer::SaveArray(out, t, len); } template static inline void Load(IInputStream* in, T& t) { TSerializer::Load(in, t); } template static inline void LoadArray(IInputStream* in, T* t, size_t len) { TArraySerializer::LoadArray(in, t, len); } template static inline void Load(IInputStream* in, T& t, TStorage& pool) { TSerializer::Load(in, t, pool); } template static inline void LoadArray(IInputStream* in, T* t, size_t len, TStorage& pool) { TArraySerializer::LoadArray(in, t, len, pool); } static inline void SaveSize(IOutputStream* rh, size_t len) { if ((ui64)len < 0xffffffff) { ::Save(rh, (ui32)len); } else { ::Save(rh, (ui32)0xffffffff); ::Save(rh, (ui64)len); } } static inline size_t LoadSize(IInputStream* rh) { ui32 oldVerSize; ui64 newVerSize; ::Load(rh, oldVerSize); if (oldVerSize != 0xffffffff) { return oldVerSize; } else { ::Load(rh, newVerSize); return newVerSize; } } template static inline void LoadSizeAndResize(IInputStream* rh, C& c) { c.resize(LoadSize(rh)); } template static inline char* AllocateFromPool(TStorage& pool, size_t len) { return static_cast(pool.Allocate(len)); } template <> class TSerializer { public: static inline void Save(IOutputStream* rh, const char* s) { size_t length = strlen(s); ::SaveSize(rh, length); ::SavePodArray(rh, s, length); } template static inline void Load(IInputStream* rh, Char*& s, TStorage& pool) { const size_t len = LoadSize(rh); char* res = AllocateFromPool(pool, len + 1); ::LoadPodArray(rh, res, len); res[len] = 0; s = res; } }; template class TVectorSerializer { using TIter = typename TVec::iterator; public: static inline void Save(IOutputStream* rh, const TVec& v) { ::SaveSize(rh, v.size()); ::SaveRange(rh, v.begin(), v.end()); } static inline void Load(IInputStream* rh, TVec& v) { ::LoadSizeAndResize(rh, v); TIter b = v.begin(); TIter e = (TIter)v.end(); ::LoadRange(rh, b, e); } template static inline void Load(IInputStream* rh, TVec& v, TStorage& pool) { ::LoadSizeAndResize(rh, v); TIter b = v.begin(); TIter e = (TIter)v.end(); ::LoadRange(rh, b, e, pool); } }; template class TSerializer>: public TVectorSerializer> { }; template class TSerializer>: public TVectorSerializer> { }; template class TSerializer>: public TVectorSerializer> { }; template class TSerializer>: public TVectorSerializer> { }; template <> class TSerializer: public TVectorSerializer { }; template <> class TSerializer: public TVectorSerializer { }; template class TSerializer>: public TVectorSerializer> { }; template class TSerializer>: public TVectorSerializer> { }; template class TSerializer>: public TVectorSerializer> { }; template class TStdArraySerializer { public: static inline void Save(IOutputStream* rh, const TArray& a) { ::SaveArray(rh, a.data(), a.size()); } static inline void Load(IInputStream* rh, TArray& a) { ::LoadArray(rh, a.data(), a.size()); } }; template class TSerializer>: public TStdArraySerializer> { }; template class TSerializer> { using TPair = std::pair; public: static inline void Save(IOutputStream* rh, const TPair& p) { ::Save(rh, p.first); ::Save(rh, p.second); } static inline void Load(IInputStream* rh, TPair& p) { ::Load(rh, p.first); ::Load(rh, p.second); } template static inline void Load(IInputStream* rh, TPair& p, TStorage& pool) { ::Load(rh, p.first, pool); ::Load(rh, p.second, pool); } }; template struct TTupleSerializer { template static inline void ReverseUseless(F&& f, Tuple&& t, std::index_sequence) { ApplyToMany( std::forward(f), // We need to do this trick because we don't want to break backward compatibility. // Tuples are being packed in reverse order. std::get::value - Indices - 1>(std::forward(t))...); } static inline void Save(IOutputStream* stream, const T& t) { ReverseUseless([&](const auto& v) { ::Save(stream, v); }, t, std::make_index_sequence::value>{}); } static inline void Load(IInputStream* stream, T& t) { ReverseUseless([&](auto& v) { ::Load(stream, v); }, t, std::make_index_sequence::value>{}); } }; template struct TSerializer>: TTupleSerializer> { }; template <> class TSerializer { public: static void Save(IOutputStream* rh, const TBuffer& buf); static void Load(IInputStream* rh, TBuffer& buf); }; template class TSetSerializerInserterBase { public: inline TSetSerializerInserterBase(TSetOrMap& s) : S_(s) { S_.clear(); } inline void Insert(const TValue& v) { S_.insert(v); } protected: TSetOrMap& S_; }; template class TSetSerializerInserter: public TSetSerializerInserterBase { using TBase = TSetSerializerInserterBase; public: inline TSetSerializerInserter(TSetOrMap& s, size_t cnt) : TBase(s) { Y_UNUSED(cnt); } }; template class TSetSerializerInserter: public TSetSerializerInserterBase { using TBase = TSetSerializerInserterBase; public: inline TSetSerializerInserter(TSetType& s, size_t cnt) : TBase(s) { Y_UNUSED(cnt); P_ = this->S_.begin(); } inline void Insert(const TValue& v) { P_ = this->S_.insert(P_, v); } private: typename TSetType::iterator P_; }; template class TSetSerializerInserter, TValue, false>: public TSetSerializerInserterBase, TValue> { using TMapType = THashMap; using TBase = TSetSerializerInserterBase; public: inline TSetSerializerInserter(TMapType& m, size_t cnt) : TBase(m) { m.reserve(cnt); } }; template class TSetSerializerInserter, TValue, false>: public TSetSerializerInserterBase, TValue> { using TMapType = THashMultiMap; using TBase = TSetSerializerInserterBase; public: inline TSetSerializerInserter(TMapType& m, size_t cnt) : TBase(m) { m.reserve(cnt); } }; template class TSetSerializerInserter, TValue, false>: public TSetSerializerInserterBase, TValue> { using TSetType = THashSet; using TBase = TSetSerializerInserterBase; public: inline TSetSerializerInserter(TSetType& s, size_t cnt) : TBase(s) { s.reserve(cnt); } }; template class TSetSerializerBase { public: static inline void Save(IOutputStream* rh, const TSetType& s) { ::SaveSize(rh, s.size()); ::SaveRange(rh, s.begin(), s.end()); } static inline void Load(IInputStream* rh, TSetType& s) { const size_t cnt = ::LoadSize(rh); TSetSerializerInserter ins(s, cnt); TValue v; for (size_t i = 0; i != cnt; ++i) { ::Load(rh, v); ins.Insert(v); } } template static inline void Load(IInputStream* rh, TSetType& s, TStorage& pool) { const size_t cnt = ::LoadSize(rh); TSetSerializerInserter ins(s, cnt); TValue v; for (size_t i = 0; i != cnt; ++i) { ::Load(rh, v, pool); ins.Insert(v); } } }; template struct TMapSerializer: public TSetSerializerBase, sorted> { }; template struct TSetSerializer: public TSetSerializerBase { }; template class TSerializer>: public TMapSerializer, true> { }; template class TSerializer>: public TMapSerializer, true> { }; template class TSerializer>: public TMapSerializer, true> { }; template class TSerializer>: public TMapSerializer, true> { }; template class TSerializer>: public TMapSerializer, false> { }; template class TSerializer>: public TMapSerializer, false> { }; template class TSerializer>: public TSetSerializer, true> { }; template class TSerializer>: public TSetSerializer, true> { }; template class TSerializer>: public TSetSerializer, false> { }; template class TSerializer> { public: static inline void Save(IOutputStream* rh, const TQueue& v) { ::Save(rh, v.Container()); } static inline void Load(IInputStream* in, TQueue& t) { ::Load(in, t.Container()); } }; template class TSerializer> { public: static inline void Save(IOutputStream* rh, const TPriorityQueue& v) { ::Save(rh, v.Container()); } static inline void Load(IInputStream* in, TPriorityQueue& t) { ::Load(in, t.Container()); } }; #ifndef __NVCC__ template struct TSerializer> { static inline void Save(IOutputStream* os, const std::optional& v) { ::Save(os, v.has_value()); if (v.has_value()) { ::Save(os, *v); } } static inline void Load(IInputStream* is, std::optional& v) { v.reset(); bool hasValue; ::Load(is, hasValue); if (hasValue) { ::Load(is, v.emplace()); } } }; namespace NPrivate { template void LoadVariantAlternative(IInputStream* is, Variant& v) { T loaded; ::Load(is, loaded); v.template emplace(std::move(loaded)); } } template struct TSerializer> { using TVar = std::variant; static_assert(sizeof...(Args) < 256, "We use ui8 to store tag"); static void Save(IOutputStream* os, const TVar& v) { ::Save(os, v.index()); std::visit([os](const auto& data) { ::Save(os, data); }, v); } static void Load(IInputStream* is, TVar& v) { ui8 index; ::Load(is, index); if (Y_UNLIKELY(index >= sizeof...(Args))) { ::NPrivate::ThrowUnexpectedVariantTagException(index); } LoadImpl(is, v, index, std::index_sequence_for{}); } private: template static void LoadImpl(IInputStream* is, TVar& v, ui8 index, std::index_sequence) { using TLoader = void (*)(IInputStream*, TVar & v); constexpr TLoader loaders[] = {::NPrivate::LoadVariantAlternative...}; loaders[index](is, v); } }; #endif template static inline void SaveLoad(IOutputStream* out, const T& t) { Save(out, t); } template static inline void SaveLoad(IInputStream* in, T& t) { Load(in, t); } template static inline void SaveMany(S* s, const Ts&... t) { ApplyToMany([&](const auto& v) { Save(s, v); }, t...); } template static inline void LoadMany(S* s, Ts&... t) { ApplyToMany([&](auto& v) { Load(s, v); }, t...); } #define Y_SAVELOAD_DEFINE(...) \ inline void Save(IOutputStream* s) const { \ ::SaveMany(s, __VA_ARGS__); \ } \ \ inline void Load(IInputStream* s) { \ ::LoadMany(s, __VA_ARGS__); \ } \ Y_SEMICOLON_GUARD #define Y_SAVELOAD_DEFINE_OVERRIDE(...) \ void Save(IOutputStream* s) const override { \ ::SaveMany(s, __VA_ARGS__); \ } \ \ void Load(IInputStream* s) override { \ ::LoadMany(s, __VA_ARGS__); \ } \ Y_SEMICOLON_GUARD template struct TNonVirtualSaver { const T* Data; void Save(IOutputStream* out) const { Data->T::Save(out); } }; template inline void LoadMany(S* s, TNonVirtualSaver t, R&... r) { const_cast(t.Data)->T::Load(s); ::LoadMany(s, r...); }