#pragma once #include #include #include #include #include #include #include // Data serialization strategy class. // Default realization can pack only limited range of types, but you can pack any data other using your own strategy class. template class TNullPacker { // Very effective package class - pack any data into zero bytes :) public: void UnpackLeaf(const char*, T& t) const { t = T(); } void PackLeaf(char*, const T&, size_t) const { } size_t MeasureLeaf(const T&) const { return 0; } size_t SkipLeaf(const char*) const { return 0; } }; template class TAsIsPacker { // this packer is not really a packer... public: void UnpackLeaf(const char* p, T& t) const { memcpy(&t, p, sizeof(T)); } void PackLeaf(char* buffer, const T& data, size_t computedSize) const { Y_ASSERT(computedSize == sizeof(data)); memcpy(buffer, &data, sizeof(T)); } size_t MeasureLeaf(const T& data) const { Y_UNUSED(data); return sizeof(T); } size_t SkipLeaf(const char*) const { return sizeof(T); } }; // Implementation namespace NPackers { template inline ui64 ConvertIntegral(const T& data); template <> inline ui64 ConvertIntegral(const i64& data) { if (data < 0) { return (static_cast(-1 * data) << 1) | 1; } else { return static_cast(data) << 1; } } namespace NImpl { template struct TConvertImpl { static inline ui64 Convert(const T& data); }; template struct TConvertImpl { static inline ui64 Convert(const T& data) { return ConvertIntegral(static_cast(data)); } }; template struct TConvertImpl { static inline ui64 Convert(const T& data) { return data; } }; } template inline ui64 ConvertIntegral(const T& data) { static_assert(std::is_integral::value, "T must be integral type"); return NImpl::TConvertImpl::value>::Convert(data); } //--------------------------------- // TIntegralPacker --- for integral types. template class TIntegralPacker { // can pack only integral types <= ui64 public: void UnpackLeaf(const char* p, T& t) const; void PackLeaf(char* buffer, const T& data, size_t size) const; size_t MeasureLeaf(const T& data) const; size_t SkipLeaf(const char* p) const; }; template <> inline size_t TIntegralPacker::MeasureLeaf(const ui64& val) const { constexpr size_t MAX_SIZE = sizeof(ui64) + sizeof(ui64) / 8; ui64 value = val; size_t len = 1; value >>= 7; for (; value && len < MAX_SIZE; value >>= 7) ++len; return len; } template <> inline void TIntegralPacker::PackLeaf(char* buffer, const ui64& val, size_t len) const { ui64 value = val; int lenmask = 0; for (size_t i = len - 1; i; --i) { buffer[i] = (char)(value & 0xFF); value >>= 8; lenmask = ((lenmask >> 1) | (1 << 7)); } buffer[0] = (char)(lenmask | value); } extern const ui8 SkipTable[]; template <> inline void TIntegralPacker::UnpackLeaf(const char* p, ui64& result) const { unsigned char ch = *(p++); size_t taillen = SkipTable[ch] - 1; result = (ch & (0x7F >> taillen)); while (taillen--) result = ((result << 8) | (*(p++) & 0xFF)); } template <> inline size_t TIntegralPacker::SkipLeaf(const char* p) const { return SkipTable[(ui8)*p]; } namespace NImpl { template struct TUnpackLeafImpl { inline void UnpackLeaf(const char* p, T& t) const; }; template struct TUnpackLeafImpl { inline void UnpackLeaf(const char* p, T& t) const { ui64 val; TIntegralPacker().UnpackLeaf(p, val); if (val & 1) { t = -1 * static_cast(val >> 1); } else { t = static_cast(val >> 1); } } }; template struct TUnpackLeafImpl { inline void UnpackLeaf(const char* p, T& t) const { ui64 tmp; TIntegralPacker().UnpackLeaf(p, tmp); t = static_cast(tmp); } }; } template inline void TIntegralPacker::UnpackLeaf(const char* p, T& t) const { NImpl::TUnpackLeafImpl::value>().UnpackLeaf(p, t); } template inline void TIntegralPacker::PackLeaf(char* buffer, const T& data, size_t size) const { TIntegralPacker().PackLeaf(buffer, ConvertIntegral(data), size); } template inline size_t TIntegralPacker::MeasureLeaf(const T& data) const { return TIntegralPacker().MeasureLeaf(ConvertIntegral(data)); } template inline size_t TIntegralPacker::SkipLeaf(const char* p) const { return TIntegralPacker().SkipLeaf(p); } //------------------------------------------- // TFPPacker --- for float/double namespace NImpl { template class TFPPackerBase { protected: typedef TIntegralPacker TPacker; union THelper { TFloat F; TUInt U; }; TFloat FromUInt(TUInt u) const { THelper h; h.U = ReverseBytes(u); return h.F; } TUInt ToUInt(TFloat f) const { THelper h; h.F = f; return ReverseBytes(h.U); } public: void UnpackLeaf(const char* c, TFloat& t) const { TUInt u = 0; TPacker().UnpackLeaf(c, u); t = FromUInt(u); } void PackLeaf(char* c, const TFloat& t, size_t sz) const { TPacker().PackLeaf(c, ToUInt(t), sz); } size_t MeasureLeaf(const TFloat& t) const { return TPacker().MeasureLeaf(ToUInt(t)); } size_t SkipLeaf(const char* c) const { return TPacker().SkipLeaf(c); } }; } class TFloatPacker: public NImpl::TFPPackerBase { }; class TDoublePacker: public NImpl::TFPPackerBase { }; //------------------------------------------- // TStringPacker --- for TString/TUtf16String and TStringBuf. template class TStringPacker { public: void UnpackLeaf(const char* p, TStringType& t) const; void PackLeaf(char* buffer, const TStringType& data, size_t size) const; size_t MeasureLeaf(const TStringType& data) const; size_t SkipLeaf(const char* p) const; }; template inline void TStringPacker::UnpackLeaf(const char* buf, TStringType& t) const { size_t len; TIntegralPacker().UnpackLeaf(buf, len); size_t start = TIntegralPacker().SkipLeaf(buf); t = TStringType((const typename TStringType::char_type*)(buf + start), len); } template inline void TStringPacker::PackLeaf(char* buf, const TStringType& str, size_t size) const { size_t len = str.size(); size_t lenChar = len * sizeof(typename TStringType::char_type); size_t start = size - lenChar; TIntegralPacker().PackLeaf(buf, len, TIntegralPacker().MeasureLeaf(len)); memcpy(buf + start, str.data(), lenChar); } template inline size_t TStringPacker::MeasureLeaf(const TStringType& str) const { size_t len = str.size(); return TIntegralPacker().MeasureLeaf(len) + len * sizeof(typename TStringType::char_type); } template inline size_t TStringPacker::SkipLeaf(const char* buf) const { size_t result = TIntegralPacker().SkipLeaf(buf); { size_t len; TIntegralPacker().UnpackLeaf(buf, len); result += len * sizeof(typename TStringType::char_type); } return result; } template class TPacker; // TContainerPacker --- for any container // Requirements to class C: // - has method size() (returns size_t) // - has subclass C::value_type // - has subclass C::const_iterator // - has methods begin() and end() (return C::const_iterator) // - has method insert(C::const_iterator, const C::value_type&) // Examples: TVector, TList, TSet // Requirements to class EP: has methods as in any packer (UnpackLeaf, PackLeaf, MeasureLeaf, SkipLeaf) that // are applicable to C::value_type template struct TContainerInfo { enum { IsVector = 0 }; }; template struct TContainerInfo> { enum { IsVector = 1 }; }; template struct TContainerInfo> { enum { IsVector = 1 }; }; template class TContainerPackerHelper { }; template <> class TContainerPackerHelper { public: template static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { p.UnpackLeafSimple(buffer, c); } }; template <> class TContainerPackerHelper { public: template static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { p.UnpackLeafVector(buffer, c); } }; template > class TContainerPacker { private: typedef C TContainer; typedef EP TElementPacker; typedef typename TContainer::const_iterator TElementIterator; void UnpackLeafSimple(const char* buffer, TContainer& c) const; void UnpackLeafVector(const char* buffer, TContainer& c) const; friend class TContainerPackerHelper::IsVector>; public: void UnpackLeaf(const char* buffer, TContainer& c) const { TContainerPackerHelper::IsVector>::UnpackLeaf(*this, buffer, c); } void PackLeaf(char* buffer, const TContainer& data, size_t size) const; size_t MeasureLeaf(const TContainer& data) const; size_t SkipLeaf(const char* buffer) const; }; template inline void TContainerPacker::UnpackLeafSimple(const char* buffer, C& result) const { size_t offset = TIntegralPacker().SkipLeaf(buffer); // first value is the total size (not needed here) size_t len; TIntegralPacker().UnpackLeaf(buffer + offset, len); offset += TIntegralPacker().SkipLeaf(buffer + offset); result.clear(); typename C::value_type value; for (size_t i = 0; i < len; i++) { TElementPacker().UnpackLeaf(buffer + offset, value); result.insert(result.end(), value); offset += TElementPacker().SkipLeaf(buffer + offset); } } template inline void TContainerPacker::UnpackLeafVector(const char* buffer, C& result) const { size_t offset = TIntegralPacker().SkipLeaf(buffer); // first value is the total size (not needed here) size_t len; TIntegralPacker().UnpackLeaf(buffer + offset, len); offset += TIntegralPacker().SkipLeaf(buffer + offset); result.resize(len); for (size_t i = 0; i < len; i++) { TElementPacker().UnpackLeaf(buffer + offset, result[i]); offset += TElementPacker().SkipLeaf(buffer + offset); } } template inline void TContainerPacker::PackLeaf(char* buffer, const C& data, size_t size) const { size_t sizeOfSize = TIntegralPacker().MeasureLeaf(size); TIntegralPacker().PackLeaf(buffer, size, sizeOfSize); size_t len = data.size(); size_t curSize = TIntegralPacker().MeasureLeaf(len); TIntegralPacker().PackLeaf(buffer + sizeOfSize, len, curSize); curSize += sizeOfSize; for (TElementIterator p = data.begin(); p != data.end(); p++) { size_t sizeChange = TElementPacker().MeasureLeaf(*p); TElementPacker().PackLeaf(buffer + curSize, *p, sizeChange); curSize += sizeChange; } Y_ASSERT(curSize == size); } template inline size_t TContainerPacker::MeasureLeaf(const C& data) const { size_t curSize = TIntegralPacker().MeasureLeaf(data.size()); for (TElementIterator p = data.begin(); p != data.end(); p++) curSize += TElementPacker().MeasureLeaf(*p); size_t extraSize = TIntegralPacker().MeasureLeaf(curSize); // Double measurement protects against sudden increases in extraSize, // e.g. when curSize is 127 and stays in one byte, but curSize + 1 requires two bytes. extraSize = TIntegralPacker().MeasureLeaf(curSize + extraSize); Y_ASSERT(extraSize == TIntegralPacker().MeasureLeaf(curSize + extraSize)); return curSize + extraSize; } template inline size_t TContainerPacker::SkipLeaf(const char* buffer) const { size_t value; TIntegralPacker().UnpackLeaf(buffer, value); return value; } // TPairPacker --- for std::pair (any two types; can be nested) // TPacker and TPacker should be valid classes template , class TPacker2 = TPacker> class TPairPacker { private: typedef std::pair TMyPair; public: void UnpackLeaf(const char* buffer, TMyPair& pair) const; void PackLeaf(char* buffer, const TMyPair& data, size_t size) const; size_t MeasureLeaf(const TMyPair& data) const; size_t SkipLeaf(const char* buffer) const; }; template inline void TPairPacker::UnpackLeaf(const char* buffer, std::pair& pair) const { TPacker1().UnpackLeaf(buffer, pair.first); size_t size = TPacker1().SkipLeaf(buffer); TPacker2().UnpackLeaf(buffer + size, pair.second); } template inline void TPairPacker::PackLeaf(char* buffer, const std::pair& data, size_t size) const { size_t size1 = TPacker1().MeasureLeaf(data.first); TPacker1().PackLeaf(buffer, data.first, size1); size_t size2 = TPacker2().MeasureLeaf(data.second); TPacker2().PackLeaf(buffer + size1, data.second, size2); Y_ASSERT(size == size1 + size2); } template inline size_t TPairPacker::MeasureLeaf(const std::pair& data) const { size_t size1 = TPacker1().MeasureLeaf(data.first); size_t size2 = TPacker2().MeasureLeaf(data.second); return size1 + size2; } template inline size_t TPairPacker::SkipLeaf(const char* buffer) const { size_t size1 = TPacker1().SkipLeaf(buffer); size_t size2 = TPacker2().SkipLeaf(buffer + size1); return size1 + size2; } //------------------------------------------------------------------------------------------ // Packer for fixed-size arrays, i.e. for std::array. // Saves memory by not storing anything about their size. // SkipLeaf skips every value, so can be slow for big arrays. // Requires std::tuple_size, TValue::operator[] and possibly TValue::value_type. template > class TArrayPacker { public: using TElemPacker = TElementPacker; enum { Size = std::tuple_size::value }; void UnpackLeaf(const char* p, TValue& t) const { const char* buf = p; for (size_t i = 0; i < Size; ++i) { TElemPacker().UnpackLeaf(buf, t[i]); buf += TElemPacker().SkipLeaf(buf); } } void PackLeaf(char* buffer, const TValue& data, size_t computedSize) const { size_t remainingSize = computedSize; char* pos = buffer; for (size_t i = 0; i < Size; ++i) { const size_t elemSize = TElemPacker().MeasureLeaf(data[i]); TElemPacker().PackLeaf(pos, data[i], Min(elemSize, remainingSize)); pos += elemSize; remainingSize -= elemSize; } } size_t MeasureLeaf(const TValue& data) const { size_t result = 0; for (size_t i = 0; i < Size; ++i) { result += TElemPacker().MeasureLeaf(data[i]); } return result; } size_t SkipLeaf(const char* p) const // this function better be fast because it is very frequently used { const char* buf = p; for (size_t i = 0; i < Size; ++i) { buf += TElemPacker().SkipLeaf(buf); } return buf - p; } }; //------------------------------------ // TPacker --- the generic packer. template class TPackerImpl; template class TPackerImpl: public TIntegralPacker { }; // No implementation for non-integral types. template class TPacker: public TPackerImpl::value> { }; template <> class TPacker: public TAsIsPacker { }; template <> class TPacker: public TAsIsPacker { }; template <> class TPacker: public TStringPacker { }; template <> class TPacker: public TStringPacker { }; template <> class TPacker: public TStringPacker { }; template <> class TPacker: public TStringPacker { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TContainerPacker> { }; template class TPacker>: public TPairPacker { }; template class TPacker>: public TArrayPacker> { }; }