#pragma once #include "codecs.h" #include "delta_codec.h" #include "tls_cache.h" #include #include #include namespace NCodecs { template class TPForCodec: public ICodec { using TUnsigned = std::make_unsigned_t; typedef TDeltaCodec TDCodec; typedef std::conditional_t TValue; static_assert(std::is_unsigned::value, "expect std:is_unsigned::value"); static const ui64 BitsInT = sizeof(TUnsigned) * 8; TDCodec DeltaCodec; public: static TStringBuf MyName(); TPForCodec() { MyTraits.AssumesStructuredInput = true; MyTraits.SizeOfInputElement = sizeof(T); MyTraits.SizeOnDecodeMultiplier = sizeof(T); } TString GetName() const override { return ToString(MyName()); } ui8 Encode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return 0; } b.Reserve(2 * s.size() + b.Size()); if (WithDelta) { auto buffer = TBufferTlsCache::TlsInstance().Item(); TBuffer& db = buffer.Get(); db.Clear(); db.Reserve(2 * s.size()); DeltaCodec.Encode(s, db); s = TStringBuf{db.data(), db.size()}; } TArrayRef tin{(const TValue*)s.data(), s.size() / sizeof(TValue)}; const ui64 sz = tin.size(); ui64 bitcounts[BitsInT + 1]; Zero(bitcounts); ui32 zeros = 0; for (const TValue* it = tin.begin(); it != tin.end(); ++it) { TUnsigned v = 1 + (TUnsigned)*it; ui64 l = MostSignificantBit(v) + 1; ++bitcounts[l]; if (!v) { ++zeros; } } // cumulative bit counts for (ui64 i = 0; i < BitsInT; ++i) { bitcounts[i + 1] += bitcounts[i]; } bool hasexceptions = zeros; ui64 optimalbits = BitsInT; { ui64 excsize = 0; ui64 minsize = sz * BitsInT; for (ui64 current = BitsInT; current; --current) { ui64 size = bitcounts[current] * current + (sz - bitcounts[current]) * (current + 6 + excsize) + zeros * (current + 6); excsize += current * bitcounts[current]; if (size < minsize) { minsize = size; optimalbits = current; hasexceptions = zeros || sz - bitcounts[current]; } } } if (!optimalbits || BitsInT == optimalbits) { b.Append((ui8)-1); b.Append(s.data(), s.size()); return 0; } else { NBitIO::TBitOutputVector bout(&b); bout.Write(0, 1); bout.Write(hasexceptions, 1); bout.Write(optimalbits, 6); for (const TValue* it = tin.begin(); it != tin.end(); ++it) { TUnsigned word = 1 + (TUnsigned)*it; ui64 len = MostSignificantBit(word) + 1; if (len > optimalbits || !word) { Y_ENSURE(hasexceptions, " "); bout.Write(0, optimalbits); bout.Write(len, 6); bout.Write(word, len); } else { bout.Write(word, optimalbits); } } return bout.GetByteReminder(); } // the rest of the last byte is zero padded. BitsInT is always > 7. } void Decode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return; } b.Reserve(s.size() * sizeof(T) + b.Size()); ui64 isplain = 0; ui64 hasexceptions = 0; ui64 bits = 0; NBitIO::TBitInput bin(s); bin.ReadK<1>(isplain); bin.ReadK<1>(hasexceptions); bin.ReadK<6>(bits); if (Y_UNLIKELY(isplain)) { s.Skip(1); if (WithDelta) { DeltaCodec.Decode(s, b); } else { b.Append(s.data(), s.size()); } } else { typename TDCodec::TDecoder decoder; if (hasexceptions) { ui64 word = 0; while (bin.Read(word, bits)) { if (word || (bin.ReadK<6>(word) && bin.Read(word, word))) { --word; TValue t = word; if (WithDelta) { if (decoder.Decode(t)) { TStringBuf r{(char*)&decoder.Result, sizeof(decoder.Result)}; b.Append(r.data(), r.size()); } } else { TStringBuf r{(char*)&t, sizeof(t)}; b.Append(r.data(), r.size()); } } } } else { ui64 word = 0; T outarr[256 / sizeof(T)]; ui32 cnt = 0; while (true) { ui64 v = bin.Read(word, bits); if ((!v) | (!word)) break; --word; TValue t = word; if (WithDelta) { if (decoder.Decode(t)) { outarr[cnt++] = decoder.Result; } } else { outarr[cnt++] = t; } if (cnt == Y_ARRAY_SIZE(outarr)) { b.Append((const char*)outarr, sizeof(outarr)); cnt = 0; } } if (cnt) { b.Append((const char*)outarr, cnt * sizeof(T)); } } } } protected: void DoLearn(ISequenceReader&) override { } }; }