#include "write_trie_backwards.h" #include "comptrie_impl.h" #include "leaf_skipper.h" #include #include namespace NCompactTrie { size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose) { if (verbose) { Cerr << "Writing down the trie..." << Endl; } // Rewrite everything from the back, removing unused pieces. const size_t chunksize = 0x10000; TVector resultData; resultData.push_back(new char[chunksize]); char* chunkend = resultData.back() + chunksize; size_t resultLength = 0; size_t chunkLength = 0; size_t counter = 0; TBuffer bufferHolder; while (enumerator.Move()) { if (verbose) ShowProgress(++counter); size_t bufferLength = 64 + enumerator.GetLeafLength(); // never know how big leaf data can be bufferHolder.Clear(); bufferHolder.Resize(bufferLength); char* buffer = bufferHolder.Data(); size_t nodelength = enumerator.RecreateNode(buffer, resultLength); Y_ASSERT(nodelength <= bufferLength); resultLength += nodelength; if (chunkLength + nodelength <= chunksize) { chunkLength += nodelength; memcpy(chunkend - chunkLength, buffer, nodelength); } else { // allocate a new chunk memcpy(chunkend - chunksize, buffer + nodelength - (chunksize - chunkLength), chunksize - chunkLength); chunkLength = chunkLength + nodelength - chunksize; resultData.push_back(new char[chunksize]); chunkend = resultData.back() + chunksize; while (chunkLength > chunksize) { // allocate a new chunks chunkLength -= chunksize; memcpy(chunkend - chunksize, buffer + chunkLength, chunksize); resultData.push_back(new char[chunksize]); chunkend = resultData.back() + chunksize; } memcpy(chunkend - chunkLength, buffer, chunkLength); } } if (verbose) Cerr << counter << Endl; // Write the whole thing down while (!resultData.empty()) { char* chunk = resultData.back(); os.Write(chunk + chunksize - chunkLength, chunkLength); chunkLength = chunksize; delete[] chunk; resultData.pop_back(); } return resultLength; } size_t WriteTrieBackwardsNoAlloc(IOutputStream& os, TReverseNodeEnumerator& enumerator, TOpaqueTrie& trie, EMinimizeMode mode) { char* data = const_cast(trie.Data); char* end = data + trie.Length; char* pos = end; TVector buf(64); while (enumerator.Move()) { size_t nodeLength = enumerator.RecreateNode(nullptr, end - pos); if (nodeLength > buf.size()) buf.resize(nodeLength); size_t realLength = enumerator.RecreateNode(buf.data(), end - pos); Y_ASSERT(realLength == nodeLength); pos -= nodeLength; memcpy(pos, buf.data(), nodeLength); } switch (mode) { case MM_NOALLOC: os.Write(pos, end - pos); break; case MM_INPLACE: memmove(data, pos, end - pos); break; default: Y_ABORT_UNLESS(false); } return end - pos; } }