#include "descriptor.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace NProtoBuf; static TString SerializeFileDescriptorSet(const FileDescriptorSet& proto) { const auto size = proto.ByteSize(); TTempBuf data(size); proto.SerializeWithCachedSizesToArray((ui8*)data.Data()); TStringStream str; { TZLibCompress comp(&str, ZLib::GZip); comp.Write(data.Data(), size); } return str.Str(); } static bool ParseFileDescriptorSet(const TStringBuf& data, FileDescriptorSet* proto) { TMemoryInput input(data.data(), data.size()); TString buf = TZLibDecompress(&input).ReadAll(); if (!proto->ParseFromArray(buf.data(), buf.size())) { return false; } return true; } TDynamicInfo::TDynamicInfo(TDynamicPrototypePtr dynamicPrototype) : DynamicPrototype(dynamicPrototype) , SkipBytes_(0) { } TDynamicInfo::~TDynamicInfo() { } TDynamicInfoRef TDynamicInfo::Create(const TStringBuf& typeConfig) { auto data = ParseTypeConfig(typeConfig); const TString& meta = Base64Decode(data.Metadata); const TString& name = data.MessageName; FileDescriptorSet set; if (!ParseFileDescriptorSet(meta, &set)) { ythrow yexception() << "can't parse metadata"; } auto info = MakeIntrusive(TDynamicPrototype::Create(set, name, true)); info->EnumFormat_ = data.EnumFormat; info->ProtoFormat_ = data.ProtoFormat; info->Recursion_ = data.Recursion; info->YtMode_ = data.YtMode; info->SkipBytes_ = data.SkipBytes; info->OptionalLists_ = data.OptionalLists; info->SyntaxAware_ = data.SyntaxAware; return info; } const Descriptor* TDynamicInfo::Descriptor() const { return DynamicPrototype->GetDescriptor(); } EEnumFormat TDynamicInfo::GetEnumFormat() const { return EnumFormat_; } ERecursionTraits TDynamicInfo::GetRecursionTraits() const { return Recursion_; } bool TDynamicInfo::GetYtMode() const { return YtMode_; } bool TDynamicInfo::GetOptionalLists() const { return OptionalLists_; } bool TDynamicInfo::GetSyntaxAware() const { return SyntaxAware_; } TAutoPtr TDynamicInfo::MakeProto() { return DynamicPrototype->CreateUnsafe(); } TAutoPtr TDynamicInfo::Parse(const TStringBuf& data) { auto mut = MakeProto(); TStringBuf tmp(data); if (SkipBytes_) { tmp = TStringBuf(tmp.data() + SkipBytes_, tmp.size() - SkipBytes_); } switch (ProtoFormat_) { case PF_PROTOBIN: { if (!mut->ParseFromArray(tmp.data(), tmp.size())) { ythrow yexception() << "can't parse protobin message"; } break; } case PF_PROTOTEXT: { io::ArrayInputStream si(tmp.data(), tmp.size()); if (!TextFormat::Parse(&si, mut.Get())) { ythrow yexception() << "can't parse prototext message"; } break; } case PF_JSON: { NJson::TJsonValue value; if (NJson::ReadJsonFastTree(tmp, &value)) { NProtobufJson::Json2Proto(value, *mut); } else { ythrow yexception() << "can't parse json value"; } break; } } return mut; } TString TDynamicInfo::Serialize(const Message& proto) { TString result; switch (ProtoFormat_) { case PF_PROTOBIN: { result.ReserveAndResize(proto.ByteSize()); if (!proto.SerializeToArray(result.begin(), result.size())) { ythrow yexception() << "can't serialize protobin message"; } break; } case PF_PROTOTEXT: { if (!TextFormat::PrintToString(proto, &result)) { ythrow yexception() << "can't serialize prototext message"; } break; } case PF_JSON: { NJson::TJsonValue value; NProtobufJson::Proto2Json(proto, value); result = NJson::WriteJson(value); break; } } return result; } TString GenerateProtobufTypeConfig( const Descriptor* descriptor, const TProtoTypeConfigOptions& options) { NJson::TJsonValue ret(NJson::JSON_MAP); ret["name"] = descriptor->full_name(); ret["meta"] = Base64Encode( SerializeFileDescriptorSet(GenerateFileDescriptorSet(descriptor))); if (options.SkipBytes > 0) { ret["skip"] = options.SkipBytes; } switch (options.ProtoFormat) { case PF_PROTOBIN: break; case PF_PROTOTEXT: ret["format"] = "prototext"; break; case PF_JSON: ret["format"] = "json"; break; } if (!options.OptionalLists) { ret["lists"]["optional"] = false; } if (options.SyntaxAware) { ret["syntax"]["aware"] = options.SyntaxAware; } switch (options.EnumFormat) { case EEnumFormat::Number: break; case EEnumFormat::Name: ret["view"]["enum"] = "name"; break; case EEnumFormat::FullName: ret["view"]["enum"] = "full_name"; break; } switch (options.Recursion) { case ERecursionTraits::Fail: break; case ERecursionTraits::Ignore: ret["view"]["recursion"] = "ignore"; break; case ERecursionTraits::Bytes: ret["view"]["recursion"] = "bytes"; break; } if (options.YtMode) { ret["view"]["yt_mode"] = true; } return NJson::WriteJson(ret, false); } TProtoTypeConfig ParseTypeConfig(const TStringBuf& config) { if (config.empty()) { ythrow yexception() << "empty metadata"; } switch (config[0]) { case '#': { auto plus = config.find('+'); if (config[0] != '#') { ythrow yexception() << "unknown version of metadata format"; } if (plus == TStringBuf::npos) { ythrow yexception() << "invalid metadata"; } TProtoTypeConfig result; result.MessageName = TStringBuf(config.begin() + 1, plus - 1); result.Metadata = TStringBuf(config.begin() + 1 + plus, config.size() - plus - 1); result.SkipBytes = 0; return result; } case '{': { NJson::TJsonValue value; if (NJson::ReadJsonFastTree(config, &value)) { TProtoTypeConfig result; TString protoFormat = value["format"].GetStringSafe("protobin"); TString enumFormat = value["view"]["enum"].GetStringSafe("number"); TString recursion = value["view"]["recursion"].GetStringSafe("fail"); result.MessageName = value["name"].GetString(); result.Metadata = value["meta"].GetString(); result.SkipBytes = value["skip"].GetIntegerSafe(0); result.OptionalLists = value["lists"]["optional"].GetBooleanSafe(true); result.SyntaxAware = value["syntax"]["aware"].GetBooleanSafe(false); result.YtMode = value["view"]["yt_mode"].GetBooleanSafe(false); if (protoFormat == "protobin") { result.ProtoFormat = PF_PROTOBIN; } else if (protoFormat == "prototext") { result.ProtoFormat = PF_PROTOTEXT; } else if (protoFormat == "json") { result.ProtoFormat = PF_JSON; } else { ythrow yexception() << "unsupported format " << protoFormat; } if (enumFormat == "number") { result.EnumFormat = EEnumFormat::Number; } else if (enumFormat == "name") { result.EnumFormat = EEnumFormat::Name; } else if (enumFormat == "full_name") { result.EnumFormat = EEnumFormat::FullName; } else { ythrow yexception() << "unsupported enum representation " << enumFormat; } if (recursion == "fail") { result.Recursion = ERecursionTraits::Fail; } else if (recursion == "ignore") { result.Recursion = ERecursionTraits::Ignore; } else if (recursion == "bytes") { result.Recursion = ERecursionTraits::Bytes; } else { ythrow yexception() << "unsupported recursion trait " << recursion; } return result; } else { ythrow yexception() << "can't parse json metadata"; } } default: ythrow yexception() << "invalid control char " << TStringBuf(config.data(), 1); } }