json_ut.cpp 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290
  1. #include "json.h"
  2. #include <library/cpp/monlib/encode/protobuf/protobuf.h>
  3. #include <library/cpp/monlib/metrics/labels.h>
  4. #include <library/cpp/json/json_reader.h>
  5. #include <library/cpp/resource/resource.h>
  6. #include <library/cpp/testing/unittest/registar.h>
  7. #include <util/stream/str.h>
  8. #include <util/string/builder.h>
  9. #include <limits>
  10. using namespace NMonitoring;
  11. namespace NMonitoring {
  12. bool operator<(const TLabel& lhs, const TLabel& rhs) {
  13. return lhs.Name() < rhs.Name() ||
  14. (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value());
  15. }
  16. }
  17. namespace {
  18. void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) {
  19. UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size());
  20. TSet<TLabel> actualSet;
  21. TSet<TLabel> expectedSet;
  22. Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) {
  23. return TLabel{l.Name(), l.Value()};
  24. });
  25. const auto& l = actual.GetLabels();
  26. Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)),
  27. [](auto&& elem) -> TLabel {
  28. return {elem.GetName(), elem.GetValue()};
  29. });
  30. TVector<TLabel> diff;
  31. SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet),
  32. std::begin(actualSet), std::end(actualSet), std::back_inserter(diff));
  33. if (diff.size() > 0) {
  34. for (auto&& l : diff) {
  35. Cerr << l << Endl;
  36. }
  37. UNIT_FAIL("Labels don't match");
  38. }
  39. }
  40. void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) {
  41. UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name);
  42. UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value);
  43. }
  44. void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) {
  45. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  46. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
  47. UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon());
  48. }
  49. void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) {
  50. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  51. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
  52. UNIT_ASSERT(std::isnan(p.GetFloat64()));
  53. }
  54. void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) {
  55. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  56. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
  57. UNIT_ASSERT(std::isinf(p.GetFloat64()));
  58. if (sign < 0) {
  59. UNIT_ASSERT(p.GetFloat64() < 0);
  60. }
  61. }
  62. void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) {
  63. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  64. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64);
  65. UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value);
  66. }
  67. void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) {
  68. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  69. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64);
  70. UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value);
  71. }
  72. void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) {
  73. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  74. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram);
  75. const NProto::THistogram& h = p.GetHistogram();
  76. UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count());
  77. UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count());
  78. for (size_t i = 0; i < h.BoundsSize(); i++) {
  79. UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>());
  80. UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i));
  81. }
  82. }
  83. void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) {
  84. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  85. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
  86. const double eps = 1e-10;
  87. const NProto::TLogHistogram& h = p.GetLogHistogram();
  88. UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps);
  89. UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount());
  90. UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower());
  91. UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count());
  92. for (size_t i = 0; i < expected.Count(); ++i) {
  93. UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps);
  94. }
  95. }
  96. void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) {
  97. UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
  98. UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
  99. auto actual = p.GetSummaryDouble();
  100. const double eps = 1e-10;
  101. UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps);
  102. UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps);
  103. UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps);
  104. UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps);
  105. UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount());
  106. }
  107. } // namespace
  108. Y_UNIT_TEST_SUITE(TJsonTest) {
  109. const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z");
  110. Y_UNIT_TEST(Encode) {
  111. auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) {
  112. TString json;
  113. TStringOutput out(json);
  114. auto e = cloud
  115. ? (buffered ? BufferedEncoderCloudJson(&out, 2, "metric") : EncoderCloudJson(&out, 2, "metric"))
  116. : (buffered ? BufferedEncoderJson(&out, 2) : EncoderJson(&out, 2));
  117. e->OnStreamBegin();
  118. { // common time
  119. e->OnCommonTime(TInstant::Seconds(1500000000));
  120. }
  121. { // common labels
  122. e->OnLabelsBegin();
  123. e->OnLabel("project", "solomon");
  124. e->OnLabelsEnd();
  125. }
  126. { // metric #1
  127. e->OnMetricBegin(EMetricType::COUNTER);
  128. {
  129. e->OnLabelsBegin();
  130. e->OnLabel("metric", "single");
  131. e->OnLabel("labels", "l1");
  132. e->OnLabelsEnd();
  133. }
  134. e->OnUint64(now, 17);
  135. e->OnMetricEnd();
  136. }
  137. { // metric #2
  138. e->OnMetricBegin(EMetricType::RATE);
  139. {
  140. e->OnLabelsBegin();
  141. e->OnLabel("metric", "single");
  142. e->OnLabel("labels", "l2");
  143. e->OnLabelsEnd();
  144. }
  145. e->OnUint64(now, 17);
  146. e->OnMetricEnd();
  147. }
  148. { // metric #3
  149. e->OnMetricBegin(EMetricType::GAUGE);
  150. e->OnDouble(now, 3.14);
  151. {
  152. e->OnLabelsBegin();
  153. e->OnLabel("metric", "single");
  154. e->OnLabel("labels", "l3");
  155. e->OnLabelsEnd();
  156. }
  157. e->OnMetricEnd();
  158. }
  159. { // metric #4
  160. e->OnMetricBegin(EMetricType::IGAUGE);
  161. e->OnInt64(now, 42);
  162. {
  163. e->OnLabelsBegin();
  164. e->OnLabel("metric", "single_igauge");
  165. e->OnLabel("labels", "l4");
  166. e->OnLabelsEnd();
  167. }
  168. e->OnMetricEnd();
  169. }
  170. { // metric #5
  171. e->OnMetricBegin(EMetricType::GAUGE);
  172. {
  173. e->OnLabelsBegin();
  174. e->OnLabel("metric", "multiple");
  175. e->OnLabel("labels", "l5");
  176. e->OnLabelsEnd();
  177. }
  178. e->OnDouble(now, std::numeric_limits<double>::quiet_NaN());
  179. e->OnDouble(now + TDuration::Seconds(15), std::numeric_limits<double>::infinity());
  180. e->OnDouble(now + TDuration::Seconds(30), -std::numeric_limits<double>::infinity());
  181. e->OnMetricEnd();
  182. }
  183. { // metric #6
  184. e->OnMetricBegin(EMetricType::COUNTER);
  185. e->OnUint64(now, 1337);
  186. e->OnUint64(now + TDuration::Seconds(15), 1338);
  187. {
  188. e->OnLabelsBegin();
  189. e->OnLabel("metric", "multiple");
  190. e->OnLabel("labels", "l6");
  191. e->OnLabelsEnd();
  192. }
  193. e->OnMetricEnd();
  194. }
  195. e->OnStreamEnd();
  196. e->Close();
  197. json += "\n";
  198. auto parseJson = [] (auto buf) {
  199. NJson::TJsonValue value;
  200. NJson::ReadJsonTree(buf, &value, true);
  201. return value;
  202. };
  203. const auto expectedJson = NResource::Find(expectedResourceKey);
  204. UNIT_ASSERT_EQUAL(parseJson(json), parseJson(expectedJson));
  205. };
  206. check(false, false, "/expected.json");
  207. check(false, true, "/expected_buffered.json");
  208. check(true, false, "/expected_cloud.json");
  209. check(true, true, "/expected_cloud_buffered.json");
  210. }
  211. TLogHistogramSnapshotPtr TestLogHistogram(ui32 v = 1) {
  212. TVector<double> buckets{0.5 * v, 0.25 * v, 0.25 * v, 0.5 * v};
  213. return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets));
  214. }
  215. Y_UNIT_TEST(HistogramAndSummaryMetricTypesAreNotSupportedByCloudJson) {
  216. const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z");
  217. auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) {
  218. encoder->OnStreamBegin();
  219. {
  220. encoder->OnMetricBegin(metricType);
  221. {
  222. encoder->OnLabelsBegin();
  223. encoder->OnLabel("name", "m");
  224. encoder->OnLabelsEnd();
  225. }
  226. switch (metricType) {
  227. case EMetricType::HIST: {
  228. auto histogram = ExponentialHistogram(6, 2);
  229. encoder->OnHistogram(now, histogram->Snapshot());
  230. break;
  231. }
  232. case EMetricType::LOGHIST: {
  233. auto histogram = TestLogHistogram();
  234. encoder->OnLogHistogram(now, histogram);
  235. break;
  236. }
  237. case EMetricType::DSUMMARY: {
  238. auto summary = MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u);
  239. encoder->OnSummaryDouble(now, summary);
  240. break;
  241. }
  242. default:
  243. Y_ABORT("unexpected metric type [%s]", ToString(metricType).c_str());
  244. }
  245. encoder->OnMetricEnd();
  246. }
  247. encoder->OnStreamEnd();
  248. encoder->Close();
  249. };
  250. auto doTest = [&](bool buffered, EMetricType metricType) {
  251. TString json;
  252. TStringOutput out(json);
  253. auto encoder = buffered ? BufferedEncoderCloudJson(&out, 2) : EncoderCloudJson(&out, 2);
  254. const TString expectedMessage = TStringBuilder()
  255. << "metric type '" << metricType << "' is not supported by cloud json format";
  256. UNIT_ASSERT_EXCEPTION_CONTAINS_C(emit(encoder.Get(), metricType), yexception, expectedMessage,
  257. TString("buffered: ") + ToString(buffered));
  258. };
  259. doTest(false, EMetricType::HIST);
  260. doTest(false, EMetricType::LOGHIST);
  261. doTest(false, EMetricType::DSUMMARY);
  262. doTest(true, EMetricType::HIST);
  263. doTest(true, EMetricType::LOGHIST);
  264. doTest(true, EMetricType::DSUMMARY);
  265. }
  266. Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) {
  267. TString json;
  268. TStringOutput out(json);
  269. auto e = BufferedEncoderJson(&out, 2);
  270. e->OnStreamBegin();
  271. {
  272. e->OnMetricBegin(EMetricType::RATE);
  273. {
  274. e->OnLabelsBegin();
  275. e->OnLabel("metric", "hello");
  276. e->OnLabel("label", "world");
  277. e->OnLabelsEnd();
  278. }
  279. e->OnUint64(TInstant::Zero(), 0);
  280. e->OnMetricEnd();
  281. }
  282. {
  283. e->OnMetricBegin(EMetricType::RATE);
  284. {
  285. e->OnLabelsBegin();
  286. e->OnLabel("label", "world");
  287. e->OnLabel("metric", "hello");
  288. e->OnLabelsEnd();
  289. }
  290. e->OnUint64(TInstant::Zero(), 1);
  291. e->OnMetricEnd();
  292. }
  293. e->OnStreamEnd();
  294. e->Close();
  295. json += "\n";
  296. TString expectedJson = NResource::Find("/merged.json");
  297. // we cannot be sure regarding the label order in the result,
  298. // so we'll have to parse the expected value and then compare it with actual
  299. NProto::TMultiSamplesList samples;
  300. IMetricEncoderPtr d = EncoderProtobuf(&samples);
  301. DecodeJson(expectedJson, d.Get());
  302. UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
  303. {
  304. const NProto::TMultiSample& s = samples.GetSamples(0);
  305. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
  306. AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}});
  307. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  308. AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
  309. }
  310. }
  311. Y_UNIT_TEST(Decode1) {
  312. NProto::TMultiSamplesList samples;
  313. {
  314. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  315. TString testJson = NResource::Find("/expected.json");
  316. DecodeJson(testJson, e.Get());
  317. }
  318. UNIT_ASSERT_VALUES_EQUAL(
  319. TInstant::MilliSeconds(samples.GetCommonTime()),
  320. TInstant::Seconds(1500000000));
  321. UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1);
  322. AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
  323. UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6);
  324. {
  325. const NProto::TMultiSample& s = samples.GetSamples(0);
  326. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
  327. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  328. AssertLabelEqual(s.GetLabels(0), "metric", "single");
  329. AssertLabelEqual(s.GetLabels(1), "labels", "l1");
  330. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  331. AssertPointEqual(s.GetPoints(0), now, ui64(17));
  332. }
  333. {
  334. const NProto::TMultiSample& s = samples.GetSamples(1);
  335. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
  336. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  337. AssertLabelEqual(s.GetLabels(0), "metric", "single");
  338. AssertLabelEqual(s.GetLabels(1), "labels", "l2");
  339. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  340. AssertPointEqual(s.GetPoints(0), now, ui64(17));
  341. }
  342. {
  343. const NProto::TMultiSample& s = samples.GetSamples(2);
  344. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  345. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  346. AssertLabelEqual(s.GetLabels(0), "metric", "single");
  347. AssertLabelEqual(s.GetLabels(1), "labels", "l3");
  348. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  349. AssertPointEqual(s.GetPoints(0), now, 3.14);
  350. }
  351. {
  352. const NProto::TMultiSample& s = samples.GetSamples(3);
  353. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE);
  354. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  355. AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge");
  356. AssertLabelEqual(s.GetLabels(1), "labels", "l4");
  357. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  358. AssertPointEqual(s.GetPoints(0), now, i64(42));
  359. }
  360. {
  361. const NProto::TMultiSample& s = samples.GetSamples(4);
  362. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  363. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  364. AssertLabelEqual(s.GetLabels(0), "metric", "multiple");
  365. AssertLabelEqual(s.GetLabels(1), "labels", "l5");
  366. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3);
  367. AssertPointEqualNan(s.GetPoints(0), now);
  368. AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1);
  369. AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11);
  370. }
  371. {
  372. const NProto::TMultiSample& s = samples.GetSamples(5);
  373. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
  374. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  375. AssertLabelEqual(s.GetLabels(0), "metric", "multiple");
  376. AssertLabelEqual(s.GetLabels(1), "labels", "l6");
  377. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  378. AssertPointEqual(s.GetPoints(0), now, ui64(1337));
  379. AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338));
  380. }
  381. }
  382. Y_UNIT_TEST(DecodeMetrics) {
  383. NProto::TMultiSamplesList samples;
  384. {
  385. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  386. TString metricsJson = NResource::Find("/metrics.json");
  387. DecodeJson(metricsJson, e.Get());
  388. }
  389. UNIT_ASSERT_VALUES_EQUAL(
  390. TInstant::MilliSeconds(samples.GetCommonTime()),
  391. TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z"));
  392. UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3);
  393. AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
  394. AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man");
  395. AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile");
  396. UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4);
  397. {
  398. const NProto::TMultiSample& s = samples.GetSamples(0);
  399. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  400. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  401. AssertLabelEqual(s.GetLabels(0), "metric", "Memory");
  402. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  403. AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0);
  404. }
  405. {
  406. const NProto::TMultiSample& s = samples.GetSamples(1);
  407. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
  408. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  409. AssertLabelEqual(s.GetLabels(0), "metric", "UserTime");
  410. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  411. AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
  412. }
  413. {
  414. const NProto::TMultiSample& s = samples.GetSamples(2);
  415. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  416. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  417. AssertLabelEqual(s.GetLabels(0), "export", "Oxygen");
  418. AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize");
  419. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  420. auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z");
  421. AssertPointEqual(s.GetPoints(0), ts, 3.14159);
  422. }
  423. {
  424. const NProto::TMultiSample& s = samples.GetSamples(3);
  425. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  426. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  427. AssertLabelEqual(s.GetLabels(0), "metric", "Writes");
  428. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  429. auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z");
  430. AssertPointEqual(s.GetPoints(0), ts1, -10.0);
  431. auto ts2 = TInstant::Seconds(1503923187);
  432. AssertPointEqual(s.GetPoints(1), ts2, 20.0);
  433. }
  434. }
  435. Y_UNIT_TEST(DecodeSensors) {
  436. NProto::TMultiSamplesList samples;
  437. {
  438. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  439. TString sensorsJson = NResource::Find("/sensors.json");
  440. DecodeJson(sensorsJson, e.Get());
  441. }
  442. UNIT_ASSERT_VALUES_EQUAL(
  443. TInstant::MilliSeconds(samples.GetCommonTime()),
  444. TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z"));
  445. UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3);
  446. AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
  447. AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man");
  448. AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile");
  449. UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4);
  450. {
  451. const NProto::TMultiSample& s = samples.GetSamples(0);
  452. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  453. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  454. AssertLabelEqual(s.GetLabels(0), "metric", "Memory");
  455. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  456. AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0);
  457. }
  458. {
  459. const NProto::TMultiSample& s = samples.GetSamples(1);
  460. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
  461. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  462. AssertLabelEqual(s.GetLabels(0), "metric", "UserTime");
  463. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  464. AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
  465. }
  466. {
  467. const NProto::TMultiSample& s = samples.GetSamples(2);
  468. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  469. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  470. AssertLabelEqual(s.GetLabels(0), "export", "Oxygen");
  471. AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize");
  472. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  473. auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z");
  474. AssertPointEqual(s.GetPoints(0), ts, 3.14159);
  475. }
  476. {
  477. const NProto::TMultiSample& s = samples.GetSamples(3);
  478. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
  479. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  480. AssertLabelEqual(s.GetLabels(0), "metric", "Writes");
  481. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  482. auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z");
  483. AssertPointEqual(s.GetPoints(0), ts1, -10.0);
  484. auto ts2 = TInstant::Seconds(1503923187);
  485. AssertPointEqual(s.GetPoints(1), ts2, 20.0);
  486. }
  487. }
  488. Y_UNIT_TEST(DecodeToEncoder) {
  489. auto testJson = NResource::Find("/test_decode_to_encode.json");
  490. TStringStream Stream_;
  491. auto encoder = BufferedEncoderJson(&Stream_, 4);
  492. DecodeJson(testJson, encoder.Get());
  493. encoder->Close();
  494. auto val1 = NJson::ReadJsonFastTree(testJson, true);
  495. auto val2 = NJson::ReadJsonFastTree(Stream_.Str(), true);
  496. UNIT_ASSERT_VALUES_EQUAL(val1, val2);
  497. }
  498. void WriteEmptySeries(const IMetricEncoderPtr& e) {
  499. e->OnStreamBegin();
  500. {
  501. e->OnMetricBegin(EMetricType::COUNTER);
  502. {
  503. e->OnLabelsBegin();
  504. e->OnLabel("foo", "bar");
  505. e->OnLabelsEnd();
  506. }
  507. e->OnMetricEnd();
  508. }
  509. e->OnStreamEnd();
  510. e->Close();
  511. }
  512. Y_UNIT_TEST(EncodeEmptySeries) {
  513. TString json;
  514. TStringOutput out(json);
  515. auto e = EncoderJson(&out, 2);
  516. WriteEmptySeries(e);
  517. json += "\n";
  518. TString expectedJson = NResource::Find("/empty_series.json");
  519. UNIT_ASSERT_NO_DIFF(json, expectedJson);
  520. }
  521. void WriteEmptyLabels(IMetricEncoderPtr& e) {
  522. e->OnStreamBegin();
  523. e->OnMetricBegin(EMetricType::COUNTER);
  524. e->OnLabelsBegin();
  525. UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception);
  526. }
  527. Y_UNIT_TEST(LabelsCannotBeEmpty) {
  528. TString json;
  529. TStringOutput out(json);
  530. auto e = EncoderJson(&out, 2);
  531. WriteEmptyLabels(e);
  532. }
  533. Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) {
  534. TString json;
  535. TStringOutput out(json);
  536. auto e = BufferedEncoderJson(&out, 2);
  537. WriteEmptyLabels(e);
  538. }
  539. Y_UNIT_TEST(EncodeEmptySeriesBuffered) {
  540. TString json;
  541. TStringOutput out(json);
  542. auto e = BufferedEncoderJson(&out, 2);
  543. WriteEmptySeries(e);
  544. json += "\n";
  545. TString expectedJson = NResource::Find("/empty_series.json");
  546. UNIT_ASSERT_NO_DIFF(json, expectedJson);
  547. }
  548. Y_UNIT_TEST(BufferedEncoderMergesMetrics) {
  549. TString json;
  550. TStringOutput out(json);
  551. auto e = BufferedEncoderJson(&out, 2);
  552. auto ts = 1;
  553. auto writeMetric = [&] (const TString& val) {
  554. e->OnMetricBegin(EMetricType::COUNTER);
  555. e->OnLabelsBegin();
  556. e->OnLabel("foo", val);
  557. e->OnLabelsEnd();
  558. e->OnUint64(TInstant::Seconds(ts++), 42);
  559. e->OnMetricEnd();
  560. };
  561. e->OnStreamBegin();
  562. writeMetric("bar");
  563. writeMetric("bar");
  564. writeMetric("baz");
  565. writeMetric("bar");
  566. e->OnStreamEnd();
  567. e->Close();
  568. json += "\n";
  569. TString expectedJson = NResource::Find("/buffered_test.json");
  570. UNIT_ASSERT_NO_DIFF(json, expectedJson);
  571. }
  572. Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) {
  573. TStringStream out;
  574. auto e = EncoderJson(&out);
  575. auto writePreamble = [&] {
  576. e->OnStreamBegin();
  577. e->OnMetricBegin(EMetricType::COUNTER);
  578. e->OnLabelsBegin();
  579. e->OnLabel("foo", "bar");
  580. e->OnLabelsEnd();
  581. };
  582. // writing two values for a metric in a row will trigger
  583. // timeseries object construction
  584. writePreamble();
  585. e->OnUint64(TInstant::Zero(), 42);
  586. UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception);
  587. e = EncoderJson(&out);
  588. writePreamble();
  589. e->OnUint64(TInstant::Zero(), 42);
  590. UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception);
  591. e = EncoderJson(&out);
  592. writePreamble();
  593. e->OnUint64(TInstant::Now(), 42);
  594. UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception);
  595. }
  596. Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) {
  597. TStringStream out;
  598. {
  599. auto e = BufferedEncoderJson(&out, 2);
  600. e->OnStreamBegin();
  601. e->OnMetricBegin(EMetricType::COUNTER);
  602. e->OnLabelsBegin();
  603. e->OnLabel("foo", "bar");
  604. e->OnLabelsEnd();
  605. // in buffered mode we are able to find values with same (in this case zero)
  606. // timestamp and discard duplicates
  607. e->OnUint64(TInstant::Zero(), 42);
  608. e->OnUint64(TInstant::Zero(), 43);
  609. e->OnUint64(TInstant::Zero(), 44);
  610. e->OnUint64(TInstant::Zero(), 45);
  611. e->OnMetricEnd();
  612. e->OnStreamEnd();
  613. }
  614. out << "\n";
  615. UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json"));
  616. }
  617. template <typename TFactory, typename TConsumer>
  618. TString EncodeToString(TFactory factory, TConsumer consumer) {
  619. TStringStream out;
  620. {
  621. IMetricEncoderPtr e = factory(&out, 2);
  622. consumer(e.Get());
  623. }
  624. out << '\n';
  625. return out.Str();
  626. }
  627. Y_UNIT_TEST(SummaryValueEncode) {
  628. auto writeDocument = [](IMetricEncoder* e) {
  629. e->OnStreamBegin();
  630. {
  631. e->OnMetricBegin(EMetricType::DSUMMARY);
  632. {
  633. e->OnLabelsBegin();
  634. e->OnLabel("metric", "temperature");
  635. e->OnLabelsEnd();
  636. }
  637. e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u));
  638. e->OnMetricEnd();
  639. }
  640. e->OnStreamEnd();
  641. };
  642. TString result1 = EncodeToString(EncoderJson, writeDocument);
  643. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json"));
  644. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  645. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json"));
  646. }
  647. ISummaryDoubleSnapshotPtr TestInfSummary() {
  648. return MakeIntrusive<TSummaryDoubleSnapshot>(
  649. std::numeric_limits<double>::quiet_NaN(),
  650. -std::numeric_limits<double>::infinity(),
  651. std::numeric_limits<double>::infinity(),
  652. 0.3,
  653. 30u);
  654. }
  655. Y_UNIT_TEST(SummaryInfEncode) {
  656. auto writeDocument = [](IMetricEncoder* e) {
  657. e->OnStreamBegin();
  658. {
  659. e->OnMetricBegin(EMetricType::DSUMMARY);
  660. {
  661. e->OnLabelsBegin();
  662. e->OnLabel("metric", "temperature");
  663. e->OnLabelsEnd();
  664. }
  665. e->OnSummaryDouble(now, TestInfSummary());
  666. e->OnMetricEnd();
  667. }
  668. e->OnStreamEnd();
  669. };
  670. TString result1 = EncodeToString(EncoderJson, writeDocument);
  671. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json"));
  672. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  673. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json"));
  674. }
  675. Y_UNIT_TEST(SummaryInfDecode) {
  676. NProto::TMultiSamplesList samples;
  677. {
  678. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  679. TString testJson = NResource::Find("/summary_inf.json");
  680. DecodeJson(testJson, e.Get());
  681. }
  682. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  683. const NProto::TMultiSample& s = samples.GetSamples(0);
  684. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
  685. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  686. AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
  687. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  688. auto actual = s.GetPoints(0).GetSummaryDouble();
  689. UNIT_ASSERT(std::isnan(actual.GetSum()));
  690. UNIT_ASSERT(actual.GetMin() < 0);
  691. UNIT_ASSERT(std::isinf(actual.GetMin()));
  692. UNIT_ASSERT(actual.GetMax() > 0);
  693. UNIT_ASSERT(std::isinf(actual.GetMax()));
  694. }
  695. Y_UNIT_TEST(SummaryValueDecode) {
  696. NProto::TMultiSamplesList samples;
  697. {
  698. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  699. TString testJson = NResource::Find("/summary_value.json");
  700. DecodeJson(testJson, e.Get());
  701. }
  702. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  703. const NProto::TMultiSample& s = samples.GetSamples(0);
  704. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
  705. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  706. AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
  707. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  708. auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u);
  709. AssertPointEqual(s.GetPoints(0), now, snapshot);
  710. }
  711. Y_UNIT_TEST(SummaryTimeSeriesEncode) {
  712. auto writeDocument = [](IMetricEncoder* e) {
  713. e->OnStreamBegin();
  714. {
  715. e->OnMetricBegin(EMetricType::DSUMMARY);
  716. {
  717. e->OnLabelsBegin();
  718. e->OnLabel("metric", "temperature");
  719. e->OnLabelsEnd();
  720. }
  721. TSummaryDoubleCollector summary;
  722. summary.Collect(0.3);
  723. summary.Collect(-0.5);
  724. summary.Collect(1.);
  725. e->OnSummaryDouble(now, summary.Snapshot());
  726. summary.Collect(-1.5);
  727. summary.Collect(0.01);
  728. e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot());
  729. e->OnMetricEnd();
  730. }
  731. e->OnStreamEnd();
  732. };
  733. TString result1 = EncodeToString(EncoderJson, writeDocument);
  734. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json"));
  735. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  736. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json"));
  737. }
  738. Y_UNIT_TEST(SummaryTimeSeriesDecode) {
  739. NProto::TMultiSamplesList samples;
  740. {
  741. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  742. TString testJson = NResource::Find("/summary_timeseries.json");
  743. DecodeJson(testJson, e.Get());
  744. }
  745. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  746. const NProto::TMultiSample& s = samples.GetSamples(0);
  747. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
  748. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  749. AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
  750. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  751. TSummaryDoubleCollector summary;
  752. summary.Collect(0.3);
  753. summary.Collect(-0.5);
  754. summary.Collect(1.);
  755. AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot());
  756. summary.Collect(-1.5);
  757. summary.Collect(0.01);
  758. AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot());
  759. }
  760. Y_UNIT_TEST(LogHistogramValueEncode) {
  761. auto writeDocument = [](IMetricEncoder* e) {
  762. e->OnStreamBegin();
  763. {
  764. e->OnMetricBegin(EMetricType::LOGHIST);
  765. {
  766. e->OnLabelsBegin();
  767. e->OnLabel("metric", "ms");
  768. e->OnLabelsEnd();
  769. }
  770. e->OnLogHistogram(now, TestLogHistogram());
  771. e->OnMetricEnd();
  772. }
  773. e->OnStreamEnd();
  774. };
  775. TString result1 = EncodeToString(EncoderJson, writeDocument);
  776. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json"));
  777. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  778. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json"));
  779. }
  780. Y_UNIT_TEST(LogHistogramValueDecode) {
  781. NProto::TMultiSamplesList samples;
  782. {
  783. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  784. TString testJson = NResource::Find("/log_histogram_value.json");
  785. DecodeJson(testJson, e.Get());
  786. }
  787. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  788. const NProto::TMultiSample& s = samples.GetSamples(0);
  789. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
  790. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  791. AssertLabelEqual(s.GetLabels(0), "metric", "ms");
  792. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  793. auto snapshot = TestLogHistogram();
  794. AssertPointEqual(s.GetPoints(0), now, *snapshot);
  795. }
  796. Y_UNIT_TEST(HistogramValueEncode) {
  797. auto writeDocument = [](IMetricEncoder* e) {
  798. e->OnStreamBegin();
  799. {
  800. e->OnMetricBegin(EMetricType::HIST);
  801. {
  802. e->OnLabelsBegin();
  803. e->OnLabel("metric", "responseTimeMillis");
  804. e->OnLabelsEnd();
  805. }
  806. // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83}
  807. auto h = ExponentialHistogram(6, 2);
  808. for (i64 i = 1; i < 100; i++) {
  809. h->Collect(i);
  810. }
  811. e->OnHistogram(now, h->Snapshot());
  812. e->OnMetricEnd();
  813. }
  814. e->OnStreamEnd();
  815. };
  816. TString result1 = EncodeToString(EncoderJson, writeDocument);
  817. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json"));
  818. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  819. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json"));
  820. }
  821. Y_UNIT_TEST(LogHistogramTimeSeriesEncode) {
  822. auto writeDocument = [](IMetricEncoder* e) {
  823. e->OnStreamBegin();
  824. {
  825. e->OnMetricBegin(EMetricType::LOGHIST);
  826. {
  827. e->OnLabelsBegin();
  828. e->OnLabel("metric", "ms");
  829. e->OnLabelsEnd();
  830. }
  831. e->OnLogHistogram(now, TestLogHistogram(1));;
  832. e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2));
  833. e->OnMetricEnd();
  834. }
  835. e->OnStreamEnd();
  836. };
  837. TString result1 = EncodeToString(EncoderJson, writeDocument);
  838. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json"));
  839. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  840. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json"));
  841. }
  842. Y_UNIT_TEST(LogHistogramTimeSeriesDecode) {
  843. NProto::TMultiSamplesList samples;
  844. {
  845. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  846. TString testJson = NResource::Find("/log_histogram_timeseries.json");
  847. DecodeJson(testJson, e.Get());
  848. }
  849. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  850. const NProto::TMultiSample& s = samples.GetSamples(0);
  851. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
  852. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  853. AssertLabelEqual(s.GetLabels(0), "metric", "ms");
  854. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  855. auto logHist = TestLogHistogram(1);
  856. AssertPointEqual(s.GetPoints(0), now, *logHist);
  857. logHist = TestLogHistogram(2);
  858. AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist);
  859. }
  860. void HistogramValueDecode(const TString& filePath) {
  861. NProto::TMultiSamplesList samples;
  862. {
  863. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  864. TString testJson = NResource::Find(filePath);
  865. DecodeJson(testJson, e.Get());
  866. }
  867. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  868. const NProto::TMultiSample& s = samples.GetSamples(0);
  869. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM);
  870. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  871. AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis");
  872. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
  873. auto h = ExponentialHistogram(6, 2);
  874. for (i64 i = 1; i < 100; i++) {
  875. h->Collect(i);
  876. }
  877. AssertPointEqual(s.GetPoints(0), now, *h->Snapshot());
  878. }
  879. Y_UNIT_TEST(HistogramValueDecode) {
  880. HistogramValueDecode("/histogram_value.json");
  881. HistogramValueDecode("/histogram_value_inf_before_bounds.json");
  882. }
  883. Y_UNIT_TEST(HistogramTimeSeriesEncode) {
  884. auto writeDocument = [](IMetricEncoder* e) {
  885. e->OnStreamBegin();
  886. {
  887. e->OnMetricBegin(EMetricType::HIST_RATE);
  888. {
  889. e->OnLabelsBegin();
  890. e->OnLabel("metric", "responseTimeMillis");
  891. e->OnLabelsEnd();
  892. }
  893. // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83}
  894. auto h = ExponentialHistogram(6, 2);
  895. for (i64 i = 1; i < 100; i++) {
  896. h->Collect(i);
  897. }
  898. e->OnHistogram(now, h->Snapshot());
  899. // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166}
  900. for (i64 i = 1; i < 100; i++) {
  901. h->Collect(i);
  902. }
  903. e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot());
  904. e->OnMetricEnd();
  905. }
  906. e->OnStreamEnd();
  907. };
  908. TString result1 = EncodeToString(EncoderJson, writeDocument);
  909. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json"));
  910. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  911. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json"));
  912. }
  913. Y_UNIT_TEST(HistogramTimeSeriesDecode) {
  914. NProto::TMultiSamplesList samples;
  915. {
  916. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  917. TString testJson = NResource::Find("/histogram_timeseries.json");
  918. DecodeJson(testJson, e.Get());
  919. }
  920. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  921. const NProto::TMultiSample& s = samples.GetSamples(0);
  922. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE);
  923. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  924. AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis");
  925. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
  926. auto h = ExponentialHistogram(6, 2);
  927. for (i64 i = 1; i < 100; i++) {
  928. h->Collect(i);
  929. }
  930. AssertPointEqual(s.GetPoints(0), now, *h->Snapshot());
  931. for (i64 i = 1; i < 100; i++) {
  932. h->Collect(i);
  933. }
  934. AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot());
  935. }
  936. Y_UNIT_TEST(IntGaugeEncode) {
  937. auto writeDocument = [](IMetricEncoder* e) {
  938. e->OnStreamBegin();
  939. {
  940. e->OnMetricBegin(EMetricType::IGAUGE);
  941. {
  942. e->OnLabelsBegin();
  943. e->OnLabel("metric", "a");
  944. e->OnLabelsEnd();
  945. }
  946. e->OnInt64(now, Min<i64>());
  947. e->OnInt64(now + TDuration::Seconds(1), -1);
  948. e->OnInt64(now + TDuration::Seconds(2), 0);
  949. e->OnInt64(now + TDuration::Seconds(3), Max<i64>());
  950. e->OnMetricEnd();
  951. }
  952. e->OnStreamEnd();
  953. };
  954. TString result1 = EncodeToString(EncoderJson, writeDocument);
  955. UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json"));
  956. TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
  957. UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json"));
  958. }
  959. Y_UNIT_TEST(InconsistentMetricTypes) {
  960. auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) {
  961. encoder.OnMetricBegin(EMetricType::GAUGE);
  962. {
  963. encoder.OnLabelsBegin();
  964. encoder.OnLabel("name", "m");
  965. encoder.OnLabel("l1", "v1");
  966. encoder.OnLabel("l2", "v2");
  967. encoder.OnLabelsEnd();
  968. }
  969. encoder.OnDouble(now, 1.0);
  970. encoder.OnMetricEnd();
  971. encoder.OnMetricBegin(EMetricType::COUNTER);
  972. {
  973. encoder.OnLabelsBegin();
  974. encoder.OnLabel("name", "m");
  975. encoder.OnLabel("l1", "v1");
  976. encoder.OnLabel("l2", "v2");
  977. encoder.OnLabelsEnd();
  978. }
  979. encoder.OnUint64(now, 1);
  980. UNIT_ASSERT_EXCEPTION_CONTAINS(encoder.OnMetricEnd(),
  981. yexception,
  982. expectedError);
  983. };
  984. {
  985. TStringStream out;
  986. auto encoder = BufferedEncoderJson(&out);
  987. encoder->OnStreamBegin();
  988. encoder->OnLabelsBegin();
  989. encoder->OnLabel("c", "cv");
  990. encoder->OnLabelsEnd();
  991. emitMetrics(*encoder,
  992. "Time series point type mismatch: expected DOUBLE but found UINT64, "
  993. "labels '{c=cv, l1=v1, l2=v2, name=m}'");
  994. }
  995. {
  996. TStringStream out;
  997. auto encoder = BufferedEncoderJson(&out);
  998. encoder->OnStreamBegin();
  999. encoder->OnLabelsBegin();
  1000. encoder->OnLabel("l1", "v100");
  1001. encoder->OnLabelsEnd();
  1002. emitMetrics(*encoder,
  1003. "Time series point type mismatch: expected DOUBLE but found UINT64, "
  1004. "labels '{l1=v1, l2=v2, name=m}'");
  1005. }
  1006. }
  1007. Y_UNIT_TEST(IntGaugeDecode) {
  1008. NProto::TMultiSamplesList samples;
  1009. {
  1010. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  1011. TString testJson = NResource::Find("/int_gauge.json");
  1012. DecodeJson(testJson, e.Get());
  1013. }
  1014. UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
  1015. const NProto::TMultiSample& s = samples.GetSamples(0);
  1016. UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE);
  1017. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  1018. AssertLabelEqual(s.GetLabels(0), "metric", "a");
  1019. UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4);
  1020. AssertPointEqual(s.GetPoints(0), now, Min<i64>());
  1021. AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1));
  1022. AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0));
  1023. AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>());
  1024. }
  1025. Y_UNIT_TEST(FuzzerRegression) {
  1026. NProto::TMultiSamplesList samples;
  1027. {
  1028. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  1029. for (auto f : { "/hist_crash.json", "/crash.json" }) {
  1030. TString testJson = NResource::Find(f);
  1031. UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception);
  1032. }
  1033. }
  1034. }
  1035. Y_UNIT_TEST(LegacyNegativeRateThrows) {
  1036. const auto input = R"({
  1037. "sensors": [
  1038. {
  1039. "mode": "deriv",
  1040. "value": -1,
  1041. "labels": { "metric": "SystemTime" }
  1042. },
  1043. }
  1044. ]}")";
  1045. NProto::TMultiSamplesList samples;
  1046. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  1047. UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception);
  1048. }
  1049. Y_UNIT_TEST(DecodeNamedMetrics) {
  1050. NProto::TMultiSamplesList samples;
  1051. {
  1052. IMetricEncoderPtr e = EncoderProtobuf(&samples);
  1053. TString metricsJson = NResource::Find("/named_metrics.json");
  1054. DecodeJson(metricsJson, e.Get(), "sensor");
  1055. }
  1056. UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2);
  1057. {
  1058. const NProto::TMultiSample& s = samples.GetSamples(0);
  1059. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
  1060. AssertLabelEqual(s.GetLabels(0), "sensor", "Memory");
  1061. }
  1062. {
  1063. const NProto::TMultiSample& s = samples.GetSamples(1);
  1064. UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
  1065. AssertLabelEqual(s.GetLabels(0), "sensor", "QueueSize");
  1066. AssertLabelEqual(s.GetLabels(1), "export", "Oxygen");
  1067. }
  1068. }
  1069. }