manage prometheus

a543f08a6f2957757c54b3f9e5a095389a51dbeb

library/cpp/monlib/encode/prometheus/prometheus.h

@@ -11,8 +11,17 @@ namespace NMonitoring {
     class TPrometheusDecodeException: public yexception {
     };
 
+    enum class EPrometheusDecodeMode {
+        DEFAULT,
+        RAW
+    };
+
+    struct TPrometheusDecodeSettings {
+        EPrometheusDecodeMode Mode{EPrometheusDecodeMode::DEFAULT};
+    };
+
     IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor");
 
-    void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor");
+    void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor", const TPrometheusDecodeSettings& settings = TPrometheusDecodeSettings{});
 
 }

library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp

@@ -71,6 +71,9 @@ namespace NMonitoring {
             using TBucketData = std::pair<TBucketBound, TBucketValue>;
             constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 };
         public:
+            THistogramBuilder(TPrometheusDecodeSettings settings)
+                : Settings_(settings) {
+            }
             TStringBuf GetName() const noexcept {
                 return Name_;
             }
@@ -125,7 +128,6 @@ namespace NMonitoring {
 
                 Bounds_.push_back(bound);
                 Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets
-
                 PrevBucket_ = { bound, value };
             }
 
@@ -150,6 +152,7 @@ namespace NMonitoring {
             TBucketBounds Bounds_;
             TBucketValues Values_;
             TBucketData PrevBucket_ = ZERO_BUCKET;
+            TPrometheusDecodeSettings Settings_;
         };
 
         ///////////////////////////////////////////////////////////////////////
@@ -168,10 +171,12 @@ namespace NMonitoring {
         ///////////////////////////////////////////////////////////////////////
         class TPrometheusReader {
         public:
-            TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel)
+            TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, const TPrometheusDecodeSettings& settings = TPrometheusDecodeSettings{})
                 : Data_(data)
                 , Consumer_(c)
                 , MetricNameLabel_(metricNameLabel)
+                , Settings_(settings)
+                , HistogramBuilder_(settings)
             {
             }
 
@@ -272,12 +277,14 @@ namespace NMonitoring {
                 TStringBuf baseName = name;
                 EPrometheusMetricType type = EPrometheusMetricType::UNTYPED;
 
-                if (auto* seenType = SeenTypes_.FindPtr(name)) {
-                    type = *seenType;
-                } else {
-                    baseName = NPrometheus::ToBaseName(name);
-                    if (auto* baseType = SeenTypes_.FindPtr(baseName)) {
-                        type = *baseType;
+                if (Settings_.Mode != EPrometheusDecodeMode::RAW) {
+                    if (auto* seenType = SeenTypes_.FindPtr(name)) {
+                        type = *seenType;
+                    } else {
+                        baseName = NPrometheus::ToBaseName(name);
+                        if (auto* baseType = SeenTypes_.FindPtr(baseName)) {
+                            type = *baseType;
+                        }
                     }
                 }
 
@@ -584,6 +591,7 @@ namespace NMonitoring {
             TStringBuf Data_;
             IMetricConsumer* Consumer_;
             TStringBuf MetricNameLabel_;
+            TPrometheusDecodeSettings Settings_;
             THashMap<TString, EPrometheusMetricType> SeenTypes_;
             THistogramBuilder HistogramBuilder_;
 
@@ -593,8 +601,8 @@ namespace NMonitoring {
         };
     } // namespace
 
-void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) {
-    TPrometheusReader reader(data, c, metricNameLabel);
+void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, const TPrometheusDecodeSettings& settings) {
+    TPrometheusReader reader(data, c, metricNameLabel, settings);
     reader.Read();
 }
 

library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp

@@ -36,11 +36,12 @@ using namespace NMonitoring;
 
 Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) {
 
-    NProto::TSingleSamplesList Decode(TStringBuf data) {
+    NProto::TSingleSamplesList Decode(TStringBuf data, const TPrometheusDecodeSettings& settings = TPrometheusDecodeSettings{}) {
         NProto::TSingleSamplesList samples;
+        ;
         {
             IMetricEncoderPtr e = EncoderProtobuf(&samples);
-            DecodePrometheus(data, e.Get());
+            DecodePrometheus(data, e.Get(), "sensor", settings);
         }
         return samples;
     }
@@ -97,33 +98,61 @@ Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) {
     }
 
     Y_UNIT_TEST(Counter) {
-        auto samples = Decode(
-                "# A normal comment.\n"
-                "#\n"
-                "# TYPE name counter\n"
-                "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n"
-                "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n"
-                "# HELP name two-line\\n doc  str\\\\ing\n");
-
-        UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
-
-        {
-            auto& s = samples.GetSamples(0);
-            UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
-            UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
-            ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
-            ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
-            ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1");
-            ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0));
-        }
-        {
-            auto& s = samples.GetSamples(1);
-            UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
-            UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
-            ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
-            ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
-            ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
-            ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2));
+        constexpr auto inputMetrics =
+            "# A normal comment.\n"
+            "#\n"
+            "# TYPE name counter\n"
+            "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n"
+            "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n"
+            "# HELP name two-line\\n doc  str\\\\ing\n";
+
+        {
+            auto samples = Decode(inputMetrics);
+            UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+            {
+                auto& s = samples.GetSamples(0);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+                ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+                ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1");
+                ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0));
+            }
+            {
+                auto& s = samples.GetSamples(1);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+                ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+                ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+                ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2));
+            }
+        }
+        {
+            TPrometheusDecodeSettings settings;
+            settings.Mode = EPrometheusDecodeMode::RAW;
+            auto samples = Decode(inputMetrics, settings);
+            UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+            {
+                auto& s = samples.GetSamples(0);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+                ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+                ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1");
+                ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(0), NAN);
+            }
+            {
+                auto& s = samples.GetSamples(1);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+                ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+                ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+                ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 2.3);
+            }
         }
     }
 
@@ -210,43 +239,58 @@ Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) {
     }
 
     Y_UNIT_TEST(Histogram) {
-        auto samples = Decode(
-                "# HELP request_duration_microseconds The response latency.\n"
-                "# TYPE request_duration_microseconds histogram\n"
-                "request_duration_microseconds_bucket{le=\"0\"} 0\n"
-                "request_duration_microseconds_bucket{le=\"100\"} 123\n"
-                "request_duration_microseconds_bucket{le=\"120\"} 412\n"
-                "request_duration_microseconds_bucket{le=\"144\"} 592\n"
-                "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n"
-                "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n"
-                "request_duration_microseconds_sum 1.7560473e+06\n"
-                "request_duration_microseconds_count 2693\n");
-
-        UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
-
-        {
-            auto& s = samples.GetSamples(0);
-            UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
-            UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
-            ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum");
-            ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3);
-        }
-        {
-            auto& s = samples.GetSamples(1);
-            UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
-            UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
-            ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count");
-            ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
-        }
-        {
-            auto& s = samples.GetSamples(2);
-            UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
-            UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
-            ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds");
-            auto hist = ExplicitHistogramSnapshot(
-                    { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND },
-                    { 0, 123, 289, 180, 932, 1169 });
-            ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+        constexpr auto inputMetrics =
+            "# HELP request_duration_microseconds The response latency.\n"
+            "# TYPE request_duration_microseconds histogram\n"
+            "request_duration_microseconds_bucket{le=\"0\"} 0\n"
+            "request_duration_microseconds_bucket{le=\"100\"} 123\n"
+            "request_duration_microseconds_bucket{le=\"120\"} 412\n"
+            "request_duration_microseconds_bucket{le=\"144\"} 592\n"
+            "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n"
+            "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n"
+            "request_duration_microseconds_sum 1.7560473e+06\n"
+            "request_duration_microseconds_count 2693\n";
+
+        {
+            auto samples = Decode(inputMetrics);
+            {
+                auto& s = samples.GetSamples(0);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum");
+                ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3);
+            }
+            {
+                auto& s = samples.GetSamples(1);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count");
+                ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
+            }
+            {
+                auto& s = samples.GetSamples(2);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+                UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds");
+                auto hist = ExplicitHistogramSnapshot(
+                    {0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND},
+                    {0, 123, 289, 180, 932, 1169});
+                ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+            }
+        }
+        {
+            TPrometheusDecodeSettings settings;
+            settings.Mode = EPrometheusDecodeMode::RAW;
+            auto samples = Decode(inputMetrics, settings);
+            UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8);
+            {
+                auto& s = samples.GetSamples(0);
+                UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+                UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+                ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_bucket");
+                ASSERT_LABEL_EQUAL(s.GetLabels(1), "le", "0");
+                ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 0);
+            }
         }
     }
 

library/cpp/monlib/encode/prometheus/ya.make

@@ -10,6 +10,8 @@ PEERDIR(
     library/cpp/monlib/encode/buffered
 )
 
+GENERATE_ENUM_SERIALIZATION_WITH_HEADER(prometheus.h)
+
 END()
 
 RECURSE(