Metric correlations (#12582)

* initial attempt at metric correlations

* fix loop

* simplify struct

* change json

* get points from query

* comment

* dont lock the host as much

* add a configuration option to enable/disable metric correlations

* remove KSfbar from header file

* lock charts

* add timeout

* cast multiplication

* add licencing info

* better licencing

* use onewayalloc

* destroy owa

CMakeLists.txt

@@ -676,6 +676,10 @@ set(RRD_PLUGIN_FILES
         database/engine/metadata_log/metalogpluginsd.h
         database/engine/metadata_log/compaction.c
         database/engine/metadata_log/compaction.h
+        database/KolmogorovSmirnovDist.c
+        database/KolmogorovSmirnovDist.h
+        database/metric_correlations.c
+        database/metric_correlations.h
         )
 
 set(WEB_PLUGIN_FILES

Makefile.am

@@ -460,6 +460,10 @@ RRD_PLUGIN_FILES = \
     database/sqlite/sqlite_aclk_alert.h \
     database/sqlite/sqlite3.c \
     database/sqlite/sqlite3.h \
+    database/KolmogorovSmirnovDist.c \
+    database/KolmogorovSmirnovDist.h \
+    database/metric_correlations.c \
+    database/metric_correlations.h \
     $(NULL)
 
 if ENABLE_DBENGINE

REDISTRIBUTED.md

@@ -180,4 +180,9 @@ connectivity is not available.
 
     Copyright 2015, Benedikt Schmitt [Unlicense License](https://unlicense.org/)
 
+-   [Kolmogorov-Smirnov distribution](http://simul.iro.umontreal.ca/ksdir/)
+
+    Copyright March 2010 by Université de Montréal, Richard Simard and Pierre L'Ecuyer
+    [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)
+
 

daemon/common.h

@@ -84,6 +84,9 @@
 #include "commands.h"
 #include "analytics.h"
 
+// metric correlations
+#include "database/metric_correlations.h"
+
 // global netdata daemon variables
 extern char *netdata_configured_hostname;
 extern char *netdata_configured_user_config_dir;

daemon/main.c

@@ -553,6 +553,10 @@ static void get_netdata_configured_variables() {
     enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm);
 #endif
 
+    // --------------------------------------------------------------------
+    // metric correlations
+    enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
+
     // --------------------------------------------------------------------
     // get various system parameters
 

database/KolmogorovSmirnovDist.c

@@ -0,0 +1,788 @@
+// SPDX-License-Identifier: GPL-3.0
+
+/********************************************************************
+ *
+ * File:          KolmogorovSmirnovDist.c
+ * Environment:   ISO C99 or ANSI C89
+ * Author:        Richard Simard
+ * Organization:  DIRO, Université de Montréal
+ * Date:          1 February 2012
+ * Version        1.1
+
+ * Copyright 1 march 2010 by Université de Montréal,
+                             Richard Simard and Pierre L'Ecuyer
+ =====================================================================
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, version 3 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+ =====================================================================*/
+
+#include "KolmogorovSmirnovDist.h"
+#include <math.h>
+#include <stdlib.h>
+
+#define num_Pi     3.14159265358979323846 /* PI */
+#define num_Ln2    0.69314718055994530941 /* log(2) */
+
+/* For x close to 0 or 1, we use the exact formulae of Ruben-Gambino in all
+   cases. For n <= NEXACT, we use exact algorithms: the Durbin matrix and
+   the Pomeranz algorithms. For n > NEXACT, we use asymptotic methods
+   except for x close to 0 where we still use the method of Durbin
+   for n <= NKOLMO. For n > NKOLMO, we use asymptotic methods only and
+   so the precision is less for x close to 0.
+   We could increase the limit NKOLMO to 10^6 to get better precision
+   for x close to 0, but at the price of a slower speed. */
+#define NEXACT 500
+#define NKOLMO 100000
+
+/* The Durbin matrix algorithm for the Kolmogorov-Smirnov distribution */
+static double DurbinMatrix (int n, double d);
+
+
+/*========================================================================*/
+#if 0
+
+/* For ANSI C89 only, not for ISO C99 */
+#define MAXI 50
+#define EPSILON 1.0e-15
+
+double log1p (double x)
+{
+   /* returns a value equivalent to log(1 + x) accurate also for small x. */
+   if (fabs (x) > 0.1) {
+      return log (1.0 + x);
+   } else {
+      double term = x;
+      double sum = x;
+      int s = 2;
+      while ((fabs (term) > EPSILON * fabs (sum)) && (s < MAXI)) {
+         term *= -x;
+         sum += term / s;
+         s++;
+      }
+      return sum;
+   }
+}
+
+#undef MAXI
+#undef EPSILON
+
+#endif
+
+/*========================================================================*/
+#define MFACT 30
+
+/* The natural logarithm of factorial n! for  0 <= n <= MFACT */
+static double LnFactorial[MFACT + 1] = {
+   0.,
+   0.,
+   0.6931471805599453,
+   1.791759469228055,
+   3.178053830347946,
+   4.787491742782046,
+   6.579251212010101,
+   8.525161361065415,
+   10.60460290274525,
+   12.80182748008147,
+   15.10441257307552,
+   17.50230784587389,
+   19.98721449566188,
+   22.55216385312342,
+   25.19122118273868,
+   27.89927138384088,
+   30.67186010608066,
+   33.50507345013688,
+   36.39544520803305,
+   39.33988418719949,
+   42.33561646075348,
+   45.3801388984769,
+   48.47118135183522,
+   51.60667556776437,
+   54.7847293981123,
+   58.00360522298051,
+   61.26170176100199,
+   64.55753862700632,
+   67.88974313718154,
+   71.257038967168,
+   74.65823634883016
+};
+
+/*------------------------------------------------------------------------*/
+
+static double getLogFactorial (int n)
+{
+   /* Returns the natural logarithm of factorial n! */
+   if (n <= MFACT) {
+      return LnFactorial[n];
+
+   } else {
+      double x = (double) (n + 1);
+      double y = 1.0 / (x * x);
+      double z = ((-(5.95238095238E-4 * y) + 7.936500793651E-4) * y -
+         2.7777777777778E-3) * y + 8.3333333333333E-2;
+      z = ((x - 0.5) * log (x) - x) + 9.1893853320467E-1 + z / x;
+      return z;
+   }
+}
+
+/*------------------------------------------------------------------------*/
+
+static double rapfac (int n)
+{
+   /* Computes n! / n^n */
+   int i;
+   double res = 1.0 / n;
+   for (i = 2; i <= n; i++) {
+      res *= (double) i / n;
+   }
+   return res;
+}
+
+
+/*========================================================================*/
+
+static double **CreateMatrixD (int N, int M)
+{
+   int i;
+   double **T2;
+
+   T2 = (double **) malloc (N * sizeof (double *));
+   T2[0] = (double *) malloc ((size_t) N * M * sizeof (double));
+   for (i = 1; i < N; i++)
+      T2[i] = T2[0] + i * M;
+   return T2;
+}
+
+
+static void DeleteMatrixD (double **T)
+{
+   free (T[0]);
+   free (T);
+}
+
+
+/*========================================================================*/
+
+static double KSPlusbarAsymp (int n, double x)
+{
+   /* Compute the probability of the KS+ distribution using an asymptotic
+      formula */
+   double t = (6.0 * n * x + 1);
+   double z = t * t / (18.0 * n);
+   double v = 1.0 - (2.0 * z * z - 4.0 * z - 1.0) / (18.0 * n);
+   if (v <= 0.0)
+      return 0.0;
+   v = v * exp (-z);
+   if (v >= 1.0)
+      return 1.0;
+   return v;
+}
+
+
+/*-------------------------------------------------------------------------*/
+
+static double KSPlusbarUpper (int n, double x)
+{
+   /* Compute the probability of the KS+ distribution in the upper tail using
+      Smirnov's stable formula */
+   const double EPSILON = 1.0E-12;
+   double q;
+   double Sum = 0.0;
+   double term;
+   double t;
+   double LogCom;
+   double LOGJMAX;
+   int j;
+   int jdiv;
+   int jmax = (int) (n * (1.0 - x));
+
+   if (n > 200000)
+      return KSPlusbarAsymp (n, x);
+
+   /* Avoid log(0) for j = jmax and q ~ 1.0 */
+   if ((1.0 - x - (double) jmax / n) <= 0.0)
+      jmax--;
+
+   if (n > 3000)
+      jdiv = 2;
+   else
+      jdiv = 3;
+
+   j = jmax / jdiv + 1;
+   LogCom = getLogFactorial (n) - getLogFactorial (j) -
+            getLogFactorial (n - j);
+   LOGJMAX = LogCom;
+
+   while (j <= jmax) {
+      q = (double) j / n + x;
+      term = LogCom + (j - 1) * log (q) + (n - j) * log1p (-q);
+      t = exp (term);
+      Sum += t;
+      LogCom += log ((double) (n - j) / (j + 1));
+      if (t <= Sum * EPSILON)
+         break;
+      j++;
+   }
+
+   j = jmax / jdiv;
+   LogCom = LOGJMAX + log ((double) (j + 1) / (n - j));
+
+   while (j > 0) {
+      q = (double) j / n + x;
+      term = LogCom + (j - 1) * log (q) + (n - j) * log1p (-q);
+      t = exp (term);
+      Sum += t;
+      LogCom += log ((double) j / (n - j + 1));
+      if (t <= Sum * EPSILON)
+         break;
+      j--;
+   }
+
+   Sum *= x;
+   /* add the term j = 0 */
+   Sum += exp (n * log1p (-x));
+   return Sum;
+}
+
+
+/*========================================================================*/
+
+static double Pelz (int n, double x)
+{
+   /* Approximating the Lower Tail-Areas of the Kolmogorov-Smirnov One-Sample
+      Statistic,
+      Wolfgang Pelz and I. J. Good,
+      Journal of the Royal Statistical Society, Series B.
+      Vol. 38, No. 2 (1976), pp. 152-156
+   */
+
+   const int JMAX = 20;
+   const double EPS = 1.0e-10;
+   const double C = 2.506628274631001;         /* sqrt(2*Pi) */
+   const double C2 = 1.2533141373155001;       /* sqrt(Pi/2) */
+   const double PI2 = num_Pi * num_Pi;
+   const double PI4 = PI2 * PI2;
+   const double RACN = sqrt ((double) n);
+   const double z = RACN * x;
+   const double z2 = z * z;
+   const double z4 = z2 * z2;
+   const double z6 = z4 * z2;
+   const double w = PI2 / (2.0 * z * z);
+   double ti, term, tom;
+   double sum;
+   int j;
+
+   term = 1;
+   j = 0;
+   sum = 0;
+   while (j <= JMAX && term > EPS * sum) {
+      ti = j + 0.5;
+      term = exp (-ti * ti * w);
+      sum += term;
+      j++;
+   }
+   sum *= C / z;
+
+   term = 1;
+   tom = 0;
+   j = 0;
+   while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
+      ti = j + 0.5;
+      term = (PI2 * ti * ti - z2) * exp (-ti * ti * w);
+      tom += term;
+      j++;
+   }
+   sum += tom * C2 / (RACN * 3.0 * z4);
+
+   term = 1;
+   tom = 0;
+   j = 0;
+   while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
+      ti = j + 0.5;
+      term = 6 * z6 + 2 * z4 + PI2 * (2 * z4 - 5 * z2) * ti * ti +
+         PI4 * (1 - 2 * z2) * ti * ti * ti * ti;
+      term *= exp (-ti * ti * w);
+      tom += term;
+      j++;
+   }
+   sum += tom * C2 / (n * 36.0 * z * z6);
+
+   term = 1;
+   tom = 0;
+   j = 1;
+   while (j <= JMAX && term > EPS * tom) {
+      ti = j;
+      term = PI2 * ti * ti * exp (-ti * ti * w);
+      tom += term;
+      j++;
+   }
+   sum -= tom * C2 / (n * 18.0 * z * z2);
+
+   term = 1;
+   tom = 0;
+   j = 0;
+   while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
+      ti = j + 0.5;
+      ti = ti * ti;
+      term = -30 * z6 - 90 * z6 * z2 + PI2 * (135 * z4 - 96 * z6) * ti +
+         PI4 * (212 * z4 - 60 * z2) * ti * ti + PI2 * PI4 * ti * ti * ti * (5 -
+         30 * z2);
+      term *= exp (-ti * w);
+      tom += term;
+      j++;
+   }
+   sum += tom * C2 / (RACN * n * 3240.0 * z4 * z6);
+
+   term = 1;
+   tom = 0;
+   j = 1;
+   while (j <= JMAX && fabs (term) > EPS * fabs (tom)) {
+      ti = j * j;
+      term = (3 * PI2 * ti * z2 - PI4 * ti * ti) * exp (-ti * w);
+      tom += term;
+      j++;
+   }
+   sum += tom * C2 / (RACN * n * 108.0 * z6);
+
+   return sum;
+}
+
+
+/*=========================================================================*/
+
+static void CalcFloorCeil (
+   int n,                         /* sample size */
+   double t,                      /* = nx */
+   double *A,                     /* A_i */
+   double *Atflo,                 /* floor (A_i - t) */
+   double *Atcei                  /* ceiling (A_i + t) */
+   )
+{
+   /* Precompute A_i, floors, and ceilings for limits of sums in the Pomeranz
+      algorithm */
+   int i;
+   int ell = (int) t;             /* floor (t) */
+   double z = t - ell;            /* t - floor (t) */
+   double w = ceil (t) - t;
+
+   if (z > 0.5) {
+      for (i = 2; i <= 2 * n + 2; i += 2)
+         Atflo[i] = i / 2 - 2 - ell;
+      for (i = 1; i <= 2 * n + 2; i += 2)
+         Atflo[i] = i / 2 - 1 - ell;
+
+      for (i = 2; i <= 2 * n + 2; i += 2)
+         Atcei[i] = i / 2 + ell;
+      for (i = 1; i <= 2 * n + 2; i += 2)
+         Atcei[i] = i / 2 + 1 + ell;
+
+   } else if (z > 0.0) {
+      for (i = 1; i <= 2 * n + 2; i++)
+         Atflo[i] = i / 2 - 1 - ell;
+
+      for (i = 2; i <= 2 * n + 2; i++)
+         Atcei[i] = i / 2 + ell;
+      Atcei[1] = 1 + ell;
+
+   } else {                       /* z == 0 */
+      for (i = 2; i <= 2 * n + 2; i += 2)
+         Atflo[i] = i / 2 - 1 - ell;
+      for (i = 1; i <= 2 * n + 2; i += 2)
+         Atflo[i] = i / 2 - ell;
+
+      for (i = 2; i <= 2 * n + 2; i += 2)
+         Atcei[i] = i / 2 - 1 + ell;
+      for (i = 1; i <= 2 * n + 2; i += 2)
+         Atcei[i] = i / 2 + ell;
+   }
+
+   if (w < z)
+      z = w;
+   A[0] = A[1] = 0;
+   A[2] = z;
+   A[3] = 1 - A[2];
+   for (i = 4; i <= 2 * n + 1; i++)
+      A[i] = A[i - 2] + 1;
+   A[2 * n + 2] = n;
+}
+
+
+/*========================================================================*/
+
+static double Pomeranz (int n, double x)
+{
+   /* The Pomeranz algorithm to compute the KS distribution */
+   const double EPS = 1.0e-15;
+   const int ENO = 350;
+   const double RENO = ldexp (1.0, ENO); /* for renormalization of V */
+   int coreno;                    /* counter: how many renormalizations */
+   const double t = n * x;
+   double w, sum, minsum;
+   int i, j, k, s;
+   int r1, r2;                    /* Indices i and i-1 for V[i][] */
+   int jlow, jup, klow, kup, kup0;
+   double *A;
+   double *Atflo;
+   double *Atcei;
+   double **V;
+   double **H;                    /* = pow(w, j) / Factorial(j) */
+
+   A = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
+   Atflo = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
+   Atcei = (double *) calloc ((size_t) (2 * n + 3), sizeof (double));
+   V = (double **) CreateMatrixD (2, n + 2);
+   H = (double **) CreateMatrixD (4, n + 2);
+
+   CalcFloorCeil (n, t, A, Atflo, Atcei);
+
+   for (j = 1; j <= n + 1; j++)
+      V[0][j] = 0;
+   for (j = 2; j <= n + 1; j++)
+      V[1][j] = 0;
+   V[1][1] = RENO;
+   coreno = 1;
+
+   /* Precompute H[][] = (A[j] - A[j-1]^k / k! for speed */
+   H[0][0] = 1;
+   w = 2.0 * A[2] / n;
+   for (j = 1; j <= n + 1; j++)
+      H[0][j] = w * H[0][j - 1] / j;
+
+   H[1][0] = 1;
+   w = (1.0 - 2.0 * A[2]) / n;
+   for (j = 1; j <= n + 1; j++)
+      H[1][j] = w * H[1][j - 1] / j;
+
+   H[2][0] = 1;
+   w = A[2] / n;
+   for (j = 1; j <= n + 1; j++)
+      H[2][j] = w * H[2][j - 1] / j;
+
+   H[3][0] = 1;
+   for (j = 1; j <= n + 1; j++)
+      H[3][j] = 0;
+
+   r1 = 0;
+   r2 = 1;
+   for (i = 2; i <= 2 * n + 2; i++) {
+      jlow = 2 + (int) Atflo[i];
+      if (jlow < 1)
+         jlow = 1;
+      jup = (int) Atcei[i];
+      if (jup > n + 1)
+         jup = n + 1;
+
+      klow = 2 + (int) Atflo[i - 1];
+      if (klow < 1)
+         klow = 1;
+      kup0 = (int) Atcei[i - 1];
+
+      /* Find to which case it corresponds */
+      w = (A[i] - A[i - 1]) / n;
+      s = -1;
+      for (j = 0; j < 4; j++) {
+         if (fabs (w - H[j][1]) <= EPS) {
+            s = j;
+            break;
+         }
+      }
+      /* assert (s >= 0, "Pomeranz: s < 0"); */
+
+      minsum = RENO;
+      r1 = (r1 + 1) & 1;          /* i - 1 */
+      r2 = (r2 + 1) & 1;          /* i */
+
+      for (j = jlow; j <= jup; j++) {
+         kup = kup0;
+         if (kup > j)
+            kup = j;
+         sum = 0;
+         for (k = kup; k >= klow; k--)
+            sum += V[r1][k] * H[s][j - k];
+         V[r2][j] = sum;
+         if (sum < minsum)
+            minsum = sum;
+      }
+
+      if (minsum < 1.0e-280) {
+         /* V is too small: renormalize to avoid underflow of probabilities */
+         for (j = jlow; j <= jup; j++)
+            V[r2][j] *= RENO;
+         coreno++;                /* keep track of log of RENO */
+      }
+   }
+
+   sum = V[r2][n + 1];
+   free (A);
+   free (Atflo);
+   free (Atcei);
+   DeleteMatrixD (H);
+   DeleteMatrixD (V);
+   w = getLogFactorial (n) - coreno * ENO * num_Ln2 + log (sum);
+   if (w >= 0.)
+      return 1.;
+   return exp (w);
+}
+
+
+/*========================================================================*/
+
+static double cdfSpecial (int n, double x)
+{
+   /* The KS distribution is known exactly for these cases */
+
+   /* For nx^2 > 18, KSfbar(n, x) is smaller than 5e-16 */
+   if ((n * x * x >= 18.0) || (x >= 1.0))
+      return 1.0;
+
+   if (x <= 0.5 / n)
+      return 0.0;
+
+   if (n == 1)
+      return 2.0 * x - 1.0;
+
+   if (x <= 1.0 / n) {
+      double t = 2.0 * x * n - 1.0;
+      double w;
+      if (n <= NEXACT) {
+         w = rapfac (n);
+         return w * pow (t, (double) n);
+      }
+      w = getLogFactorial (n) + n * log (t / n);
+      return exp (w);
+   }
+
+   if (x >= 1.0 - 1.0 / n) {
+      return 1.0 - 2.0 * pow (1.0 - x, (double) n);
+   }
+
+   return -1.0;
+}
+
+
+/*========================================================================*/
+
+double KScdf (int n, double x)
+{
+   const double w = n * x * x;
+   double u = cdfSpecial (n, x);
+   if (u >= 0.0)
+      return u;
+
+   if (n <= NEXACT) {
+      if (w < 0.754693)
+         return DurbinMatrix (n, x);
+      if (w < 4.0)
+         return Pomeranz (n, x);
+      return 1.0 - KSfbar (n, x);
+   }
+
+   if ((w * x * n <= 7.0) && (n <= NKOLMO))
+      return DurbinMatrix (n, x);
+
+   return Pelz (n, x);
+}
+
+
+/*=========================================================================*/
+
+static double fbarSpecial (int n, double x)
+{
+   const double w = n * x * x;
+
+   if ((w >= 370.0) || (x >= 1.0))
+      return 0.0;
+   if ((w <= 0.0274) || (x <= 0.5 / n))
+      return 1.0;
+   if (n == 1)
+      return 2.0 - 2.0 * x;
+
+   if (x <= 1.0 / n) {
+      double z;
+      double t = 2.0 * x * n - 1.0;
+      if (n <= NEXACT) {
+         z = rapfac (n);
+         return 1.0 - z * pow (t, (double) n);
+      }
+      z = getLogFactorial (n) + n * log (t / n);
+      return 1.0 - exp (z);
+   }
+
+   if (x >= 1.0 - 1.0 / n) {
+      return 2.0 * pow (1.0 - x, (double) n);
+   }
+   return -1.0;
+}
+
+
+/*========================================================================*/
+
+double KSfbar (int n, double x)
+{
+   const double w = n * x * x;
+   double v = fbarSpecial (n, x);
+   if (v >= 0.0)
+      return v;
+
+   if (n <= NEXACT) {
+      if (w < 4.0)
+         return 1.0 - KScdf (n, x);
+      else
+         return 2.0 * KSPlusbarUpper (n, x);
+   }
+
+   if (w >= 2.65)
+      return 2.0 * KSPlusbarUpper (n, x);
+
+   return 1.0 - KScdf (n, x);
+}
+
+
+/*=========================================================================
+
+The following implements the Durbin matrix algorithm and was programmed by
+G. Marsaglia, Wai Wan Tsang and Jingbo Wong.
+
+I have made small modifications in their program. (Richard Simard)
+
+
+
+=========================================================================*/
+
+/*
+ The C program to compute Kolmogorov's distribution
+
+             K(n,d) = Prob(D_n < d),         where
+
+      D_n = max(x_1-0/n,x_2-1/n...,x_n-(n-1)/n,1/n-x_1,2/n-x_2,...,n/n-x_n)
+
+    with  x_1<x_2,...<x_n  a purported set of n independent uniform [0,1)
+    random variables sorted into increasing order.
+    See G. Marsaglia, Wai Wan Tsang and Jingbo Wong,
+       J.Stat.Software, 8, 18, pp 1--4, (2003).
+*/
+
+#define NORM 1.0e140
+#define INORM 1.0e-140
+#define LOGNORM 140
+
+
+/* Matrix product */
+static void mMultiply (double *A, double *B, double *C, int m);
+
+/* Matrix power */
+static void mPower (double *A, int eA, double *V, int *eV, int m, int n);
+
+
+static double DurbinMatrix (int n, double d)
+{
+   int k, m, i, j, g, eH, eQ;
+   double h, s, *H, *Q;
+   /* OMIT NEXT TWO LINES IF YOU REQUIRE >7 DIGIT ACCURACY IN THE RIGHT TAIL */
+#if 0
+   s = d * d * n;
+   if (s > 7.24 || (s > 3.76 && n > 99))
+      return 1 - 2 * exp (-(2.000071 + .331 / sqrt (n) + 1.409 / n) * s);
+#endif
+   k = (int) (n * d) + 1;
+   m = 2 * k - 1;
+   h = k - n * d;
+   H = (double *) malloc ((m * m) * sizeof (double));
+   Q = (double *) malloc ((m * m) * sizeof (double));
+   for (i = 0; i < m; i++)
+      for (j = 0; j < m; j++)
+         if (i - j + 1 < 0)
+            H[i * m + j] = 0;
+         else
+            H[i * m + j] = 1;
+   for (i = 0; i < m; i++) {
+      H[i * m] -= pow (h, (double) (i + 1));
+      H[(m - 1) * m + i] -= pow (h, (double) (m - i));
+   }
+   H[(m - 1) * m] += (2 * h - 1 > 0 ? pow (2 * h - 1, (double) m) : 0);
+   for (i = 0; i < m; i++)
+      for (j = 0; j < m; j++)
+         if (i - j + 1 > 0)
+            for (g = 1; g <= i - j + 1; g++)
+               H[i * m + j] /= g;
+   eH = 0;
+   mPower (H, eH, Q, &eQ, m, n);
+   s = Q[(k - 1) * m + k - 1];
+
+   for (i = 1; i <= n; i++) {
+      s = s * (double) i / n;
+      if (s < INORM) {
+         s *= NORM;
+         eQ -= LOGNORM;
+      }
+   }
+   s *= pow (10., (double) eQ);
+   free (H);
+   free (Q);
+   return s;
+}
+
+
+static void mMultiply (double *A, double *B, double *C, int m)
+{
+   int i, j, k;
+   double s;
+   for (i = 0; i < m; i++)
+      for (j = 0; j < m; j++) {
+         s = 0.;
+         for (k = 0; k < m; k++)
+            s += A[i * m + k] * B[k * m + j];
+         C[i * m + j] = s;
+      }
+}
+
+
+static void renormalize (double *V, int m, int *p)
+{
+   int i;
+   for (i = 0; i < m * m; i++)
+      V[i] *= INORM;
+   *p += LOGNORM;
+}
+
+
+static void mPower (double *A, int eA, double *V, int *eV, int m, int n)
+{
+   double *B;
+   int eB, i;
+   if (n == 1) {
+      for (i = 0; i < m * m; i++)
+         V[i] = A[i];
+      *eV = eA;
+      return;
+   }
+   mPower (A, eA, V, eV, m, n / 2);
+   B = (double *) malloc ((m * m) * sizeof (double));
+   mMultiply (V, V, B, m);
+   eB = 2 * (*eV);
+   if (B[(m / 2) * m + (m / 2)] > NORM)
+      renormalize (B, m, &eB);
+
+   if (n % 2 == 0) {
+      for (i = 0; i < m * m; i++)
+         V[i] = B[i];
+      *eV = eB;
+   } else {
+      mMultiply (A, B, V, m);
+      *eV = eA + eB;
+   }
+
+   if (V[(m / 2) * m + (m / 2)] > NORM)
+      renormalize (V, m, eV);
+   free (B);
+}

database/KolmogorovSmirnovDist.h

@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-3.0
+
+#ifndef KOLMOGOROVSMIRNOVDIST_H
+#define KOLMOGOROVSMIRNOVDIST_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/********************************************************************
+ *
+ * File:          KolmogorovSmirnovDist.h
+ * Environment:   ISO C99 or ANSI C89
+ * Author:        Richard Simard
+ * Organization:  DIRO, Université de Montréal
+ * Date:          1 February 2012
+ * Version        1.1
+ *
+ * Copyright March 2010 by Université de Montréal,
+                           Richard Simard and Pierre L'Ecuyer
+ =====================================================================
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, version 3 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+ =====================================================================*/
+/*
+ *
+ * The Kolmogorov-Smirnov test statistic D_n is defined by
+ *
+ *        D_n = sup_x |F(x) - S_n(x)|
+ *
+ * where n is the sample size, F(x) is a completely specified theoretical
+ * distribution, and S_n(x) is an empirical distribution function.
+ *
+ *
+ * The function
+ *
+ *        double KScdf (int n, double x);
+ *
+ * computes the cumulative probability P[D_n <= x] of the 2-sided 1-sample
+ * Kolmogorov-Smirnov distribution with sample size n at x.
+ * It returns at least 13 decimal digits of precision for n <= 500,
+ * at least 7 decimal digits of precision for 500 < n <= 100000,
+ * and a few correct decimal digits for n > 100000.
+ *
+ */
+
+double KScdf (int n, double x);
+
+
+/*
+ * The function
+ *
+ *        double KSfbar (int n, double x);
+ *
+ * computes the complementary cumulative probability P[D_n >= x] of the
+ * 2-sided 1-sample Kolmogorov-Smirnov distribution with sample size n at x.
+ * It returns at least 10 decimal digits of precision for n <= 500,
+ * at least 6 decimal digits of precision for 500 < n <= 200000,
+ * and a few correct decimal digits for n > 200000.
+ *
+ */
+
+double KSfbar (int n, double x);
+
+
+/*
+ * NOTE:
+ * The ISO C99 function log1p of the standard math library does not exist in
+ * ANSI C89. Here, it is programmed explicitly in KolmogorovSmirnovDist.c.
+
+ * For ANSI C89 compilers, change the preprocessor condition to make it
+ * available.
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

database/metric_correlations.c

@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+#include "KolmogorovSmirnovDist.h"
+
+#define MAX_POINTS 10000
+int enable_metric_correlations = CONFIG_BOOLEAN_YES;
+
+struct charts {
+    RRDSET *st;
+    struct charts *next;
+};
+
+struct per_dim {
+    char *dimension;
+    calculated_number baseline[MAX_POINTS];
+    calculated_number highlight[MAX_POINTS];
+
+    double baseline_diffs[MAX_POINTS];
+    double highlight_diffs[MAX_POINTS];
+};
+
+int find_index(double arr[], long int n, double K, long int start)
+{
+    for (long int i = start; i < n; i++) {
+        if (K<arr[i]){
+            return i;
+        }
+    }
+    return n;
+}
+
+int compare(const void *left, const void *right) {
+    double lt = *(double *)left;
+    double rt = *(double *)right;
+
+    if(unlikely(lt < rt)) return -1;
+    if(unlikely(lt > rt)) return 1;
+    return 0;
+}
+
+void kstwo(double data1[], long int n1, double data2[], long int n2, double *d, double *prob)
+{
+	double en1, en2, en, data_all[MAX_POINTS*2], cdf1[MAX_POINTS], cdf2[MAX_POINTS], cddiffs[MAX_POINTS];
+    double min = 0.0, max = 0.0;
+    qsort(data1, n1, sizeof(double), compare);
+    qsort(data2, n2, sizeof(double), compare);
+
+    for (int i = 0; i < n1; i++)
+        data_all[i] = data1[i];
+    for (int i = 0; i < n2; i++)
+        data_all[n1 + i] = data2[i];
+
+    en1 = (double)n1;
+	en2 = (double)n2;
+    *d = 0.0;
+    cddiffs[0]=0; //for uninitialized warning
+
+    for (int i=0; i<n1+n2;i++)
+        cdf1[i] = find_index(data1, n1, data_all[i], 0) / en1; //TODO, use the start to reduce loops
+
+    for (int i=0; i<n1+n2;i++)
+        cdf2[i] = find_index(data2, n2, data_all[i], 0) / en2;
+
+    for ( int i=0;i<n2+n1;i++)
+        cddiffs[i] = cdf1[i] - cdf2[i];
+
+    min = cddiffs[0];
+    for ( int i=0;i<n2+n1;i++) {
+        if (cddiffs[i] < min)
+            min = cddiffs[i];
+    }
+
+    //clip min
+    if (fabs(min) < 0) min = 0;
+    else if (fabs(min) > 1) min = 1;
+
+    max = fabs(cddiffs[0]);
+    for ( int i=0;i<n2+n1;i++)
+        if (cddiffs[i] >= max) max = cddiffs[i];
+
+    if (fabs(min) < max)
+        *d = max;
+    else
+        *d = fabs(min);
+
+    
+    
+    en = (en1*en2 / (en1 + en2));
+    *prob = KSfbar(round(en), *d);
+}
+
+void fill_nan (struct per_dim *d, long int hp, long int bp)
+{
+    int k;
+
+    for (k = 0; k < bp; k++) {
+        if (isnan(d->baseline[k])) {
+            d->baseline[k] = 0.0;
+        }
+    }
+
+    for (k = 0; k < hp; k++) {
+        if (isnan(d->highlight[k])) {
+            d->highlight[k] = 0.0;
+        }
+    }
+}
+
+//TODO check counters
+void run_diffs_and_rev (struct per_dim *d, long int hp, long int bp)
+{
+    int k, j;
+
+    for (k = 0, j = bp; k < bp - 1; k++, j--)
+        d->baseline_diffs[k] = (double)d->baseline[j - 2] - (double)d->baseline[j - 1];
+    for (k = 0, j = hp; k < hp - 1; k++, j--) {
+        d->highlight_diffs[k] = (double)d->highlight[j - 2] - (double)d->highlight[j - 1];
+    }
+}
+
+int run_metric_correlations (BUFFER *wb, RRDSET *st, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
+{
+    uint32_t options = 0x00000000;
+    int group_method = RRDR_GROUPING_AVERAGE;
+    long group_time = 0;
+    struct context_param  *context_param_list = NULL;
+    long c;
+    int i=0, j=0;
+    int b_dims = 0;
+    long int baseline_points = 0, highlight_points = 0;
+
+    struct per_dim *pd = NULL;
+
+    //TODO get everything in one go, when baseline is right before highlight
+    //get baseline
+    ONEWAYALLOC *owa = onewayalloc_create(0);
+    RRDR *rb = rrd2rrdr(owa, st, max_points, baseline_after, baseline_before, group_method, group_time, options, NULL, context_param_list, 0);
+    if(!rb) {
+        info("Cannot generate metric correlations output with these parameters on this chart.");
+        onewayalloc_destroy(owa);
+        return 0;
+    } else {
+        baseline_points = rrdr_rows(rb);
+        pd = mallocz(sizeof(struct per_dim) * rb->d);
+        b_dims = rb->d;
+        for (c = 0; c != rrdr_rows(rb) ; ++c) {
+            RRDDIM *d;
+            for (j = 0, d = rb->st->dimensions ; d && j < rb->d ; ++j, d = d->next) {
+                calculated_number *cn = &rb->v[ c * rb->d ];
+                if (!c) {
+                    //TODO use points from query
+                    pd[j].dimension = strdupz (d->name);
+                    pd[j].baseline[c] = cn[j];
+                } else {
+                    pd[j].baseline[c] = cn[j];
+                }
+            }
+        }
+    }
+    rrdr_free(owa, rb);
+    onewayalloc_destroy(owa);
+    if (!pd)
+        return 0;
+
+    //get highlight
+    owa = onewayalloc_create(0);
+    RRDR *rh = rrd2rrdr(owa, st, max_points, highlight_after, highlight_before, group_method, group_time, options, NULL, context_param_list, 0);
+    if(!rh) {
+        info("Cannot generate metric correlations output with these parameters on this chart.");
+        freez(pd);
+        onewayalloc_destroy(owa);
+        return 0;
+    } else {
+        if (rh->d != b_dims) {
+            //TODO handle different dims
+            rrdr_free(owa, rh);
+            onewayalloc_destroy(owa);
+            freez(pd);
+            return 0;
+        }
+        highlight_points = rrdr_rows(rh);
+        for (c = 0; c != rrdr_rows(rh) ; ++c) {
+            RRDDIM *d;
+            for (j = 0, d = rh->st->dimensions ; d && j < rh->d ; ++j, d = d->next) {
+                calculated_number *cn = &rh->v[ c * rh->d ];
+                    pd[j].highlight[c] = cn[j];
+            }
+        }
+    }
+    rrdr_free(owa, rh);
+    onewayalloc_destroy(owa);
+
+    for (i = 0; i < b_dims; i++) {
+        fill_nan(&pd[i], highlight_points, baseline_points);
+    }
+
+    for (i = 0; i < b_dims; i++) {
+        run_diffs_and_rev(&pd[i], highlight_points, baseline_points);
+    }
+
+    double d=0, prob=0;
+    for (i=0;i < j ;i++) {
+        if (baseline_points && highlight_points) {
+            kstwo(pd[i].baseline_diffs, baseline_points-1, pd[i].highlight_diffs, highlight_points-1, &d, &prob);
+            buffer_sprintf(wb, "\t\t\t\t\"%s\": %f", pd[i].dimension, prob);
+            if (i != j-1)
+                buffer_sprintf(wb, ",\n");
+            else
+                buffer_sprintf(wb, "\n");
+        }
+    }
+
+    freez(pd);
+    return j;
+}
+
+void metric_correlations (RRDHOST *host, BUFFER *wb, long long baseline_after, long long baseline_before, long long highlight_after, long long highlight_before, long long max_points)
+{
+    info ("Running metric correlations, highlight_after: %lld, highlight_before: %lld, baseline_after: %lld, baseline_before: %lld, max_points: %lld", highlight_after, highlight_before, baseline_after, baseline_before, max_points);
+
+    if (!enable_metric_correlations) {
+        error("Metric correlations functionality is not enabled.");
+        buffer_strcat(wb, "{\"error\": \"Metric correlations functionality is not enabled.\" }");
+        return;
+    }
+
+    if (highlight_before <= highlight_after || baseline_before <= baseline_after) {
+        error("Invalid baseline or highlight ranges.");
+        buffer_strcat(wb, "{\"error\": \"Invalid baseline or highlight ranges.\" }");
+        return;
+    }
+
+    long long dims = 0, total_dims = 0;
+    RRDSET *st;
+    size_t c = 0;
+    BUFFER *wdims = buffer_create(1000);
+
+    if (!max_points || max_points > MAX_POINTS)
+        max_points = MAX_POINTS;
+
+    //dont lock here and wait for results
+    //get the charts and run mc after
+    //should not be a problem for the query
+    struct charts *charts = NULL;
+    rrdhost_rdlock(host);
+    rrdset_foreach_read(st, host) {
+        if (rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
+            struct charts *chart = callocz(1, sizeof(struct charts));
+            chart->st = st;
+            chart->next = NULL;
+            if (charts) {
+                chart->next = charts;
+            }
+            charts = chart;
+        }
+    }
+    rrdhost_unlock(host);
+
+    buffer_strcat(wb, "{\n\t\"correlated_charts\": {");
+
+    for (struct charts *ch = charts; ch; ch = ch->next) {
+        buffer_flush(wdims);
+        dims = run_metric_correlations(wdims, ch->st, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
+        if (dims) {
+            if (c)
+                buffer_strcat(wb, "\t\t},");
+            buffer_strcat(wb, "\n\t\t\"");
+            buffer_strcat(wb, ch->st->id);
+            buffer_strcat(wb, "\": {\n");
+            buffer_strcat(wb, "\t\t\t\"context\": \"");
+            buffer_strcat(wb, ch->st->context);
+            buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
+            buffer_sprintf(wb, "%s", buffer_tostring(wdims));
+            buffer_strcat(wb, "\t\t\t}\n");
+            total_dims += dims;
+            c++;
+        }
+    }
+    buffer_strcat(wb, "\t\t}\n");
+    buffer_sprintf(wb, "\t},\n\t\"total_dimensions_count\": %lld\n}", total_dims);
+
+    if (!total_dims) {
+        buffer_flush(wb);
+        buffer_strcat(wb, "{\"error\": \"No results from metric correlations.\" }");
+    }
+
+    struct charts* ch;
+    while(charts){
+        ch = charts;
+        charts = charts->next;
+        rrdset_unlock(ch->st);
+        free(ch);
+    }
+
+    buffer_free(wdims);
+    info ("Done running metric correlations");
+}

database/metric_correlations.h

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METRIC_CORRELATIONS_H
+#define NETDATA_METRIC_CORRELATIONS_H 1
+
+extern int enable_metric_correlations;
+
+void metric_correlations (RRDHOST *host, BUFFER *wb, long long selected_after, long long selected_before, long long reference_after, long long reference_before, long long max_points);
+
+#endif //NETDATA_METRIC_CORRELATIONS_H

web/api/web_api_v1.c

@@ -1299,6 +1299,50 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client
     return HTTP_RESP_OK;
 }
 
+int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) {
+    if (!netdata_ready)
+        return HTTP_RESP_BACKEND_FETCH_FAILED;
+
+    long long baseline_after = 0, baseline_before = 0, highlight_after = 0, highlight_before = 0, max_points = 0;
+ 
+    while (url) {
+        char *value = mystrsep(&url, "&");
+        if (!value || !*value)
+            continue;
+
+        char *name = mystrsep(&value, "=");
+        if (!name || !*name)
+            continue;
+        if (!value || !*value)
+            continue;
+
+        if (!strcmp(name, "baseline_after"))
+            baseline_after = (long long) strtoul(value, NULL, 0);
+        else if (!strcmp(name, "baseline_before"))
+            baseline_before = (long long) strtoul(value, NULL, 0);
+        else if (!strcmp(name, "highlight_after"))
+            highlight_after = (long long) strtoul(value, NULL, 0);
+        else if (!strcmp(name, "highlight_before"))
+            highlight_before = (long long) strtoul(value, NULL, 0);
+        else if (!strcmp(name, "max_points"))
+            max_points = (long long) strtoul(value, NULL, 0);
+        
+    }
+
+    BUFFER *wb = w->response.data;
+    buffer_flush(wb);
+    wb->contenttype = CT_APPLICATION_JSON;
+    buffer_no_cacheable(wb);
+
+    if (!highlight_after || !highlight_before)
+        buffer_strcat(wb, "{\"error\": \"Missing or invalid required highlight after and before parameters.\" }");
+    else {
+        metric_correlations(host, wb, baseline_after, baseline_before, highlight_after, highlight_before, max_points);
+    }
+
+    return HTTP_RESP_OK;
+}
+
 static struct api_command {
     const char *command;
     uint32_t hash;
@@ -1330,8 +1374,9 @@ static struct api_command {
         { "ml_info",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info            },
 #endif
 
-        { "manage/health",   0, WEB_CLIENT_ACL_MGMT,      web_client_api_request_v1_mgmt_health     },
-        { "aclk",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state      },
+        { "manage/health",       0, WEB_CLIENT_ACL_MGMT,      web_client_api_request_v1_mgmt_health         },
+        { "aclk",                0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state          },
+        { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations },
         // terminator
         { NULL,              0, WEB_CLIENT_ACL_NONE,      NULL                                      },
 };