recalculate last_collected_total (#13945)

collectors/plugins.d/pluginsd_parser.c

@@ -1056,8 +1056,6 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
 {
     char *last_collected_ut_str = get_word(words, num_words, 1);
     char *last_updated_ut_str = get_word(words, num_words, 2);
-    char *last_collected_total_str = get_word(words, num_words, 3);
-    char *collected_total_str = get_word(words, num_words, 4);
 
     RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
     RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
@@ -1082,9 +1080,6 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
         st->last_updated.tv_usec = last_updated_ut % USEC_PER_SEC;
     }
 
-    st->last_collected_total = last_collected_total_str ? strtoll(last_collected_total_str, NULL, 0) : 0;
-    st->collected_total = collected_total_str ? strtoll(collected_total_str, NULL, 0) : 0;
-
     st->counter++;
     st->counter_done++;
 

database/rrd.h

@@ -607,9 +607,6 @@ struct rrdset {
     struct timeval last_updated;                    // when this data set was last updated (updated every time the rrd_stats_done() function)
     struct timeval last_collected_time;             // when did this data set last collected values
 
-    total_number collected_total;                   // used internally to calculate percentages
-    total_number last_collected_total;              // used internally to calculate percentages
-
     size_t rrdlabels_last_saved_version;
 
     DICTIONARY *functions_view;                     // collector functions this rrdset supports, can be NULL

database/rrddim.c

@@ -601,7 +601,7 @@ struct rrddim_map_save_v019 {
     long double last_calculated_value;              // ignored
     long double last_stored_value;                  // ignored
     long long collected_value;                      // ignored
-    long long last_collected_value;                 // ignored
+    long long last_collected_value;                 // load and save
     long double collected_volume;                   // ignored
     long double stored_volume;                      // ignored
     void *next;                                     // ignored
@@ -624,6 +624,7 @@ void rrddim_memory_file_update(RRDDIM *rd) {
 
     rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
     rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
+    rd_on_file->last_collected_value = rd->last_collected_value;
 }
 
 void rrddim_memory_file_free(RRDDIM *rd) {
@@ -703,6 +704,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
     }
 
     if(!reset) {
+        rd->last_collected_value = rd_on_file->last_collected_value;
+
         if(rd_on_file->algorithm != rd->algorithm)
             info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
                  fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));

database/rrdset.c

@@ -1510,13 +1510,16 @@ after_first_database_work:
     if(unlikely(rrdhost_has_rrdpush_sender_enabled(st->rrdhost)))
         rrdset_done_push(st);
 
+    uint32_t has_reset_value = 0;
+
     size_t rda_slots = dictionary_entries(st->rrddim_root_index);
     struct rda_item *rda_base = rrdset_thread_rda(&rda_slots);
 
     size_t dim_id;
     size_t dimensions = 0;
     struct rda_item *rda = rda_base;
-    st->collected_total = 0;
+    total_number collected_total = 0;
+    total_number last_collected_total = 0;
     rrddim_foreach_read(rd, st) {
         if(rd_dfe.counter >= rda_slots)
             break;
@@ -1535,7 +1538,24 @@ after_first_database_work:
 
         // calculate totals
         if(likely(rd->updated)) {
-            st->collected_total += rd->collected_value;
+            // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
+            // to reset the calculation (it will give zero as the calculation for this second)
+            if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->last_collected_value > rd->collected_value)) {
+                debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
+                , rrdset_id(st)
+                , rrddim_name(rd)
+                , rd->last_collected_value
+                , rd->collected_value
+                );
+
+                if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
+                    has_reset_value = 1;
+
+                rd->last_collected_value = rd->collected_value;
+            }
+
+            last_collected_total += rd->last_collected_value;
+            collected_total += rd->collected_value;
 
             if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
                 error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
@@ -1554,8 +1574,6 @@ after_first_database_work:
     rrdset_debug(st, "last_stored_ut  = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
     rrdset_debug(st, "next_store_ut   = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
 
-    uint32_t has_reset_value = 0;
-
     // process all dimensions to calculate their values
     // based on the collected figures only
     // at this stage we do not interpolate anything
@@ -1599,7 +1617,7 @@ after_first_database_work:
                 break;
 
             case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
-                if(unlikely(!st->collected_total))
+                if(unlikely(!collected_total))
                     rd->calculated_value = 0;
                 else
                     // the percentage of the current value
@@ -1607,7 +1625,7 @@ after_first_database_work:
                     rd->calculated_value =
                             (NETDATA_DOUBLE)100
                             * (NETDATA_DOUBLE)rd->collected_value
-                            / (NETDATA_DOUBLE)st->collected_total;
+                            / (NETDATA_DOUBLE)collected_total;
 
                 rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
                             " * " COLLECTED_NUMBER_FORMAT
@@ -1615,7 +1633,7 @@ after_first_database_work:
                           , rrddim_name(rd)
                           , rd->calculated_value
                           , rd->collected_value
-                          , st->collected_total
+                          , collected_total
                 );
                 break;
 
@@ -1694,31 +1712,15 @@ after_first_database_work:
                     continue;
                 }
 
-                // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
-                // to reset the calculation (it will give zero as the calculation for this second)
-                if(unlikely(rd->last_collected_value > rd->collected_value)) {
-                    debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
-                          , rrdset_id(st)
-                          , rrddim_name(rd)
-                          , rd->last_collected_value
-                          , rd->collected_value
-                    );
-
-                    if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
-                        has_reset_value = 1;
-
-                    rd->last_collected_value = rd->collected_value;
-                }
-
                 // the percentage of the current increment
                 // over the increment of all dimensions together
-                if(unlikely(st->collected_total == st->last_collected_total))
+                if(unlikely(collected_total == last_collected_total))
                     rd->calculated_value = 0;
                 else
                     rd->calculated_value =
                             (NETDATA_DOUBLE)100
                             * (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value)
-                            / (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total);
+                            / (NETDATA_DOUBLE)(collected_total - last_collected_total);
 
                 rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
                             " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
@@ -1726,7 +1728,7 @@ after_first_database_work:
                           , rrddim_name(rd)
                           , rd->calculated_value
                           , rd->collected_value, rd->last_collected_value
-                          , st->collected_total, st->last_collected_total
+                          , collected_total, last_collected_total
                 );
                 break;
 
@@ -1780,8 +1782,6 @@ after_first_database_work:
     );
 
 after_second_database_work:
-    st->last_collected_total  = st->collected_total;
-
     for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
         rd = rda->rd;
         if(unlikely(!rd)) continue;
@@ -1953,8 +1953,8 @@ struct rrdset_map_save_v019 {
     usec_t usec_since_last_update;                  // NEEDS TO BE UPDATED - maintained on load
     struct timeval last_updated;                    // NEEDS TO BE UPDATED - check to reset all - fixed on load
     struct timeval last_collected_time;             // ignored
-    long long collected_total;                      // NEEDS TO BE UPDATED - maintained on load
-    long long last_collected_total;                 // NEEDS TO BE UPDATED - maintained on load
+    long long collected_total;                      // ignored
+    long long last_collected_total;                 // ignored
     void *rrdfamily;                                // ignored
     void *rrdhost;                                  // ignored
     void *next;                                     // ignored
@@ -1978,8 +1978,6 @@ void rrdset_memory_file_update(RRDSET *st) {
     st_on_file->usec_since_last_update = st->usec_since_last_update;
     st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
     st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
-    st_on_file->collected_total = st->collected_total;
-    st_on_file->last_collected_total = st->last_collected_total;
 }
 
 const char *rrdset_cache_filename(RRDSET *st) {
@@ -2071,8 +2069,6 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo
     st->usec_since_last_update = st_on_file->usec_since_last_update;
     st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
     st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
-    st->collected_total = st_on_file->collected_total;
-    st->last_collected_total = st_on_file->last_collected_total;
 
     // link it to st
     st->st_on_file = st_on_file;

health/health.c

@@ -809,7 +809,6 @@ static void health_execute_delayed_initializations(RRDHOST *host) {
         rrddimvar_index_init(st);
 
         rrdsetvar_add_and_leave_released(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_FLAG_NONE);
-        rrdsetvar_add_and_leave_released(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_FLAG_NONE);
         rrdsetvar_add_and_leave_released(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_FLAG_NONE);
         rrdsetvar_add_and_leave_released(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_FLAG_NONE);
         rrdsetvar_add_and_leave_released(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_FLAG_NONE);

streaming/replication.c

@@ -143,11 +143,9 @@ static void replicate_chart_collection_state(BUFFER *wb, RRDSET *st) {
     }
     rrddim_foreach_done(rd);
 
-    buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " %llu %llu " TOTAL_NUMBER_FORMAT " " TOTAL_NUMBER_FORMAT "\n",
+    buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " %llu %llu\n",
                    (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec,
-                   (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec,
-                   st->last_collected_total,
-                   st->collected_total
+                   (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec
     );
 }