sqlite_aclk_alert.c 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132
  1. // SPDX-License-Identifier: GPL-3.0-or-later
  2. #include "sqlite_functions.h"
  3. #include "sqlite_aclk_alert.h"
  4. #ifdef ENABLE_ACLK
  5. #include "../../aclk/aclk_alarm_api.h"
  6. #endif
  7. #define SQL_GET_ALERT_REMOVE_TIME "SELECT when_key FROM health_log_%s WHERE alarm_id = %u " \
  8. "AND unique_id > %u AND unique_id < %u " \
  9. "AND new_status = -2;"
  10. time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after_unique_id, char *uuid_str) {
  11. sqlite3_stmt *res = NULL;
  12. time_t when = 0;
  13. char sql[ACLK_SYNC_QUERY_SIZE];
  14. snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_GET_ALERT_REMOVE_TIME, uuid_str, alarm_id, after_unique_id, before_unique_id);
  15. int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
  16. if (rc != SQLITE_OK) {
  17. error_report("Failed to prepare statement when trying to find removed gap.");
  18. return 0;
  19. }
  20. rc = sqlite3_step_monitored(res);
  21. if (likely(rc == SQLITE_ROW)) {
  22. when = (time_t) sqlite3_column_int64(res, 0);
  23. }
  24. rc = sqlite3_finalize(res);
  25. if (unlikely(rc != SQLITE_OK))
  26. error_report("Failed to finalize statement when trying to find removed gap, rc = %d", rc);
  27. return when;
  28. }
  29. #define SQL_UPDATE_FILTERED_ALERT "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u where filtered_alert_unique_id = %u"
  30. void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str) {
  31. char sql[ACLK_SYNC_QUERY_SIZE];
  32. snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, SQL_UPDATE_FILTERED_ALERT, uuid_str, ae->unique_id, unique_id);
  33. sqlite3_exec_monitored(db_meta, sql, 0, 0, NULL);
  34. ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
  35. }
  36. #define SQL_SELECT_ALERT_BY_UNIQUE_ID "SELECT hl.unique_id FROM health_log_%s hl, alert_hash ah WHERE hl.unique_id = %u " \
  37. "AND hl.config_hash_id = ah.hash_id " \
  38. "AND ah.warn IS NULL AND ah.crit IS NULL;"
  39. static inline bool is_event_from_alert_variable_config(uint32_t unique_id, char *uuid_str) {
  40. sqlite3_stmt *res = NULL;
  41. int rc = 0;
  42. bool ret = false;
  43. char sql[ACLK_SYNC_QUERY_SIZE];
  44. snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_ALERT_BY_UNIQUE_ID, uuid_str, unique_id);
  45. rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
  46. if (rc != SQLITE_OK) {
  47. error_report("Failed to prepare statement when trying to check for alert variables.");
  48. return false;
  49. }
  50. rc = sqlite3_step_monitored(res);
  51. if (likely(rc == SQLITE_ROW)) {
  52. ret = true;
  53. }
  54. rc = sqlite3_finalize(res);
  55. if (unlikely(rc != SQLITE_OK))
  56. error_report("Failed to finalize statement when trying to check for alert variables, rc = %d", rc);
  57. return ret;
  58. }
  59. #define MAX_REMOVED_PERIOD 604800 //a week
  60. //decide if some events should be sent or not
  61. #define SQL_SELECT_ALERT_BY_ID "SELECT hl.new_status, hl.config_hash_id, hl.unique_id FROM health_log_%s hl, aclk_alert_%s aa " \
  62. "WHERE hl.unique_id = aa.filtered_alert_unique_id " \
  63. "AND hl.alarm_id = %u " \
  64. "ORDER BY alarm_event_id DESC LIMIT 1;"
  65. int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
  66. {
  67. sqlite3_stmt *res = NULL;
  68. char uuid_str[UUID_STR_LEN];
  69. uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
  70. int send = 1;
  71. if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED) {
  72. return 0;
  73. }
  74. if (unlikely(uuid_is_null(ae->config_hash_id)))
  75. return 0;
  76. char sql[ACLK_SYNC_QUERY_SIZE];
  77. uuid_t config_hash_id;
  78. RRDCALC_STATUS status;
  79. uint32_t unique_id;
  80. //get the previous sent event of this alarm_id
  81. //base the search on the last filtered event
  82. snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_ALERT_BY_ID, uuid_str, uuid_str, ae->alarm_id);
  83. int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
  84. if (rc != SQLITE_OK) {
  85. error_report("Failed to prepare statement when trying to filter alert events.");
  86. send = 1;
  87. return send;
  88. }
  89. rc = sqlite3_step_monitored(res);
  90. if (likely(rc == SQLITE_ROW)) {
  91. status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
  92. if (sqlite3_column_type(res, 1) != SQLITE_NULL)
  93. uuid_copy(config_hash_id, *((uuid_t *) sqlite3_column_blob(res, 1)));
  94. unique_id = (uint32_t) sqlite3_column_int64(res, 2);
  95. } else {
  96. send = 1;
  97. goto done;
  98. }
  99. if (ae->new_status != (RRDCALC_STATUS)status) {
  100. send = 1;
  101. goto done;
  102. }
  103. if (uuid_memcmp(&ae->config_hash_id, &config_hash_id)) {
  104. send = 1;
  105. goto done;
  106. }
  107. //same status, same config
  108. if (ae->new_status == RRDCALC_STATUS_CLEAR || ae->new_status == RRDCALC_STATUS_UNDEFINED) {
  109. send = 0;
  110. update_filtered(ae, unique_id, uuid_str);
  111. goto done;
  112. }
  113. //detect a long off period of the agent, TODO make global
  114. if (ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) {
  115. time_t when = removed_when(ae->alarm_id, ae->unique_id, unique_id, uuid_str);
  116. if (when && (when + (time_t)MAX_REMOVED_PERIOD) < ae->when) {
  117. send = 1;
  118. goto done;
  119. } else {
  120. send = 0;
  121. update_filtered(ae, unique_id, uuid_str);
  122. goto done;
  123. }
  124. }
  125. done:
  126. rc = sqlite3_finalize(res);
  127. if (unlikely(rc != SQLITE_OK))
  128. error_report("Failed to finalize statement when trying to filter alert events, rc = %d", rc);
  129. return send;
  130. }
  131. // will replace call to aclk_update_alarm in health/health_log.c
  132. // and handle both cases
  133. #define SQL_QUEUE_ALERT_TO_CLOUD "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
  134. "VALUES (@alert_unique_id, unixepoch(), @alert_unique_id) ON CONFLICT (alert_unique_id) do nothing;"
  135. int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
  136. {
  137. if(!service_running(SERVICE_ACLK))
  138. return 0;
  139. if (!claimed())
  140. return 0;
  141. if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED) {
  142. return 0;
  143. }
  144. CHECK_SQLITE_CONNECTION(db_meta);
  145. if (!skip_filter) {
  146. if (!should_send_to_cloud(host, ae)) {
  147. return 0;
  148. }
  149. }
  150. char uuid_str[UUID_STR_LEN];
  151. uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
  152. if (is_event_from_alert_variable_config(ae->unique_id, uuid_str))
  153. return 0;
  154. sqlite3_stmt *res_alert = NULL;
  155. char sql[ACLK_SYNC_QUERY_SIZE];
  156. snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_QUEUE_ALERT_TO_CLOUD, uuid_str);
  157. int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res_alert, 0);
  158. if (unlikely(rc != SQLITE_OK)) {
  159. error_report("Failed to prepare statement to store alert event");
  160. return 1;
  161. }
  162. rc = sqlite3_bind_int(res_alert, 1, (int) ae->unique_id);
  163. if (unlikely(rc != SQLITE_OK))
  164. goto bind_fail;
  165. rc = execute_insert(res_alert);
  166. if (unlikely(rc != SQLITE_DONE)) {
  167. error_report("Failed to store alert event %u, rc = %d", ae->unique_id, rc);
  168. goto bind_fail;
  169. }
  170. ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
  171. rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
  172. bind_fail:
  173. if (unlikely(sqlite3_finalize(res_alert) != SQLITE_OK))
  174. error_report("Failed to reset statement in store alert event, rc = %d", rc);
  175. return 0;
  176. }
  177. int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
  178. {
  179. #ifdef ENABLE_ACLK
  180. switch(status) {
  181. case RRDCALC_STATUS_REMOVED:
  182. return ALARM_STATUS_REMOVED;
  183. case RRDCALC_STATUS_UNDEFINED:
  184. return ALARM_STATUS_NOT_A_NUMBER;
  185. case RRDCALC_STATUS_CLEAR:
  186. return ALARM_STATUS_CLEAR;
  187. case RRDCALC_STATUS_WARNING:
  188. return ALARM_STATUS_WARNING;
  189. case RRDCALC_STATUS_CRITICAL:
  190. return ALARM_STATUS_CRITICAL;
  191. default:
  192. return ALARM_STATUS_UNKNOWN;
  193. }
  194. #else
  195. UNUSED(status);
  196. return 1;
  197. #endif
  198. }
  199. void aclk_push_alert_event(struct aclk_sync_host_config *wc)
  200. {
  201. #ifndef ENABLE_ACLK
  202. UNUSED(wc);
  203. #else
  204. int rc;
  205. if (unlikely(!wc->alert_updates)) {
  206. log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
  207. return;
  208. }
  209. char *claim_id = get_agent_claimid();
  210. if (unlikely(!claim_id))
  211. return;
  212. if (unlikely(!wc->host)) {
  213. freez(claim_id);
  214. return;
  215. }
  216. BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
  217. int limit = ACLK_MAX_ALERT_UPDATES;
  218. sqlite3_stmt *res = NULL;
  219. buffer_sprintf(sql, "select aa.sequence_id, hl.unique_id, hl.alarm_id, hl.config_hash_id, hl.updated_by_id, hl.when_key, " \
  220. " hl.duration, hl.non_clear_duration, hl.flags, hl.exec_run_timestamp, hl.delay_up_to_timestamp, hl.name, " \
  221. " hl.chart, hl.family, hl.exec, hl.recipient, hl.source, hl.units, hl.info, hl.exec_code, hl.new_status, " \
  222. " hl.old_status, hl.delay, hl.new_value, hl.old_value, hl.last_repeat, hl.chart_context, hl.transition_id, hl.alarm_event_id " \
  223. " from health_log_%s hl, aclk_alert_%s aa " \
  224. " where hl.unique_id = aa.alert_unique_id and aa.date_submitted is null " \
  225. " order by aa.sequence_id asc limit %d;", wc->uuid_str, wc->uuid_str, limit);
  226. rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
  227. if (rc != SQLITE_OK) {
  228. // Try to create tables
  229. if (wc->host)
  230. sql_create_health_log_table(wc->host);
  231. BUFFER *sql_fix = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
  232. buffer_sprintf(sql_fix, TABLE_ACLK_ALERT, wc->uuid_str);
  233. rc = db_execute(db_meta, buffer_tostring(sql_fix));
  234. if (unlikely(rc))
  235. error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));
  236. else {
  237. buffer_flush(sql_fix);
  238. buffer_sprintf(sql_fix, INDEX_ACLK_ALERT, wc->uuid_str, wc->uuid_str);
  239. if (unlikely(db_execute(db_meta, buffer_tostring(sql_fix))))
  240. error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));
  241. }
  242. buffer_free(sql_fix);
  243. // Try again
  244. rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
  245. if (rc != SQLITE_OK) {
  246. error_report("Failed to prepare statement when trying to send an alert update via ACLK");
  247. buffer_free(sql);
  248. freez(claim_id);
  249. return;
  250. }
  251. }
  252. char uuid_str[UUID_STR_LEN];
  253. uint64_t first_sequence_id = 0;
  254. uint64_t last_sequence_id = 0;
  255. static __thread uint64_t log_first_sequence_id = 0;
  256. static __thread uint64_t log_last_sequence_id = 0;
  257. while (sqlite3_step_monitored(res) == SQLITE_ROW) {
  258. struct alarm_log_entry alarm_log;
  259. char old_value_string[100 + 1];
  260. char new_value_string[100 + 1];
  261. alarm_log.node_id = wc->node_id;
  262. alarm_log.claim_id = claim_id;
  263. alarm_log.chart = strdupz((char *)sqlite3_column_text(res, 12));
  264. alarm_log.name = strdupz((char *)sqlite3_column_text(res, 11));
  265. alarm_log.family = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) : NULL;
  266. //alarm_log.batch_id = wc->alerts_batch_id;
  267. //alarm_log.sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
  268. alarm_log.when = (time_t) sqlite3_column_int64(res, 5);
  269. uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 3)), uuid_str);
  270. alarm_log.config_hash = strdupz((char *)uuid_str);
  271. alarm_log.utc_offset = wc->host->utc_offset;
  272. alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host));
  273. alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
  274. strdupz((char *)string2str(wc->host->health.health_default_exec));
  275. alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16));
  276. char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
  277. health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) :
  278. strdupz("UNKNOWN=0=UNKNOWN");
  279. alarm_log.command = strdupz(edit_command);
  280. alarm_log.duration = (time_t) sqlite3_column_int64(res, 6);
  281. alarm_log.non_clear_duration = (time_t) sqlite3_column_int64(res, 7);
  282. alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20));
  283. alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 21));
  284. alarm_log.delay = (int) sqlite3_column_int(res, 22);
  285. alarm_log.delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 10);
  286. alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 25);
  287. alarm_log.silenced = ((sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) ||
  288. (sqlite3_column_type(res, 15) != SQLITE_NULL &&
  289. !strncmp((char *)sqlite3_column_text(res, 15), "silent", 6))) ?
  290. 1 :
  291. 0;
  292. alarm_log.value_string =
  293. sqlite3_column_type(res, 23) == SQLITE_NULL ?
  294. strdupz((char *)"-") :
  295. strdupz((char *)format_value_and_unit(
  296. new_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 17), -1));
  297. alarm_log.old_value_string =
  298. sqlite3_column_type(res, 24) == SQLITE_NULL ?
  299. strdupz((char *)"-") :
  300. strdupz((char *)format_value_and_unit(
  301. old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
  302. alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
  303. alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
  304. alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
  305. alarm_log.rendered_info = sqlite3_column_type(res, 18) == SQLITE_NULL ?
  306. strdupz((char *)"") :
  307. strdupz((char *)sqlite3_column_text(res, 18));
  308. alarm_log.chart_context = sqlite3_column_type(res, 26) == SQLITE_NULL ?
  309. strdupz((char *)"") :
  310. strdupz((char *)sqlite3_column_text(res, 26));
  311. uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 27)), uuid_str);
  312. alarm_log.transition_id = sqlite3_column_type(res, 27) == SQLITE_NULL ?
  313. strdupz((char *)"") :
  314. strdupz((char *)uuid_str);
  315. alarm_log.event_id = (time_t) sqlite3_column_int64(res, 28);
  316. aclk_send_alarm_log_entry(&alarm_log);
  317. if (first_sequence_id == 0)
  318. first_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
  319. if (log_first_sequence_id == 0)
  320. log_first_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
  321. last_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
  322. log_last_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
  323. destroy_alarm_log_entry(&alarm_log);
  324. freez(edit_command);
  325. }
  326. if (first_sequence_id) {
  327. buffer_flush(sql);
  328. buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=unixepoch() "
  329. "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
  330. wc->uuid_str, first_sequence_id, last_sequence_id);
  331. if (unlikely(db_execute(db_meta, buffer_tostring(sql))))
  332. error_report("Failed to mark ACLK alert entries as submitted for host %s", rrdhost_hostname(wc->host));
  333. // Mark to do one more check
  334. rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
  335. } else {
  336. if (log_first_sequence_id)
  337. log_access(
  338. "ACLK RES [%s (%s)]: ALERTS SENT from %" PRIu64 " to %" PRIu64 "",
  339. wc->node_id,
  340. wc->host ? rrdhost_hostname(wc->host) : "N/A",
  341. log_first_sequence_id,
  342. log_last_sequence_id);
  343. log_first_sequence_id = 0;
  344. log_last_sequence_id = 0;
  345. }
  346. rc = sqlite3_finalize(res);
  347. if (unlikely(rc != SQLITE_OK))
  348. error_report("Failed to finalize statement to send alert entries from the database, rc = %d", rc);
  349. freez(claim_id);
  350. buffer_free(sql);
  351. #endif
  352. }
  353. void aclk_push_alert_events_for_all_hosts(void)
  354. {
  355. RRDHOST *host;
  356. dfe_start_reentrant(rrdhost_root_index, host) {
  357. if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
  358. continue;
  359. internal_error(true, "ACLK SYNC: Scanning host %s", rrdhost_hostname(host));
  360. rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
  361. struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
  362. if (likely(wc))
  363. aclk_push_alert_event(wc);
  364. }
  365. dfe_done(host);
  366. }
  367. void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
  368. {
  369. char uuid_str[UUID_STR_LEN];
  370. uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
  371. BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
  372. buffer_sprintf(sql,"delete from aclk_alert_%s; " \
  373. "insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
  374. "select unique_id alert_unique_id, unixepoch(), unique_id alert_unique_id from health_log_%s " \
  375. "where new_status <> 0 and new_status <> -2 and config_hash_id is not null and updated_by_id = 0 " \
  376. "order by unique_id asc on conflict (alert_unique_id) do nothing;", uuid_str, uuid_str, uuid_str);
  377. netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
  378. if (unlikely(db_execute(db_meta, buffer_tostring(sql))))
  379. error_report("Failed to queue existing ACLK alert events for host %s", rrdhost_hostname(host));
  380. netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
  381. buffer_free(sql);
  382. rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
  383. }
  384. void aclk_send_alarm_configuration(char *config_hash)
  385. {
  386. if (unlikely(!config_hash))
  387. return;
  388. struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) localhost->aclk_sync_host_config;
  389. if (unlikely(!wc))
  390. return;
  391. log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
  392. aclk_push_alert_config(wc->node_id, config_hash);
  393. }
  394. #define SQL_SELECT_ALERT_CONFIG "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin," \
  395. "module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info," \
  396. "options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after," \
  397. "p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;"
  398. int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash __maybe_unused)
  399. {
  400. int rc = 0;
  401. #ifdef ENABLE_ACLK
  402. CHECK_SQLITE_CONNECTION(db_meta);
  403. sqlite3_stmt *res = NULL;
  404. struct aclk_sync_host_config *wc = NULL;
  405. RRDHOST *host = find_host_by_node_id(node_id);
  406. if (unlikely(!host)) {
  407. freez(config_hash);
  408. freez(node_id);
  409. return 1;
  410. }
  411. wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
  412. if (unlikely(!wc)) {
  413. freez(config_hash);
  414. freez(node_id);
  415. return 1;
  416. }
  417. rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_ALERT_CONFIG, -1, &res, 0);
  418. if (rc != SQLITE_OK) {
  419. error_report("Failed to prepare statement when trying to fetch an alarm hash configuration");
  420. return 1;
  421. }
  422. uuid_t hash_uuid;
  423. if (uuid_parse(config_hash, hash_uuid))
  424. return 1;
  425. rc = sqlite3_bind_blob(res, 1, &hash_uuid , sizeof(hash_uuid), SQLITE_STATIC);
  426. if (unlikely(rc != SQLITE_OK))
  427. goto bind_fail;
  428. struct aclk_alarm_configuration alarm_config;
  429. struct provide_alarm_configuration p_alarm_config;
  430. p_alarm_config.cfg_hash = NULL;
  431. if (sqlite3_step_monitored(res) == SQLITE_ROW) {
  432. alarm_config.alarm = sqlite3_column_bytes(res, 0) > 0 ? strdupz((char *)sqlite3_column_text(res, 0)) : NULL;
  433. alarm_config.tmpl = sqlite3_column_bytes(res, 1) > 0 ? strdupz((char *)sqlite3_column_text(res, 1)) : NULL;
  434. alarm_config.on_chart = sqlite3_column_bytes(res, 2) > 0 ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;
  435. alarm_config.classification = sqlite3_column_bytes(res, 3) > 0 ? strdupz((char *)sqlite3_column_text(res, 3)) : NULL;
  436. alarm_config.type = sqlite3_column_bytes(res, 4) > 0 ? strdupz((char *)sqlite3_column_text(res, 4)) : NULL;
  437. alarm_config.component = sqlite3_column_bytes(res, 5) > 0 ? strdupz((char *)sqlite3_column_text(res, 5)) : NULL;
  438. alarm_config.os = sqlite3_column_bytes(res, 6) > 0 ? strdupz((char *)sqlite3_column_text(res, 6)) : NULL;
  439. alarm_config.hosts = sqlite3_column_bytes(res, 7) > 0 ? strdupz((char *)sqlite3_column_text(res, 7)) : NULL;
  440. alarm_config.plugin = sqlite3_column_bytes(res, 8) > 0 ? strdupz((char *)sqlite3_column_text(res, 8)) : NULL;
  441. alarm_config.module = sqlite3_column_bytes(res, 9) > 0 ? strdupz((char *)sqlite3_column_text(res, 9)) : NULL;
  442. alarm_config.charts = sqlite3_column_bytes(res, 10) > 0 ? strdupz((char *)sqlite3_column_text(res, 10)) : NULL;
  443. alarm_config.families = sqlite3_column_bytes(res, 11) > 0 ? strdupz((char *)sqlite3_column_text(res, 11)) : NULL;
  444. alarm_config.lookup = sqlite3_column_bytes(res, 12) > 0 ? strdupz((char *)sqlite3_column_text(res, 12)) : NULL;
  445. alarm_config.every = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) : NULL;
  446. alarm_config.units = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) : NULL;
  447. alarm_config.green = sqlite3_column_bytes(res, 15) > 0 ? strdupz((char *)sqlite3_column_text(res, 15)) : NULL;
  448. alarm_config.red = sqlite3_column_bytes(res, 16) > 0 ? strdupz((char *)sqlite3_column_text(res, 16)) : NULL;
  449. alarm_config.calculation_expr = sqlite3_column_bytes(res, 17) > 0 ? strdupz((char *)sqlite3_column_text(res, 17)) : NULL;
  450. alarm_config.warning_expr = sqlite3_column_bytes(res, 18) > 0 ? strdupz((char *)sqlite3_column_text(res, 18)) : NULL;
  451. alarm_config.critical_expr = sqlite3_column_bytes(res, 19) > 0 ? strdupz((char *)sqlite3_column_text(res, 19)) : NULL;
  452. alarm_config.recipient = sqlite3_column_bytes(res, 20) > 0 ? strdupz((char *)sqlite3_column_text(res, 20)) : NULL;
  453. alarm_config.exec = sqlite3_column_bytes(res, 21) > 0 ? strdupz((char *)sqlite3_column_text(res, 21)) : NULL;
  454. alarm_config.delay = sqlite3_column_bytes(res, 22) > 0 ? strdupz((char *)sqlite3_column_text(res, 22)) : NULL;
  455. alarm_config.repeat = sqlite3_column_bytes(res, 23) > 0 ? strdupz((char *)sqlite3_column_text(res, 23)) : NULL;
  456. alarm_config.info = sqlite3_column_bytes(res, 24) > 0 ? strdupz((char *)sqlite3_column_text(res, 24)) : NULL;
  457. alarm_config.options = sqlite3_column_bytes(res, 25) > 0 ? strdupz((char *)sqlite3_column_text(res, 25)) : NULL;
  458. alarm_config.host_labels = sqlite3_column_bytes(res, 26) > 0 ? strdupz((char *)sqlite3_column_text(res, 26)) : NULL;
  459. alarm_config.p_db_lookup_dimensions = NULL;
  460. alarm_config.p_db_lookup_method = NULL;
  461. alarm_config.p_db_lookup_options = NULL;
  462. alarm_config.p_db_lookup_after = 0;
  463. alarm_config.p_db_lookup_before = 0;
  464. if (sqlite3_column_bytes(res, 30) > 0) {
  465. alarm_config.p_db_lookup_dimensions = sqlite3_column_bytes(res, 27) > 0 ? strdupz((char *)sqlite3_column_text(res, 27)) : NULL;
  466. alarm_config.p_db_lookup_method = sqlite3_column_bytes(res, 28) > 0 ? strdupz((char *)sqlite3_column_text(res, 28)) : NULL;
  467. BUFFER *tmp_buf = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
  468. buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 29));
  469. alarm_config.p_db_lookup_options = strdupz((char *)buffer_tostring(tmp_buf));
  470. buffer_free(tmp_buf);
  471. alarm_config.p_db_lookup_after = sqlite3_column_int(res, 30);
  472. alarm_config.p_db_lookup_before = sqlite3_column_int(res, 31);
  473. }
  474. alarm_config.p_update_every = sqlite3_column_int(res, 32);
  475. p_alarm_config.cfg_hash = strdupz((char *) config_hash);
  476. p_alarm_config.cfg = alarm_config;
  477. }
  478. if (likely(p_alarm_config.cfg_hash)) {
  479. log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
  480. aclk_send_provide_alarm_cfg(&p_alarm_config);
  481. freez(p_alarm_config.cfg_hash);
  482. destroy_aclk_alarm_configuration(&alarm_config);
  483. }
  484. else
  485. log_access("ACLK STA [%s (%s)]: Alert config for %s not found.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
  486. bind_fail:
  487. rc = sqlite3_finalize(res);
  488. if (unlikely(rc != SQLITE_OK))
  489. error_report("Failed to reset statement when pushing alarm config hash, rc = %d", rc);
  490. freez(config_hash);
  491. freez(node_id);
  492. #endif
  493. return rc;
  494. }
  495. // Start streaming alerts
  496. void aclk_start_alert_streaming(char *node_id, bool resets)
  497. {
  498. if (unlikely(!node_id))
  499. return;
  500. uuid_t node_uuid;
  501. if (uuid_parse(node_id, node_uuid))
  502. return;
  503. RRDHOST *host = find_host_by_node_id(node_id);
  504. if (unlikely(!host))
  505. return;
  506. struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
  507. if (unlikely(!wc))
  508. return;
  509. if (unlikely(!host->health.health_enabled)) {
  510. log_access("ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);
  511. return;
  512. }
  513. if (resets) {
  514. log_access("ACLK REQ [%s (%s)]: STREAM ALERTS ENABLED (RESET REQUESTED)", node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
  515. sql_queue_existing_alerts_to_aclk(host);
  516. } else
  517. log_access("ACLK REQ [%s (%s)]: STREAM ALERTS ENABLED", node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
  518. wc->alert_updates = 1;
  519. wc->alert_queue_removed = SEND_REMOVED_AFTER_HEALTH_LOOPS;
  520. }
  521. #define SQL_QUEUE_REMOVE_ALERTS "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
  522. "SELECT unique_id alert_unique_id, UNIXEPOCH(), unique_id alert_unique_id FROM health_log_%s " \
  523. "WHERE new_status = -2 AND updated_by_id = 0 AND unique_id NOT IN " \
  524. "(SELECT alert_unique_id FROM aclk_alert_%s) " \
  525. "AND config_hash_id NOT IN (select hash_id from alert_hash where warn is null and crit is null) " \
  526. "ORDER BY unique_id ASC " \
  527. "ON CONFLICT (alert_unique_id) DO NOTHING;"
  528. void sql_process_queue_removed_alerts_to_aclk(char *node_id)
  529. {
  530. struct aclk_sync_host_config *wc;
  531. RRDHOST *host = find_host_by_node_id(node_id);
  532. freez(node_id);
  533. if (unlikely(!host || !(wc = host->aclk_sync_host_config)))
  534. return;
  535. char sql[ACLK_SYNC_QUERY_SIZE * 2];
  536. snprintfz(sql,ACLK_SYNC_QUERY_SIZE * 2 - 1, SQL_QUEUE_REMOVE_ALERTS, wc->uuid_str, wc->uuid_str, wc->uuid_str);
  537. if (unlikely(db_execute(db_meta, sql))) {
  538. log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS FAILED", wc->node_id, rrdhost_hostname(wc->host));
  539. error_report("Failed to queue ACLK alert removed entries for host %s", rrdhost_hostname(wc->host));
  540. }
  541. else
  542. log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, rrdhost_hostname(wc->host));
  543. rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
  544. wc->alert_queue_removed = 0;
  545. }
  546. void sql_queue_removed_alerts_to_aclk(RRDHOST *host)
  547. {
  548. if (unlikely(!host->aclk_sync_host_config))
  549. return;
  550. if (!claimed() || !host->node_id)
  551. return;
  552. char node_id[UUID_STR_LEN];
  553. uuid_unparse_lower(*host->node_id, node_id);
  554. aclk_push_node_removed_alerts(node_id);
  555. }
  556. void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unused, char *snapshot_uuid)
  557. {
  558. uuid_t node_uuid;
  559. if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
  560. return;
  561. RRDHOST *host = find_host_by_node_id(node_id);
  562. if (unlikely(!host)) {
  563. log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
  564. return;
  565. }
  566. struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
  567. if (unlikely(!wc)) {
  568. log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
  569. return;
  570. }
  571. log_access(
  572. "IN [%s (%s)]: Request to send alerts snapshot, snapshot_uuid %s",
  573. node_id,
  574. wc->host ? rrdhost_hostname(wc->host) : "N/A",
  575. snapshot_uuid);
  576. if (wc->alerts_snapshot_uuid && !strcmp(wc->alerts_snapshot_uuid,snapshot_uuid))
  577. return;
  578. __sync_synchronize();
  579. wc->alerts_snapshot_uuid = strdupz(snapshot_uuid);
  580. __sync_synchronize();
  581. aclk_push_node_alert_snapshot(node_id);
  582. }
  583. #ifdef ENABLE_ACLK
  584. void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host)
  585. {
  586. char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
  587. char config_hash_id[UUID_STR_LEN];
  588. uuid_unparse_lower(ae->config_hash_id, config_hash_id);
  589. char transition_id[UUID_STR_LEN];
  590. uuid_unparse_lower(ae->transition_id, transition_id);
  591. alarm_log->chart = strdupz(ae_chart_name(ae));
  592. alarm_log->name = strdupz(ae_name(ae));
  593. alarm_log->family = strdupz(ae_family(ae));
  594. alarm_log->batch_id = 0;
  595. alarm_log->sequence_id = 0;
  596. alarm_log->when = (time_t)ae->when;
  597. alarm_log->config_hash = strdupz((char *)config_hash_id);
  598. alarm_log->utc_offset = host->utc_offset;
  599. alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host));
  600. alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health.health_default_exec));
  601. alarm_log->conf_source = ae->source ? strdupz(ae_source(ae)) : strdupz((char *)"");
  602. alarm_log->command = strdupz((char *)edit_command);
  603. alarm_log->duration = (time_t)ae->duration;
  604. alarm_log->non_clear_duration = (time_t)ae->non_clear_duration;
  605. alarm_log->status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->new_status);
  606. alarm_log->old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->old_status);
  607. alarm_log->delay = (int)ae->delay;
  608. alarm_log->delay_up_to_timestamp = (time_t)ae->delay_up_to_timestamp;
  609. alarm_log->last_repeat = (time_t)ae->last_repeat;
  610. alarm_log->silenced =
  611. ((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp(ae_recipient(ae), "silent", 6))) ?
  612. 1 :
  613. 0;
  614. alarm_log->value_string = strdupz(ae_new_value_string(ae));
  615. alarm_log->old_value_string = strdupz(ae_old_value_string(ae));
  616. alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0;
  617. alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0;
  618. alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
  619. alarm_log->rendered_info = strdupz(ae_info(ae));
  620. alarm_log->chart_context = strdupz(ae_chart_context(ae));
  621. alarm_log->transition_id = strdupz((char *)transition_id);
  622. alarm_log->event_id = (uint64_t) ae->alarm_event_id;
  623. freez(edit_command);
  624. }
  625. #endif
  626. #ifdef ENABLE_ACLK
  627. static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, uint32_t mark)
  628. {
  629. ALARM_ENTRY *ae = host->health_log.alarms;
  630. while (ae) {
  631. if (ae->alarm_id == alarm_id && ae->unique_id >mark &&
  632. (ae->new_status != RRDCALC_STATUS_WARNING && ae->new_status != RRDCALC_STATUS_CRITICAL))
  633. return 1;
  634. ae = ae->next;
  635. }
  636. return 0;
  637. }
  638. #endif
  639. #define ALARM_EVENTS_PER_CHUNK 10
  640. void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
  641. {
  642. #ifdef ENABLE_ACLK
  643. RRDHOST *host = find_host_by_node_id(node_id);
  644. if (unlikely(!host)) {
  645. log_access("AC [%s (N/A)]: Node id not found", node_id);
  646. freez(node_id);
  647. return;
  648. }
  649. freez(node_id);
  650. struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
  651. // we perhaps we don't need this for snapshots
  652. if (unlikely(!wc->alert_updates)) {
  653. log_access(
  654. "ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.",
  655. wc->node_id,
  656. wc->host ? rrdhost_hostname(wc->host) : "N/A");
  657. return;
  658. }
  659. if (unlikely(!wc->alerts_snapshot_uuid))
  660. return;
  661. char *claim_id = get_agent_claimid();
  662. if (unlikely(!claim_id))
  663. return;
  664. log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_uuid %s", wc->node_id, rrdhost_hostname(wc->host), wc->alerts_snapshot_uuid);
  665. uint32_t cnt = 0;
  666. char uuid_str[UUID_STR_LEN];
  667. uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
  668. netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
  669. ALARM_ENTRY *ae = host->health_log.alarms;
  670. for (; ae; ae = ae->next) {
  671. if (likely(ae->updated_by_id))
  672. continue;
  673. if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))
  674. continue;
  675. if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))
  676. continue;
  677. if (is_event_from_alert_variable_config(ae->unique_id, uuid_str))
  678. continue;
  679. cnt++;
  680. }
  681. if (cnt) {
  682. uint32_t chunk = 1, chunks = 0;
  683. chunks = (cnt / ALARM_EVENTS_PER_CHUNK) + (cnt % ALARM_EVENTS_PER_CHUNK != 0);
  684. ae = host->health_log.alarms;
  685. cnt = 0;
  686. struct alarm_snapshot alarm_snap;
  687. alarm_snap.node_id = wc->node_id;
  688. alarm_snap.claim_id = claim_id;
  689. alarm_snap.snapshot_uuid = wc->alerts_snapshot_uuid;
  690. alarm_snap.chunks = chunks;
  691. alarm_snap.chunk = chunk;
  692. alarm_snapshot_proto_ptr_t snapshot_proto = NULL;
  693. for (; ae; ae = ae->next) {
  694. if (likely(ae->updated_by_id))
  695. continue;
  696. if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))
  697. continue;
  698. if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))
  699. continue;
  700. if (is_event_from_alert_variable_config(ae->unique_id, uuid_str))
  701. continue;
  702. cnt++;
  703. struct alarm_log_entry alarm_log;
  704. alarm_log.node_id = wc->node_id;
  705. alarm_log.claim_id = claim_id;
  706. if (!snapshot_proto)
  707. snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);
  708. health_alarm_entry2proto_nolock(&alarm_log, ae, host);
  709. add_alarm_log_entry2snapshot(snapshot_proto, &alarm_log);
  710. if (cnt == ALARM_EVENTS_PER_CHUNK) {
  711. aclk_send_alarm_snapshot(snapshot_proto);
  712. cnt = 0;
  713. if (chunk < chunks) {
  714. chunk++;
  715. struct alarm_snapshot alarm_snap;
  716. alarm_snap.node_id = wc->node_id;
  717. alarm_snap.claim_id = claim_id;
  718. alarm_snap.snapshot_uuid = wc->alerts_snapshot_uuid;
  719. alarm_snap.chunks = chunks;
  720. alarm_snap.chunk = chunk;
  721. snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);
  722. }
  723. }
  724. destroy_alarm_log_entry(&alarm_log);
  725. }
  726. if (cnt)
  727. aclk_send_alarm_snapshot(snapshot_proto);
  728. }
  729. netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
  730. wc->alerts_snapshot_uuid = NULL;
  731. freez(claim_id);
  732. #endif
  733. }
  734. #define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE filtered_alert_unique_id + %d < UNIXEPOCH();"
  735. void sql_aclk_alert_clean_dead_entries(RRDHOST *host)
  736. {
  737. char uuid_str[UUID_STR_LEN];
  738. uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
  739. char sql[ACLK_SYNC_QUERY_SIZE];
  740. snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_DELETE_ALERT_ENTRIES, uuid_str, MAX_REMOVED_PERIOD);
  741. char *err_msg = NULL;
  742. int rc = sqlite3_exec_monitored(db_meta, sql, NULL, NULL, &err_msg);
  743. if (rc != SQLITE_OK) {
  744. error_report("Failed when trying to clean stale ACLK alert entries from aclk_alert_%s, error message \"%s\"", uuid_str, err_msg);
  745. sqlite3_free(err_msg);
  746. }
  747. }
  748. #define SQL_GET_MIN_MAX_ALERT_SEQ "SELECT MIN(sequence_id), MAX(sequence_id), " \
  749. "(SELECT MAX(sequence_id) FROM aclk_alert_%s WHERE date_submitted IS NOT NULL) " \
  750. "FROM aclk_alert_%s WHERE date_submitted IS NULL;"
  751. int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status)
  752. {
  753. int rc;
  754. struct aclk_sync_host_config *wc = NULL;
  755. wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
  756. if (!wc)
  757. return 1;
  758. proto_alert_status->alert_updates = wc->alert_updates;
  759. char sql[ACLK_SYNC_QUERY_SIZE];
  760. sqlite3_stmt *res = NULL;
  761. snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_GET_MIN_MAX_ALERT_SEQ, wc->uuid_str, wc->uuid_str);
  762. rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
  763. if (rc != SQLITE_OK) {
  764. error_report("Failed to prepare statement to get alert log status from the database.");
  765. return 1;
  766. }
  767. while (sqlite3_step_monitored(res) == SQLITE_ROW) {
  768. proto_alert_status->pending_min_sequence_id = sqlite3_column_bytes(res, 0) > 0 ? (uint64_t) sqlite3_column_int64(res, 0) : 0;
  769. proto_alert_status->pending_max_sequence_id = sqlite3_column_bytes(res, 1) > 0 ? (uint64_t) sqlite3_column_int64(res, 1) : 0;
  770. proto_alert_status->last_submitted_sequence_id = sqlite3_column_bytes(res, 2) > 0 ? (uint64_t) sqlite3_column_int64(res, 2) : 0;
  771. }
  772. rc = sqlite3_finalize(res);
  773. if (unlikely(rc != SQLITE_OK))
  774. error_report("Failed to finalize statement to get alert log status from the database, rc = %d", rc);
  775. return 0;
  776. }
  777. void aclk_send_alarm_checkpoint(char *node_id, char *claim_id __maybe_unused)
  778. {
  779. if (unlikely(!node_id))
  780. return;
  781. struct aclk_sync_host_config *wc = NULL;
  782. RRDHOST *host = find_host_by_node_id(node_id);
  783. if (unlikely(!host))
  784. return;
  785. wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
  786. if (unlikely(!wc)) {
  787. log_access("ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT REQUEST RECEIVED FOR INVALID NODE", node_id);
  788. return;
  789. }
  790. log_access("ACLK REQ [%s (%s)]: ALERTS CHECKPOINT REQUEST RECEIVED", node_id, rrdhost_hostname(host));
  791. wc->alert_checkpoint_req = SEND_CHECKPOINT_AFTER_HEALTH_LOOPS;
  792. }
  793. typedef struct active_alerts {
  794. char *name;
  795. char *chart;
  796. RRDCALC_STATUS status;
  797. } active_alerts_t;
  798. static inline int compare_active_alerts(const void * a, const void * b) {
  799. active_alerts_t *active_alerts_a = (active_alerts_t *)a;
  800. active_alerts_t *active_alerts_b = (active_alerts_t *)b;
  801. if( !(strcmp(active_alerts_a->name, active_alerts_b->name)) )
  802. {
  803. return strcmp(active_alerts_a->chart, active_alerts_b->chart);
  804. }
  805. else
  806. return strcmp(active_alerts_a->name, active_alerts_b->name);
  807. }
  808. #define BATCH_ALLOCATED 10
  809. void aclk_push_alarm_checkpoint(RRDHOST *host __maybe_unused)
  810. {
  811. #ifdef ENABLE_ACLK
  812. struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
  813. if (unlikely(!wc)) {
  814. log_access("ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT REQUEST RECEIVED FOR INVALID NODE", rrdhost_hostname(host));
  815. return;
  816. }
  817. if (rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS)) {
  818. //postpone checkpoint send
  819. wc->alert_checkpoint_req+=3;
  820. log_access("ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT POSTPONED", rrdhost_hostname(host));
  821. return;
  822. }
  823. RRDCALC *rc;
  824. uint32_t cnt = 0;
  825. size_t len = 0;
  826. active_alerts_t *active_alerts = callocz(BATCH_ALLOCATED, sizeof(active_alerts_t));
  827. foreach_rrdcalc_in_rrdhost_read(host, rc) {
  828. if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
  829. continue;
  830. if (rc->status == RRDCALC_STATUS_WARNING ||
  831. rc->status == RRDCALC_STATUS_CRITICAL) {
  832. if (cnt && !(cnt % BATCH_ALLOCATED)) {
  833. active_alerts = reallocz(active_alerts, (BATCH_ALLOCATED * ((cnt / BATCH_ALLOCATED) + 1)) * sizeof(active_alerts_t));
  834. }
  835. active_alerts[cnt].name = (char *)rrdcalc_name(rc);
  836. len += string_strlen(rc->name);
  837. active_alerts[cnt].chart = (char *)rrdcalc_chart_name(rc);
  838. len += string_strlen(rc->chart);
  839. active_alerts[cnt].status = rc->status;
  840. len++;
  841. cnt++;
  842. }
  843. }
  844. foreach_rrdcalc_in_rrdhost_done(rc);
  845. BUFFER *alarms_to_hash;
  846. if (cnt) {
  847. qsort (active_alerts, cnt, sizeof(active_alerts_t), compare_active_alerts);
  848. alarms_to_hash = buffer_create(len, NULL);
  849. for (uint32_t i=0;i<cnt;i++) {
  850. buffer_strcat(alarms_to_hash, active_alerts[i].name);
  851. buffer_strcat(alarms_to_hash, active_alerts[i].chart);
  852. if (active_alerts[i].status == RRDCALC_STATUS_WARNING)
  853. buffer_strcat(alarms_to_hash, "W");
  854. else if (active_alerts[i].status == RRDCALC_STATUS_CRITICAL)
  855. buffer_strcat(alarms_to_hash, "C");
  856. }
  857. } else {
  858. alarms_to_hash = buffer_create(1, NULL);
  859. buffer_strcat(alarms_to_hash, "");
  860. len = 0;
  861. }
  862. freez(active_alerts);
  863. char hash[SHA256_DIGEST_LENGTH + 1];
  864. if (hash256_string((const unsigned char *)buffer_tostring(alarms_to_hash), len, hash)) {
  865. hash[SHA256_DIGEST_LENGTH] = 0;
  866. struct alarm_checkpoint alarm_checkpoint;
  867. char *claim_id = get_agent_claimid();
  868. alarm_checkpoint.claim_id = claim_id;
  869. alarm_checkpoint.node_id = wc->node_id;
  870. alarm_checkpoint.checksum = (char *)hash;
  871. aclk_send_provide_alarm_checkpoint(&alarm_checkpoint);
  872. freez(claim_id);
  873. log_access("ACLK RES [%s (%s)]: ALERTS CHECKPOINT SENT", wc->node_id, rrdhost_hostname(host));
  874. } else {
  875. log_access("ACLK RES [%s (%s)]: FAILED TO CREATE ALERTS CHECKPOINT HASH", wc->node_id, rrdhost_hostname(host));
  876. }
  877. wc->alert_checkpoint_req = 0;
  878. buffer_free(alarms_to_hash);
  879. #endif
  880. }