alarm-notify.sh.in 74 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - custom notifications by @ktsaou
  32. # - syslog messages by @Ferroin
  33. # - Microsoft Team notification by @tioumen
  34. # - RocketChat notifications by @Hermsi1337 #3777
  35. # -----------------------------------------------------------------------------
  36. # testing notifications
  37. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  38. if [ "${2}" = "test" ]; then
  39. recipient="${1}"
  40. else
  41. recipient="${2}"
  42. fi
  43. [ -z "${recipient}" ] && recipient="sysadmin"
  44. id=1
  45. last="CLEAR"
  46. test_res=0
  47. for x in "WARNING" "CRITICAL" "CLEAR"; do
  48. echo >&2
  49. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  50. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  51. #shellcheck disable=SC2181
  52. if [ $? -ne 0 ]; then
  53. echo >&2 "# FAILED"
  54. test_res=1
  55. else
  56. echo >&2 "# OK"
  57. fi
  58. last="${x}"
  59. id=$((id + 1))
  60. done
  61. exit $test_res
  62. fi
  63. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  64. export LC_ALL=C
  65. # -----------------------------------------------------------------------------
  66. PROGRAM_NAME="$(basename "${0}")"
  67. logdate() {
  68. date "+%Y-%m-%d %H:%M:%S"
  69. }
  70. log() {
  71. local status="${1}"
  72. shift
  73. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  74. }
  75. warning() {
  76. log WARNING "${@}"
  77. }
  78. error() {
  79. log ERROR "${@}"
  80. }
  81. info() {
  82. log INFO "${@}"
  83. }
  84. fatal() {
  85. log FATAL "${@}"
  86. exit 1
  87. }
  88. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  89. debug() {
  90. [ "${debug}" = "1" ] && log DEBUG "${@}"
  91. }
  92. docurl() {
  93. if [ -z "${curl}" ]; then
  94. error "${curl} is unset."
  95. return 1
  96. fi
  97. if [ "${debug}" = "1" ]; then
  98. echo >&2 "--- BEGIN curl command ---"
  99. printf >&2 "%q " ${curl} "${@}"
  100. echo >&2
  101. echo >&2 "--- END curl command ---"
  102. local out code ret
  103. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  104. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  105. ret=$?
  106. echo >&2 "--- BEGIN received response ---"
  107. cat >&2 "${out}"
  108. echo >&2
  109. echo >&2 "--- END received response ---"
  110. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  111. rm "${out}"
  112. echo "${code}"
  113. return ${ret}
  114. fi
  115. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  116. return $?
  117. }
  118. # -----------------------------------------------------------------------------
  119. # List of all the notification mechanisms we support.
  120. # Used in a couple of places to write more compact code.
  121. method_names="
  122. email
  123. pushover
  124. pushbullet
  125. telegram
  126. slack
  127. alerta
  128. flock
  129. discord
  130. hipchat
  131. twilio
  132. messagebird
  133. pd
  134. fleep
  135. syslog
  136. custom
  137. msteam
  138. kavenegar
  139. prowl
  140. awssns
  141. rocketchat
  142. sms
  143. "
  144. # -----------------------------------------------------------------------------
  145. # this is to be overwritten by the config file
  146. custom_sender() {
  147. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  148. }
  149. # -----------------------------------------------------------------------------
  150. # check for BASH v4+ (required for associative arrays)
  151. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  152. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  153. fi
  154. # -----------------------------------------------------------------------------
  155. # defaults to allow running this script by hand
  156. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  157. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  158. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  159. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  160. # -----------------------------------------------------------------------------
  161. # parse command line parameters
  162. if [ ${1} = "unittest" ]; then
  163. unittest=1 # enable unit testing mode
  164. roles="${2}" # the role that should be used for unit testing
  165. cfgfile="${3}" # the location of the config file to use for unit testing
  166. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  167. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  168. else
  169. roles="${1}" # the roles that should be notified for this event
  170. args_host="${2}" # the host generated this event
  171. unique_id="${3}" # the unique id of this event
  172. alarm_id="${4}" # the unique id of the alarm that generated this event
  173. event_id="${5}" # the incremental id of the event, for this alarm id
  174. when="${6}" # the timestamp this event occurred
  175. name="${7}" # the name of the alarm, as given in netdata health.d entries
  176. chart="${8}" # the name of the chart (type.id)
  177. family="${9}" # the family of the chart
  178. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  179. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  180. value="${12}" # the current value of the alarm
  181. old_value="${13}" # the previous value of the alarm
  182. src="${14}" # the line number and file the alarm has been configured
  183. duration="${15}" # the duration in seconds of the previous alarm state
  184. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  185. units="${17}" # the units of the value
  186. info="${18}" # a short description of the alarm
  187. value_string="${19}" # friendly value (with units)
  188. # shellcheck disable=SC2034
  189. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  190. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  191. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  192. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  193. total_warnings="${23}" # Total number of alarms in WARNING state
  194. total_critical="${24}" # Total number of alarms in CRITICAL state
  195. fi
  196. # -----------------------------------------------------------------------------
  197. # find a suitable hostname to use, if netdata did not supply a hostname
  198. if [ -z ${args_host} ]; then
  199. this_host=$(hostname -s 2>/dev/null)
  200. host="${this_host}"
  201. args_host="${this_host}"
  202. else
  203. host="${args_host}"
  204. fi
  205. # -----------------------------------------------------------------------------
  206. # screen statuses we don't need to send a notification
  207. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  208. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  209. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  210. exit 1
  211. fi
  212. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  213. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  214. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  215. exit 1
  216. fi
  217. # -----------------------------------------------------------------------------
  218. # load configuration
  219. # By default fetch images from the global public registry.
  220. # This is required by default, since all notification methods need to download
  221. # images via the Internet, and private registries might not be reachable.
  222. # This can be overwritten at the configuration file.
  223. images_base_url="https://registry.my-netdata.io"
  224. # curl options to use
  225. curl_options=""
  226. # hostname handling
  227. use_fqdn="NO"
  228. # needed commands
  229. # if empty they will be searched in the system path
  230. curl=
  231. sendmail=
  232. # enable / disable features
  233. for method_name in ${method_names^^}; do
  234. declare SEND_${method_name}="YES"
  235. declare DEFAULT_RECIPIENT_${method_name}
  236. done
  237. for method_name in ${method_names}; do
  238. declare -A role_recipients_${method_name}
  239. done
  240. # slack configs
  241. SLACK_WEBHOOK_URL=
  242. # Microsoft Team configs
  243. MSTEAM_WEBHOOK_URL=
  244. # rocketchat configs
  245. ROCKETCHAT_WEBHOOK_URL=
  246. # alerta configs
  247. ALERTA_WEBHOOK_URL=
  248. ALERTA_API_KEY=
  249. # flock configs
  250. FLOCK_WEBHOOK_URL=
  251. # discord configs
  252. DISCORD_WEBHOOK_URL=
  253. # pushover configs
  254. PUSHOVER_APP_TOKEN=
  255. # pushbullet configs
  256. PUSHBULLET_ACCESS_TOKEN=
  257. PUSHBULLET_SOURCE_DEVICE=
  258. # twilio configs
  259. TWILIO_ACCOUNT_SID=
  260. TWILIO_ACCOUNT_TOKEN=
  261. TWILIO_NUMBER=
  262. # hipchat configs
  263. HIPCHAT_SERVER=
  264. HIPCHAT_AUTH_TOKEN=
  265. # messagebird configs
  266. MESSAGEBIRD_ACCESS_KEY=
  267. MESSAGEBIRD_NUMBER=
  268. # kavenegar configs
  269. KAVENEGAR_API_KEY=
  270. KAVENEGAR_SENDER=
  271. # telegram configs
  272. TELEGRAM_BOT_TOKEN=
  273. # kafka configs
  274. SEND_KAFKA="YES"
  275. KAFKA_URL=
  276. KAFKA_SENDER_IP=
  277. # pagerduty.com configs
  278. PD_SERVICE_KEY=
  279. # fleep.io configs
  280. FLEEP_SENDER="${host}"
  281. # Amazon SNS configs
  282. AWSSNS_MESSAGE_FORMAT=
  283. # syslog configs
  284. SYSLOG_FACILITY=
  285. # email configs
  286. EMAIL_SENDER=
  287. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  288. EMAIL_THREADING=
  289. # irc configs
  290. IRC_NICKNAME=
  291. IRC_REALNAME=
  292. IRC_NETWORK=
  293. # load the stock and user configuration files
  294. # these will overwrite the variables above
  295. if [ ${unittest} ]; then
  296. if source "${cfgfile}"; then
  297. error "Failed to load requested config file."
  298. exit 1
  299. fi
  300. else
  301. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  302. if [ -f "${CONFIG}" ]; then
  303. debug "Loading config file '${CONFIG}'..."
  304. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  305. else
  306. warning "Cannot find file '${CONFIG}'."
  307. fi
  308. done
  309. fi
  310. # If we didn't autodetect the character set for e-mail and it wasn't
  311. # set by the user, we need to set it to a reasonable default. UTF-8
  312. # should be correct for almost all modern UNIX systems.
  313. if [ -z ${EMAIL_CHARSET} ]; then
  314. EMAIL_CHARSET="UTF-8"
  315. fi
  316. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  317. # unless we're sending an alarm for a slave system which we can't get the
  318. # FQDN of easily.
  319. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  320. host="$(hostname -f 2>/dev/null)"
  321. fi
  322. # -----------------------------------------------------------------------------
  323. # filter a recipient based on alarm event severity
  324. filter_recipient_by_criticality() {
  325. local method="${1}" x="${2}" r s
  326. shift
  327. r="${x/|*/}" # the recipient
  328. s="${x/*|/}" # the severity required for notifying this recipient
  329. # no severity filtering for this person
  330. [ "${r}" = "${s}" ] && return 0
  331. # the severity is invalid
  332. s="${s^^}"
  333. if [ "${s}" != "CRITICAL" ]; then
  334. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  335. return 0
  336. fi
  337. # create the status tracking directory for this user
  338. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  339. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  340. case "${status}" in
  341. CRITICAL)
  342. # make sure he will get future notifications for this alarm too
  343. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  344. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  345. return 0
  346. ;;
  347. WARNING)
  348. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  349. # we do not remove the file, so that he will get future notifications of this alarm
  350. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  351. return 0
  352. fi
  353. ;;
  354. *)
  355. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  356. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  357. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  358. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  359. return 0
  360. fi
  361. ;;
  362. esac
  363. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  364. return 1
  365. }
  366. # -----------------------------------------------------------------------------
  367. # verify the delivery methods supported
  368. # check slack
  369. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  370. # check rocketchat
  371. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  372. # check alerta
  373. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  374. # check flock
  375. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  376. # check discord
  377. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  378. # check pushover
  379. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  380. # check pushbullet
  381. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  382. # check twilio
  383. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  384. # check hipchat
  385. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  386. # check messagebird
  387. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  388. # check kavenegar
  389. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  390. # check telegram
  391. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  392. # check kafka
  393. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  394. # check irc
  395. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  396. # check fleep
  397. #shellcheck disable=SC2153
  398. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  399. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  400. [ "${SEND_SLACK}" = "YES" ] ||
  401. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  402. [ "${SEND_ALERTA}" = "YES" ] ||
  403. [ "${SEND_PD}" = "YES" ] ||
  404. [ "${SEND_FLOCK}" = "YES" ] ||
  405. [ "${SEND_DISCORD}" = "YES" ] ||
  406. [ "${SEND_HIPCHAT}" = "YES" ] ||
  407. [ "${SEND_TWILIO}" = "YES" ] ||
  408. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  409. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  410. [ "${SEND_TELEGRAM}" = "YES" ] ||
  411. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  412. [ "${SEND_KAFKA}" = "YES" ] ||
  413. [ "${SEND_FLEEP}" = "YES" ] ||
  414. [ "${SEND_PROWL}" = "YES" ] ||
  415. [ "${SEND_CUSTOM}" = "YES" ] ||
  416. [ "${SEND_MSTEAM}" = "YES" ]; then
  417. # if we need curl, check for the curl command
  418. if [ -z "${curl}" ]; then
  419. curl="$(command -v curl 2>/dev/null)"
  420. fi
  421. if [ -z "${curl}" ]; then
  422. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  423. SEND_PUSHOVER="NO"
  424. SEND_PUSHBULLET="NO"
  425. SEND_TELEGRAM="NO"
  426. SEND_SLACK="NO"
  427. SEND_MSTEAM="NO"
  428. SEND_ROCKETCHAT="NO"
  429. SEND_ALERTA="NO"
  430. SEND_PD="NO"
  431. SEND_FLOCK="NO"
  432. SEND_DISCORD="NO"
  433. SEND_TWILIO="NO"
  434. SEND_HIPCHAT="NO"
  435. SEND_MESSAGEBIRD="NO"
  436. SEND_KAVENEGAR="NO"
  437. SEND_KAFKA="NO"
  438. SEND_FLEEP="NO"
  439. SEND_PROWL="NO"
  440. SEND_CUSTOM="NO"
  441. fi
  442. fi
  443. if [ "${SEND_SMS}" = "YES" ]; then
  444. if [ -z "${sendsms}" ]; then
  445. sendsms="$(command -v sendsms 2>/dev/null)"
  446. fi
  447. if [ -z "${sendsms}" ]; then
  448. SEND_SMS="NO"
  449. fi
  450. fi
  451. # if we need sendmail, check for the sendmail command
  452. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  453. sendmail="$(command -v sendmail 2>/dev/null)"
  454. if [ -z "${sendmail}" ]; then
  455. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  456. SEND_EMAIL="NO"
  457. fi
  458. fi
  459. # if we need logger, check for the logger command
  460. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  461. logger="$(command -v logger 2>/dev/null)"
  462. if [ -z "${logger}" ]; then
  463. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  464. SEND_SYSLOG="NO"
  465. fi
  466. fi
  467. # if we need aws, check for the aws command
  468. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  469. aws="$(command -v aws 2>/dev/null)"
  470. if [ -z "${aws}" ]; then
  471. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  472. SEND_AWSSNS="NO"
  473. fi
  474. fi
  475. # -----------------------------------------------------------------------------
  476. # find the recipients' addresses per method
  477. # netdata may call us with multiple roles, and roles may have multiple but
  478. # overlapping recipients - so, here we find the unique recipients.
  479. for method_name in ${method_names}; do
  480. send_var="SEND_${method_name^^}"
  481. if [ "${!send_var}" = "NO" ]; then
  482. continue
  483. fi
  484. declare -A arr_var=()
  485. for x in ${roles//,/ }; do
  486. # the roles 'silent' and 'disabled' mean:
  487. # don't send a notification for this role
  488. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  489. continue
  490. fi
  491. role_recipients="role_recipients_${method_name}[$x]"
  492. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  493. a="${!role_recipients}"
  494. [ -z "${a}" ] && a="${!default_recipient_var}"
  495. for r in ${a//,/ }; do
  496. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  497. done
  498. done
  499. # build the list of recipients
  500. to_var="to_${method_name}"
  501. declare to_${method_name}="${!arr_var[*]}"
  502. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  503. done
  504. # -----------------------------------------------------------------------------
  505. # handle fixup of the email recipient list.
  506. fix_to_email() {
  507. to_email=
  508. while [ -n "${1}" ]; do
  509. [ -n "${to_email}" ] && to_email="${to_email}, "
  510. to_email="${to_email}${1}"
  511. shift 1
  512. done
  513. }
  514. # ${to_email} without quotes here
  515. fix_to_email ${to_email}
  516. # -----------------------------------------------------------------------------
  517. # handle output if we're running in unit test mode
  518. if [ ${unittest} ]; then
  519. for method_name in ${method_names}; do
  520. to_var="to_${method_name}"
  521. echo "results: ${method_name}: ${!to_var}"
  522. done
  523. exit 0
  524. fi
  525. # -----------------------------------------------------------------------------
  526. # check that we have at least a method enabled
  527. proceed=0
  528. for method in "${SEND_EMAIL}" \
  529. "${SEND_PUSHOVER}" \
  530. "${SEND_TELEGRAM}" \
  531. "${SEND_SLACK}" \
  532. "${SEND_ROCKETCHAT}" \
  533. "${SEND_ALERTA}" \
  534. "${SEND_FLOCK}" \
  535. "${SEND_DISCORD}" \
  536. "${SEND_TWILIO}" \
  537. "${SEND_HIPCHAT}" \
  538. "${SEND_MESSAGEBIRD}" \
  539. "${SEND_KAVENEGAR}" \
  540. "${SEND_PUSHBULLET}" \
  541. "${SEND_KAFKA}" \
  542. "${SEND_PD}" \
  543. "${SEND_FLEEP}" \
  544. "${SEND_PROWL}" \
  545. "${SEND_CUSTOM}" \
  546. "${SEND_IRC}" \
  547. "${SEND_AWSSNS}" \
  548. "${SEND_SYSLOG}" \
  549. "${SEND_SMS}" \
  550. "${SEND_MSTEAM}"; do
  551. if [ "${method}" == "YES" ]; then
  552. proceed=1
  553. break
  554. fi
  555. done
  556. if [ "$proceed" -eq 0 ]; then
  557. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  558. fi
  559. # -----------------------------------------------------------------------------
  560. # get the date the alarm happened
  561. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  562. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  563. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  564. [ -z "${date}" ] && date=$(date 2>/dev/null)
  565. # ----------------------------------------------------------------------------
  566. # prepare some extra headers if we've been asked to thread e-mails
  567. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  568. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\nReferences: <${chart}-${name}@${host}>"
  569. else
  570. email_thread_headers=
  571. fi
  572. # -----------------------------------------------------------------------------
  573. # function to URL encode a string
  574. urlencode() {
  575. local string="${1}" strlen encoded pos c o
  576. strlen=${#string}
  577. for ((pos = 0; pos < strlen; pos++)); do
  578. c=${string:pos:1}
  579. case "${c}" in
  580. [-_.~a-zA-Z0-9])
  581. o="${c}"
  582. ;;
  583. *)
  584. printf -v o '%%%02x' "'${c}"
  585. ;;
  586. esac
  587. encoded+="${o}"
  588. done
  589. REPLY="${encoded}"
  590. echo "${REPLY}"
  591. }
  592. # -----------------------------------------------------------------------------
  593. # function to convert a duration in seconds, to a human readable duration
  594. # using DAYS, MINUTES, SECONDS
  595. duration4human() {
  596. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  597. d=$((s / 86400))
  598. s=$((s - (d * 86400)))
  599. h=$((s / 3600))
  600. s=$((s - (h * 3600)))
  601. m=$((s / 60))
  602. s=$((s - (m * 60)))
  603. if [ ${d} -gt 0 ]; then
  604. [ ${m} -ge 30 ] && h=$((h + 1))
  605. [ ${d} -gt 1 ] && ds="days"
  606. [ ${h} -gt 1 ] && hs="hours"
  607. if [ ${h} -gt 0 ]; then
  608. ret="${d} ${ds} and ${h} ${hs}"
  609. else
  610. ret="${d} ${ds}"
  611. fi
  612. elif [ ${h} -gt 0 ]; then
  613. [ ${s} -ge 30 ] && m=$((m + 1))
  614. [ ${h} -gt 1 ] && hs="hours"
  615. [ ${m} -gt 1 ] && ms="minutes"
  616. if [ ${m} -gt 0 ]; then
  617. ret="${h} ${hs} and ${m} ${ms}"
  618. else
  619. ret="${h} ${hs}"
  620. fi
  621. elif [ ${m} -gt 0 ]; then
  622. [ ${m} -gt 1 ] && ms="minutes"
  623. [ ${s} -gt 1 ] && ss="seconds"
  624. if [ ${s} -gt 0 ]; then
  625. ret="${m} ${ms} and ${s} ${ss}"
  626. else
  627. ret="${m} ${ms}"
  628. fi
  629. else
  630. [ ${s} -gt 1 ] && ss="seconds"
  631. ret="${s} ${ss}"
  632. fi
  633. REPLY="${ret}"
  634. echo "${REPLY}"
  635. }
  636. # -----------------------------------------------------------------------------
  637. # email sender
  638. send_email() {
  639. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  640. if [ "${SEND_EMAIL}" = "YES" ]; then
  641. if [ -n "${EMAIL_SENDER}" ]; then
  642. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  643. # the name includes double quotes
  644. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  645. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  646. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  647. # the name includes single quotes
  648. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  649. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  650. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  651. # the name does not have any quotes
  652. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  653. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  654. fi
  655. fi
  656. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  657. [ -n "${sender_name}" ] && opts+=(-F "${sender_name}")
  658. if [ "${debug}" = "1" ]; then
  659. echo >&2 "--- BEGIN sendmail command ---"
  660. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  661. echo >&2
  662. echo >&2 "--- END sendmail command ---"
  663. fi
  664. "${sendmail}" -t "${opts[@]}"
  665. ret=$?
  666. if [ ${ret} -eq 0 ]; then
  667. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  668. return 0
  669. else
  670. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  671. return 1
  672. fi
  673. fi
  674. return 1
  675. }
  676. # -----------------------------------------------------------------------------
  677. # pushover sender
  678. send_pushover() {
  679. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  680. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  681. # https://pushover.net/api
  682. priority=-2
  683. case "${status}" in
  684. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  685. WARNING) priority=0 ;; # normal priority: respect quiet hours
  686. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  687. *) priority=-2 ;; # lowest priority: no notification at all
  688. esac
  689. for user in ${usertokens}; do
  690. httpcode=$(docurl \
  691. --form-string "token=${apptoken}" \
  692. --form-string "user=${user}" \
  693. --form-string "html=1" \
  694. --form-string "title=${title}" \
  695. --form-string "message=${message}" \
  696. --form-string "timestamp=${when}" \
  697. --form-string "url=${url}" \
  698. --form-string "url_title=Open netdata dashboard to view the alarm" \
  699. --form-string "priority=${priority}" \
  700. https://api.pushover.net/1/messages.json)
  701. if [ "${httpcode}" = "200" ]; then
  702. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  703. sent=$((sent + 1))
  704. else
  705. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  706. fi
  707. done
  708. [ ${sent} -gt 0 ] && return 0
  709. fi
  710. return 1
  711. }
  712. # -----------------------------------------------------------------------------
  713. # pushbullet sender
  714. send_pushbullet() {
  715. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  716. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  717. #https://docs.pushbullet.com/#create-push
  718. for user in ${recipients}; do
  719. httpcode=$(docurl \
  720. --header 'Access-Token: '${userapikey}'' \
  721. --header 'Content-Type: application/json' \
  722. --data-binary @<(
  723. cat <<EOF
  724. {"title": "${title}",
  725. "type": "link",
  726. "email": "${user}",
  727. "body": "$(echo -n ${message})",
  728. "url": "${url}",
  729. "source_device_iden": "${source_device}"}
  730. EOF
  731. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  732. if [ "${httpcode}" = "200" ]; then
  733. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  734. sent=$((sent + 1))
  735. else
  736. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  737. fi
  738. done
  739. [ ${sent} -gt 0 ] && return 0
  740. fi
  741. return 1
  742. }
  743. # -----------------------------------------------------------------------------
  744. # kafka sender
  745. send_kafka() {
  746. local httpcode sent=0
  747. if [ "${SEND_KAFKA}" = "YES" ]; then
  748. httpcode=$(docurl -X POST \
  749. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  750. "${KAFKA_URL}")
  751. if [ "${httpcode}" = "204" ]; then
  752. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  753. sent=$((sent + 1))
  754. else
  755. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  756. fi
  757. [ ${sent} -gt 0 ] && return 0
  758. fi
  759. return 1
  760. }
  761. # -----------------------------------------------------------------------------
  762. # pagerduty.com sender
  763. send_pd() {
  764. local recipients="${1}" sent=0
  765. unset t
  766. case ${status} in
  767. CLEAR) t='resolve' ;;
  768. WARNING) t='trigger' ;;
  769. CRITICAL) t='trigger' ;;
  770. esac
  771. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  772. for PD_SERVICE_KEY in ${recipients}; do
  773. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  774. payload="$(
  775. cat <<EOF
  776. {
  777. "service_key": "${PD_SERVICE_KEY}",
  778. "event_type": "${t}",
  779. "incident_key" : "${alarm_id}",
  780. "description": "${d}",
  781. "details": {
  782. "value_w_units": "${value_string}",
  783. "when": "${when}",
  784. "duration" : "${duration}",
  785. "roles": "${roles}",
  786. "alarm_id" : "${alarm_id}",
  787. "name" : "${name}",
  788. "chart" : "${chart}",
  789. "family" : "${family}",
  790. "status" : "${status}",
  791. "old_status" : "${old_status}",
  792. "value" : "${value}",
  793. "old_value" : "${old_value}",
  794. "src" : "${src}",
  795. "non_clear_duration" : "${non_clear_duration}",
  796. "units" : "${units}",
  797. "info" : "${info}"
  798. }
  799. }
  800. EOF
  801. )"
  802. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  803. if [ "${httpcode}" = "200" ]; then
  804. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  805. sent=$((sent + 1))
  806. else
  807. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  808. fi
  809. done
  810. [ ${sent} -gt 0 ] && return 0
  811. fi
  812. return 1
  813. }
  814. # -----------------------------------------------------------------------------
  815. # twilio sender
  816. send_twilio() {
  817. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  818. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  819. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  820. for user in ${recipients}; do
  821. httpcode=$(docurl -X POST \
  822. --data-urlencode "From=${twilionumber}" \
  823. --data-urlencode "To=${user}" \
  824. --data-urlencode "Body=${title} ${message}" \
  825. -u "${accountsid}:${accounttoken}" \
  826. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  827. if [ "${httpcode}" = "201" ]; then
  828. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  829. sent=$((sent + 1))
  830. else
  831. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  832. fi
  833. done
  834. [ ${sent} -gt 0 ] && return 0
  835. fi
  836. return 1
  837. }
  838. # -----------------------------------------------------------------------------
  839. # hipchat sender
  840. send_hipchat() {
  841. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  842. # remove <small></small> from the message
  843. message="${message//<small>/}"
  844. message="${message//<\/small>/}"
  845. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  846. # Valid values: html, text.
  847. # Defaults to 'html'.
  848. msg_format="html"
  849. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  850. case "${status}" in
  851. WARNING) color="yellow" ;;
  852. CRITICAL) color="red" ;;
  853. CLEAR) color="green" ;;
  854. *) color="gray" ;;
  855. esac
  856. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  857. # Each recipient's notification preferences are taken into account.
  858. # Defaults to false.
  859. notify="true"
  860. for room in ${recipients}; do
  861. httpcode=$(docurl -X POST \
  862. -H "Content-type: application/json" \
  863. -H "Authorization: Bearer ${authtoken}" \
  864. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  865. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  866. if [ "${httpcode}" = "204" ]; then
  867. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  868. sent=$((sent + 1))
  869. else
  870. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  871. fi
  872. done
  873. [ ${sent} -gt 0 ] && return 0
  874. fi
  875. return 1
  876. }
  877. # -----------------------------------------------------------------------------
  878. # messagebird sender
  879. send_messagebird() {
  880. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  881. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  882. #https://developers.messagebird.com/docs/messaging
  883. for user in ${recipients}; do
  884. httpcode=$(docurl -X POST \
  885. --data-urlencode "originator=${messagebirdnumber}" \
  886. --data-urlencode "recipients=${user}" \
  887. --data-urlencode "body=${title} ${message}" \
  888. --data-urlencode "datacoding=auto" \
  889. -H "Authorization: AccessKey ${accesskey}" \
  890. "https://rest.messagebird.com/messages")
  891. if [ "${httpcode}" = "201" ]; then
  892. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  893. sent=$((sent + 1))
  894. else
  895. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  896. fi
  897. done
  898. [ ${sent} -gt 0 ] && return 0
  899. fi
  900. return 1
  901. }
  902. # -----------------------------------------------------------------------------
  903. # kavenegar sender
  904. send_kavenegar() {
  905. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  906. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  907. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  908. for user in ${recipients}; do
  909. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  910. --data-urlencode "sender=${kavenegarsender}" \
  911. --data-urlencode "receptor=${user}" \
  912. --data-urlencode "message=${title} ${message}")
  913. if [ "${httpcode}" = "200" ]; then
  914. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  915. sent=$((sent + 1))
  916. else
  917. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  918. fi
  919. done
  920. [ ${sent} -gt 0 ] && return 0
  921. fi
  922. return 1
  923. }
  924. # -----------------------------------------------------------------------------
  925. # telegram sender
  926. send_telegram() {
  927. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  928. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  929. case "${status}" in
  930. WARNING) emoji="⚠️" ;;
  931. CRITICAL) emoji="🔴" ;;
  932. CLEAR) emoji="✅" ;;
  933. *) emoji="⚪️" ;;
  934. esac
  935. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  936. for chatid in ${chatids}; do
  937. # https://core.telegram.org/bots/api#sendmessage
  938. httpcode=$(docurl ${disableNotification} \
  939. --data-urlencode "parse_mode=HTML" \
  940. --data-urlencode "disable_web_page_preview=true" \
  941. --data-urlencode "text=${emoji} ${message}" \
  942. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  943. if [ "${httpcode}" = "200" ]; then
  944. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  945. sent=$((sent + 1))
  946. elif [ "${httpcode}" = "401" ]; then
  947. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  948. else
  949. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  950. fi
  951. done
  952. [ ${sent} -gt 0 ] && return 0
  953. fi
  954. return 1
  955. }
  956. # -----------------------------------------------------------------------------
  957. # Microsoft Team sender
  958. send_msteam() {
  959. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  960. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  961. case "${status}" in
  962. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}" ;;
  963. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}" ;;
  964. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}" ;;
  965. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}" ;;
  966. esac
  967. for channel in ${channels}; do
  968. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  969. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  970. payload="$(
  971. cat <<EOF
  972. {
  973. "@context": "http://schema.org/extensions",
  974. "@type": "MessageCard",
  975. "themeColor": "${color}",
  976. "title": "$icon Alert ${status} from netdata for ${host}",
  977. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  978. "potentialAction": [
  979. {
  980. "@type": "OpenUri",
  981. "name": "Netdata",
  982. "targets": [
  983. { "os": "default", "uri": "${goto_url}" }
  984. ]
  985. }
  986. ]
  987. }
  988. EOF
  989. )"
  990. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  991. webhook="${webhook//CHANNEL/${channel}}"
  992. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  993. if [ "${httpcode}" = "200" ]; then
  994. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  995. sent=$((sent + 1))
  996. else
  997. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  998. fi
  999. done
  1000. [ ${sent} -gt 0 ] && return 0
  1001. return 1
  1002. }
  1003. # slack sender
  1004. send_slack() {
  1005. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1006. [ "${SEND_SLACK}" != "YES" ] && return 1
  1007. case "${status}" in
  1008. WARNING) color="warning" ;;
  1009. CRITICAL) color="danger" ;;
  1010. CLEAR) color="good" ;;
  1011. *) color="#777777" ;;
  1012. esac
  1013. for channel in ${channels}; do
  1014. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1015. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1016. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1017. if [ "${channel}" = "#" ]; then
  1018. ch=""
  1019. chstr="without specifying a channel"
  1020. else
  1021. ch="\"channel\": \"${channel}\","
  1022. chstr="to '${channel}'"
  1023. fi
  1024. payload="$(
  1025. cat <<EOF
  1026. {
  1027. $ch
  1028. "username": "netdata on ${host}",
  1029. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1030. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1031. "attachments": [
  1032. {
  1033. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1034. "color": "${color}",
  1035. "title": "${alarm}",
  1036. "title_link": "${goto_url}",
  1037. "text": "${info}",
  1038. "fields": [
  1039. {
  1040. "title": "${chart}",
  1041. "short": true
  1042. },
  1043. {
  1044. "title": "${family}",
  1045. "short": true
  1046. }
  1047. ],
  1048. "thumb_url": "${image}",
  1049. "footer": "by ${host}",
  1050. "ts": ${when}
  1051. }
  1052. ]
  1053. }
  1054. EOF
  1055. )"
  1056. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1057. if [ "${httpcode}" = "200" ]; then
  1058. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1059. sent=$((sent + 1))
  1060. else
  1061. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1062. fi
  1063. done
  1064. [ ${sent} -gt 0 ] && return 0
  1065. return 1
  1066. }
  1067. # -----------------------------------------------------------------------------
  1068. # rocketchat sender
  1069. send_rocketchat() {
  1070. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1071. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1072. case "${status}" in
  1073. WARNING) color="warning" ;;
  1074. CRITICAL) color="danger" ;;
  1075. CLEAR) color="good" ;;
  1076. *) color="#777777" ;;
  1077. esac
  1078. for channel in ${channels}; do
  1079. payload="$(
  1080. cat <<EOF
  1081. {
  1082. "channel": "#${channel}",
  1083. "alias": "netdata on ${host}",
  1084. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1085. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1086. "attachments": [
  1087. {
  1088. "color": "${color}",
  1089. "title": "${alarm}",
  1090. "title_link": "${goto_url}",
  1091. "text": "${info}",
  1092. "fields": [
  1093. {
  1094. "title": "${chart}",
  1095. "short": true,
  1096. "value": "chart"
  1097. },
  1098. {
  1099. "title": "${family}",
  1100. "short": true,
  1101. "value": "family"
  1102. }
  1103. ],
  1104. "thumb_url": "${image}",
  1105. "ts": "${when}"
  1106. }
  1107. ]
  1108. }
  1109. EOF
  1110. )"
  1111. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1112. if [ "${httpcode}" = "200" ]; then
  1113. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1114. sent=$((sent + 1))
  1115. else
  1116. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1117. fi
  1118. done
  1119. [ ${sent} -gt 0 ] && return 0
  1120. return 1
  1121. }
  1122. # -----------------------------------------------------------------------------
  1123. # alerta sender
  1124. send_alerta() {
  1125. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1126. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1127. case "${status}" in
  1128. CRITICAL) severity="critical" ;;
  1129. WARNING) severity="warning" ;;
  1130. CLEAR) severity="cleared" ;;
  1131. *) severity="indeterminate" ;;
  1132. esac
  1133. if [[ ${chart} == httpcheck* ]]; then
  1134. resource=$chart
  1135. event=$name
  1136. else
  1137. resource="${host}:${family}"
  1138. event="${chart}.${name}"
  1139. fi
  1140. for channel in ${channels}; do
  1141. payload="$(
  1142. cat <<EOF
  1143. {
  1144. "resource": "${resource}",
  1145. "event": "${event}",
  1146. "environment": "${channel}",
  1147. "severity": "${severity}",
  1148. "service": ["Netdata"],
  1149. "group": "Performance",
  1150. "value": "${value_string}",
  1151. "text": "${info}",
  1152. "tags": ["alarm_id:${alarm_id}"],
  1153. "attributes": {
  1154. "roles": "${roles}",
  1155. "name": "${name}",
  1156. "chart": "${chart}",
  1157. "family": "${family}",
  1158. "source": "${src}",
  1159. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1160. },
  1161. "origin": "netdata/${host}",
  1162. "type": "netdataAlarm",
  1163. "rawData": "${BASH_ARGV[@]}"
  1164. }
  1165. EOF
  1166. )"
  1167. if [ -n "${ALERTA_API_KEY}" ]; then
  1168. auth="Key ${ALERTA_API_KEY}"
  1169. fi
  1170. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1171. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1172. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1173. sent=$((sent + 1))
  1174. elif [ "${httpcode}" = "202" ]; then
  1175. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1176. else
  1177. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1178. fi
  1179. done
  1180. [ ${sent} -gt 0 ] && return 0
  1181. return 1
  1182. }
  1183. # -----------------------------------------------------------------------------
  1184. # flock sender
  1185. send_flock() {
  1186. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1187. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1188. case "${status}" in
  1189. WARNING) color="warning" ;;
  1190. CRITICAL) color="danger" ;;
  1191. CLEAR) color="good" ;;
  1192. *) color="#777777" ;;
  1193. esac
  1194. for channel in ${channels}; do
  1195. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1196. \"sendAs\": {
  1197. \"name\" : \"netdata on ${host}\",
  1198. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1199. },
  1200. \"text\": \"${host} *${status_message}*\",
  1201. \"timestamp\": \"${when}\",
  1202. \"attachments\": [
  1203. {
  1204. \"description\": \"${chart} (${family}) - ${info}\",
  1205. \"color\": \"${color}\",
  1206. \"title\": \"${alarm}\",
  1207. \"url\": \"${goto_url}\",
  1208. \"text\": \"${info}\",
  1209. \"views\": {
  1210. \"image\": {
  1211. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1212. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1213. \"filename\": \"${image}\"
  1214. }
  1215. }
  1216. }
  1217. ]
  1218. }")
  1219. if [ "${httpcode}" = "200" ]; then
  1220. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1221. sent=$((sent + 1))
  1222. else
  1223. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1224. fi
  1225. done
  1226. [ ${sent} -gt 0 ] && return 0
  1227. return 1
  1228. }
  1229. # -----------------------------------------------------------------------------
  1230. # discord sender
  1231. send_discord() {
  1232. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1233. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1234. case "${status}" in
  1235. WARNING) color="warning" ;;
  1236. CRITICAL) color="danger" ;;
  1237. CLEAR) color="good" ;;
  1238. *) color="#777777" ;;
  1239. esac
  1240. for channel in ${channels}; do
  1241. username="netdata on ${host}"
  1242. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1243. payload="$(
  1244. cat <<EOF
  1245. {
  1246. "channel": "#${channel}",
  1247. "username": "${username}",
  1248. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1249. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1250. "attachments": [
  1251. {
  1252. "color": "${color}",
  1253. "title": "${alarm}",
  1254. "title_link": "${goto_url}",
  1255. "text": "${info}",
  1256. "fields": [
  1257. {
  1258. "title": "${chart}",
  1259. "value": "${family}"
  1260. }
  1261. ],
  1262. "thumb_url": "${image}",
  1263. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1264. "footer": "${host}",
  1265. "ts": ${when}
  1266. }
  1267. ]
  1268. }
  1269. EOF
  1270. )"
  1271. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1272. if [ "${httpcode}" = "200" ]; then
  1273. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1274. sent=$((sent + 1))
  1275. else
  1276. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1277. fi
  1278. done
  1279. [ ${sent} -gt 0 ] && return 0
  1280. return 1
  1281. }
  1282. # -----------------------------------------------------------------------------
  1283. # fleep sender
  1284. send_fleep() {
  1285. local httpcode sent=0 webhooks="${1}" data message
  1286. if [ "${SEND_FLEEP}" = "YES" ]; then
  1287. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1288. for hook in ${webhooks}; do
  1289. data="{ "
  1290. data="${data} 'message': '${message}', "
  1291. data="${data} 'user': '${FLEEP_SENDER}' "
  1292. data="${data} }"
  1293. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1294. if [ "${httpcode}" = "200" ]; then
  1295. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1296. sent=$((sent + 1))
  1297. else
  1298. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1299. fi
  1300. done
  1301. [ ${sent} -gt 0 ] && return 0
  1302. fi
  1303. return 1
  1304. }
  1305. # -----------------------------------------------------------------------------
  1306. # Prowl sender
  1307. send_prowl() {
  1308. local httpcode sent=0 data message keys prio=0 alarm_url event
  1309. if [ "${SEND_PROWL}" = "YES" ]; then
  1310. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1311. message="description=${message}"
  1312. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1313. keys="apikey=${keys}"
  1314. app="application=Netdata"
  1315. case "${status}" in
  1316. CRITICAL)
  1317. prio=2
  1318. ;;
  1319. WARNING)
  1320. prio=1
  1321. ;;
  1322. esac
  1323. prio="priority=${prio}"
  1324. alarm_url="$(urlencode ${goto_url})"
  1325. alarm_url="url=${alarm_url}"
  1326. event="$(urlencode "${host} ${status_message}")"
  1327. event="event=${event}"
  1328. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1329. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1330. if [ "${httpcode}" = "200" ]; then
  1331. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1332. sent=1
  1333. else
  1334. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1335. fi
  1336. [ ${sent} -gt 0 ] && return 0
  1337. fi
  1338. return 1
  1339. }
  1340. # -----------------------------------------------------------------------------
  1341. # irc sender
  1342. send_irc() {
  1343. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1344. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ]; then
  1345. case "${status}" in
  1346. WARNING) color="warning" ;;
  1347. CRITICAL) color="danger" ;;
  1348. CLEAR) color="good" ;;
  1349. *) color="#777777" ;;
  1350. esac
  1351. for CHANNEL in ${CHANNELS}; do
  1352. error=0
  1353. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${MESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" 6667)
  1354. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1355. for code in ${reply_codes}; do
  1356. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1357. error=1
  1358. break
  1359. fi
  1360. done
  1361. if [ "${error}" -eq 0 ]; then
  1362. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1363. sent=$((sent + 1))
  1364. else
  1365. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1366. fi
  1367. done
  1368. fi
  1369. [ ${sent} -gt 0 ] && return 0
  1370. return 1
  1371. }
  1372. # -----------------------------------------------------------------------------
  1373. # Amazon SNS sender
  1374. send_awssns() {
  1375. local targets="${1}" message='' sent=0 region=''
  1376. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1377. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1378. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1379. for target in ${targets}; do
  1380. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1381. region="$(echo ${target} | cut -f 4 -d ':')"
  1382. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1383. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1384. sent=$((sent + 1))
  1385. else
  1386. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1387. fi
  1388. done
  1389. [ ${sent} -gt 0 ] && return 0
  1390. return 1
  1391. }
  1392. # -----------------------------------------------------------------------------
  1393. # syslog sender
  1394. send_syslog() {
  1395. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1396. local priority='' message='' host='' port='' prefix=''
  1397. local temp1='' temp2=''
  1398. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1399. if [ "${status}" = "CRITICAL" ]; then
  1400. level='crit'
  1401. elif [ "${status}" = "WARNING" ]; then
  1402. level='warning'
  1403. fi
  1404. for target in ${targets}; do
  1405. priority="${facility}.${level}"
  1406. message=''
  1407. host=''
  1408. port=''
  1409. prefix=''
  1410. temp1=''
  1411. temp2=''
  1412. prefix=$(echo ${target} | cut -d '/' -f 2)
  1413. temp1=$(echo ${target} | cut -d '/' -f 1)
  1414. if [ ${prefix} != ${temp1} ]; then
  1415. if (echo ${temp1} | grep -q '@'); then
  1416. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1417. host=$(echo ${temp1} | cut -d '@' -f 2)
  1418. if [ ${temp2} != ${host} ]; then
  1419. priority=${temp2}
  1420. fi
  1421. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1422. if (echo ${host} | grep -E -q '\[.*\]'); then
  1423. if (echo ${port} | grep -q ']'); then
  1424. port=''
  1425. else
  1426. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1427. fi
  1428. else
  1429. if [ ${port} = ${host} ]; then
  1430. port=''
  1431. else
  1432. host=$(echo ${host} | cut -d ':' -f 1)
  1433. fi
  1434. fi
  1435. else
  1436. priority=${temp1}
  1437. fi
  1438. fi
  1439. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1440. if [ ${host} ]; then
  1441. logger_options="${logger_options} -n ${host}"
  1442. if [ ${port} ]; then
  1443. logger_options="${logger_options} -P ${port}"
  1444. fi
  1445. fi
  1446. ${logger} -p ${priority} ${logger_options} "${message}"
  1447. done
  1448. return $?
  1449. }
  1450. # -----------------------------------------------------------------------------
  1451. # SMS sender
  1452. send_sms() {
  1453. local recipients="${1}" errcode errmessage sent=0
  1454. # Human readable SMS
  1455. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1456. # limit it to 160 characters
  1457. msg="${msg:0:160}"
  1458. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1459. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1460. for phone in ${recipients}; do
  1461. errmessage=$($sendsms $phone "$msg" 2>&1)
  1462. errcode=$?
  1463. if [ ${errcode} -eq 0 ]; then
  1464. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1465. sent=$((sent + 1))
  1466. else
  1467. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1468. fi
  1469. done
  1470. [ ${sent} -gt 0 ] && return 0
  1471. fi
  1472. return 1
  1473. }
  1474. # -----------------------------------------------------------------------------
  1475. # prepare the content of the notification
  1476. # the url to send the user on click
  1477. urlencode "${args_host}" >/dev/null
  1478. url_host="${REPLY}"
  1479. urlencode "${chart}" >/dev/null
  1480. url_chart="${REPLY}"
  1481. urlencode "${family}" >/dev/null
  1482. url_family="${REPLY}"
  1483. urlencode "${name}" >/dev/null
  1484. url_name="${REPLY}"
  1485. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
  1486. GOTOCLOUD=0
  1487. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1488. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1489. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1490. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1491. fi
  1492. fi
  1493. if [ ! -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1494. GOTOCLOUD=1
  1495. fi
  1496. fi
  1497. if [ ${GOTOCLOUD} -eq 0 ]; then
  1498. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  1499. else
  1500. goto_url="https://netdata.cloud/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  1501. fi
  1502. # the severity of the alarm
  1503. severity="${status}"
  1504. # the time the alarm was raised
  1505. duration4human ${duration} >/dev/null
  1506. duration_txt="${REPLY}"
  1507. duration4human ${non_clear_duration} >/dev/null
  1508. non_clear_duration_txt="${REPLY}"
  1509. raised_for="(was ${old_status,,} for ${duration_txt})"
  1510. # the key status message
  1511. status_message="status unknown"
  1512. # the color of the alarm
  1513. color="grey"
  1514. # the alarm value
  1515. alarm="${name//_/ } = ${value_string}"
  1516. # the image of the alarm
  1517. image="${images_base_url}/images/banner-icon-144x144.png"
  1518. # prepare the title based on status
  1519. case "${status}" in
  1520. CRITICAL)
  1521. image="${images_base_url}/images/alert-128-red.png"
  1522. status_message="is critical"
  1523. color="#ca414b"
  1524. ;;
  1525. WARNING)
  1526. image="${images_base_url}/images/alert-128-orange.png"
  1527. status_message="needs attention"
  1528. color="#ffc107"
  1529. ;;
  1530. CLEAR)
  1531. image="${images_base_url}/images/check-mark-2-128-green.png"
  1532. status_message="recovered"
  1533. color="#77ca6d"
  1534. ;;
  1535. esac
  1536. if [ "${status}" = "CLEAR" ]; then
  1537. severity="Recovered from ${old_status}"
  1538. if [ ${non_clear_duration} -gt ${duration} ]; then
  1539. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1540. fi
  1541. # don't show the value when the status is CLEAR
  1542. # for certain alarms, this value might not have any meaning
  1543. alarm="${name//_/ } ${raised_for}"
  1544. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  1545. severity="Escalated to ${status}"
  1546. if [ ${non_clear_duration} -gt ${duration} ]; then
  1547. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1548. fi
  1549. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  1550. severity="Demoted to ${status}"
  1551. if [ ${non_clear_duration} -gt ${duration} ]; then
  1552. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1553. fi
  1554. else
  1555. raised_for=
  1556. fi
  1557. # prepare HTML versions of elements
  1558. info_html=
  1559. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  1560. raised_for_html=
  1561. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1562. # -----------------------------------------------------------------------------
  1563. # send the slack notification
  1564. # slack aggregates posts from the same username
  1565. # so we use "${host} ${status}" as the bot username, to make them diff
  1566. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1567. SENT_SLACK=$?
  1568. # -----------------------------------------------------------------------------
  1569. # send the Microsoft notification
  1570. # Microsoft team aggregates posts from the same username
  1571. # so we use "${host} ${status}" as the bot username, to make them diff
  1572. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1573. SENT_MSTEAM=$?
  1574. # -----------------------------------------------------------------------------
  1575. # send the rocketchat notification
  1576. # rocketchat aggregates posts from the same username
  1577. # so we use "${host} ${status}" as the bot username, to make them diff
  1578. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1579. SENT_ROCKETCHAT=$?
  1580. # -----------------------------------------------------------------------------
  1581. # send the alerta notification
  1582. # alerta aggregates posts from the same username
  1583. # so we use "${host} ${status}" as the bot username, to make them diff
  1584. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1585. SENT_ALERTA=$?
  1586. # -----------------------------------------------------------------------------
  1587. # send the flock notification
  1588. # flock aggregates posts from the same username
  1589. # so we use "${host} ${status}" as the bot username, to make them diff
  1590. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1591. SENT_FLOCK=$?
  1592. # -----------------------------------------------------------------------------
  1593. # send the discord notification
  1594. # discord aggregates posts from the same username
  1595. # so we use "${host} ${status}" as the bot username, to make them diff
  1596. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1597. SENT_DISCORD=$?
  1598. # -----------------------------------------------------------------------------
  1599. # send the pushover notification
  1600. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1601. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1602. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1603. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1604. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1605. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1606. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1607. <small><small>The source of this alarm is line ${src}</small></small>
  1608. "
  1609. SENT_PUSHOVER=$?
  1610. # -----------------------------------------------------------------------------
  1611. # send the pushbullet notification
  1612. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  1613. Severity: ${severity}\\n
  1614. Chart: ${chart}\\n
  1615. Family: ${family}\\n
  1616. $(date -d @${when})\\n
  1617. The source of this alarm is line ${src}"
  1618. SENT_PUSHBULLET=$?
  1619. # -----------------------------------------------------------------------------
  1620. # send the twilio SMS
  1621. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1622. Severity: ${severity}
  1623. Chart: ${chart}
  1624. Family: ${family}
  1625. ${info}"
  1626. SENT_TWILIO=$?
  1627. # -----------------------------------------------------------------------------
  1628. # send the messagebird SMS
  1629. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1630. Severity: ${severity}
  1631. Chart: ${chart}
  1632. Family: ${family}
  1633. ${info}"
  1634. SENT_MESSAGEBIRD=$?
  1635. # -----------------------------------------------------------------------------
  1636. # send the kavenegar SMS
  1637. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1638. Severity: ${severity}
  1639. Chart: ${chart}
  1640. Family: ${family}
  1641. ${info}"
  1642. SENT_KAVENEGAR=$?
  1643. # -----------------------------------------------------------------------------
  1644. # send the telegram.org message
  1645. # https://core.telegram.org/bots/api#formatting-options
  1646. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1647. ${chart} (${family})
  1648. <a href=\"${goto_url}\">${alarm}</a>
  1649. <i>${info}</i>"
  1650. SENT_TELEGRAM=$?
  1651. # -----------------------------------------------------------------------------
  1652. # send the kafka message
  1653. send_kafka
  1654. SENT_KAFKA=$?
  1655. # -----------------------------------------------------------------------------
  1656. # send the pagerduty.com message
  1657. send_pd "${to_pd}"
  1658. SENT_PD=$?
  1659. # -----------------------------------------------------------------------------
  1660. # send the fleep message
  1661. send_fleep "${to_fleep}"
  1662. SENT_FLEEP=$?
  1663. # -----------------------------------------------------------------------------
  1664. # send the Prowl message
  1665. send_prowl "${to_prowl}"
  1666. SENT_PROWL=$?
  1667. # -----------------------------------------------------------------------------
  1668. # send the irc message
  1669. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1670. Severity: ${severity}
  1671. Chart: ${chart}
  1672. Family: ${family}
  1673. ${info}"
  1674. SENT_IRC=$?
  1675. # -----------------------------------------------------------------------------
  1676. # send the SMS message with smstools3
  1677. send_sms "${to_sms}"
  1678. SENT_SMS=$?
  1679. # -----------------------------------------------------------------------------
  1680. # send the custom message
  1681. send_custom() {
  1682. # is it enabled?
  1683. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1684. # do we have any sender?
  1685. [ -z "${1}" ] && return 1
  1686. # call the custom_sender function
  1687. custom_sender "${@}"
  1688. }
  1689. send_custom "${to_custom}"
  1690. SENT_CUSTOM=$?
  1691. # -----------------------------------------------------------------------------
  1692. # send hipchat message
  1693. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1694. ${host} ${status_message}<br/> \
  1695. <b>${alarm}</b> ${info_html}<br/> \
  1696. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1697. <b>${date}${raised_for_html}</b><br/> \
  1698. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1699. (source of alarm ${src}) \
  1700. "
  1701. SENT_HIPCHAT=$?
  1702. # -----------------------------------------------------------------------------
  1703. # send the Amazon SNS message
  1704. send_awssns "${to_awssns}"
  1705. SENT_AWSSNS=$?
  1706. # -----------------------------------------------------------------------------
  1707. # send the syslog message
  1708. send_syslog "${to_syslog}"
  1709. SENT_SYSLOG=$?
  1710. # -----------------------------------------------------------------------------
  1711. # send the email
  1712. send_email <<EOF
  1713. To: ${to_email}
  1714. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1715. MIME-Version: 1.0
  1716. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1717. ${email_thread_headers}
  1718. This is a MIME-encoded multipart message
  1719. --multipart-boundary
  1720. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1721. Content-Disposition: inline
  1722. Content-Transfer-Encoding: 8bit
  1723. ${host} ${status_message}
  1724. ${alarm} ${info}
  1725. ${raised_for}
  1726. Chart : ${chart}
  1727. Family : ${family}
  1728. Severity: ${severity}
  1729. URL : ${goto_url}
  1730. Source : ${src}
  1731. Date : ${date}
  1732. Notification generated on ${host}
  1733. Evaluated Expression : ${calc_expression}
  1734. Expression Variables : ${calc_param_values}
  1735. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1736. --multipart-boundary
  1737. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1738. Content-Disposition: inline
  1739. Content-Transfer-Encoding: 8bit
  1740. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1741. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1742. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1743. <table>
  1744. <tbody>
  1745. <tr>
  1746. <td style="vertical-align: top;" valign="top"></td>
  1747. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1748. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1749. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1750. <tbody>
  1751. <tr>
  1752. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1753. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1754. </td>
  1755. </tr>
  1756. <tr>
  1757. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1758. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1759. </td>
  1760. </tr>
  1761. <tr>
  1762. <td style="vertical-align: top;" valign="top">
  1763. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1764. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  1765. <tbody>
  1766. <tr>
  1767. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  1768. <span>${chart}</span>
  1769. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  1770. </td>
  1771. </tr>
  1772. <tr style="margin: 0; padding: 0;">
  1773. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1774. <span><b>${alarm}</b>${info_html}</span>
  1775. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  1776. </td>
  1777. </tr>
  1778. <tr>
  1779. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1780. <span>${family}</span>
  1781. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  1782. </td>
  1783. </tr>
  1784. <tr style="margin: 0; padding: 0;">
  1785. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1786. <span>${severity}</span>
  1787. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  1788. </td>
  1789. </tr>
  1790. <tr style="margin: 0; padding: 0;">
  1791. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  1792. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  1793. </td>
  1794. </tr>
  1795. <tr style="margin: 0; padding: 0;">
  1796. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1797. <span>${calc_expression}</span>
  1798. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
  1799. </td>
  1800. </tr>
  1801. <tr style="margin: 0; padding: 0;">
  1802. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1803. <span>${calc_param_values}</span>
  1804. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
  1805. </td>
  1806. </tr>
  1807. <tr style="margin: 0; padding: 0;">
  1808. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1809. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1810. </td>
  1811. </tr>
  1812. <tr style="margin: 0; padding: 0;">
  1813. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  1814. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  1815. </td>
  1816. </tr>
  1817. <tr style="text-align: center; margin: 0; padding: 0;">
  1818. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  1819. </td>
  1820. </tr>
  1821. <tr style="text-align: center; margin: 0; padding: 0;">
  1822. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  1823. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  1824. </td>
  1825. </tr>
  1826. </tbody>
  1827. </table>
  1828. </div>
  1829. </td>
  1830. </tr>
  1831. </tbody>
  1832. </table>
  1833. </div>
  1834. </td>
  1835. </tr>
  1836. </tbody>
  1837. </table>
  1838. </body>
  1839. </html>
  1840. --multipart-boundary--
  1841. EOF
  1842. SENT_EMAIL=$?
  1843. # -----------------------------------------------------------------------------
  1844. # let netdata know
  1845. for state in "${SENT_EMAIL}" \
  1846. "${SENT_PUSHOVER}" \
  1847. "${SENT_TELEGRAM}" \
  1848. "${SENT_SLACK}" \
  1849. "${SENT_ROCKETCHAT}" \
  1850. "${SENT_ALERTA}" \
  1851. "${SENT_FLOCK}" \
  1852. "${SENT_DISCORD}" \
  1853. "${SENT_TWILIO}" \
  1854. "${SENT_HIPCHAT}" \
  1855. "${SENT_MESSAGEBIRD}" \
  1856. "${SENT_KAVENEGAR}" \
  1857. "${SENT_PUSHBULLET}" \
  1858. "${SENT_KAFKA}" \
  1859. "${SENT_PD}" \
  1860. "${SENT_FLEEP}" \
  1861. "${SENT_PROWL}" \
  1862. "${SENT_CUSTOM}" \
  1863. "${SENT_IRC}" \
  1864. "${SENT_AWSSNS}" \
  1865. "${SENT_SYSLOG}" \
  1866. "${SENT_SMS}" \
  1867. "${SENT_MSTEAM}"; do
  1868. if [ "${state}" -eq 0 ]; then
  1869. # we sent something
  1870. exit 0
  1871. fi
  1872. done
  1873. # we did not send anything
  1874. exit 1