alarm-notify.sh.in 77 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - irc notifications by @manosf
  32. # - custom notifications by @ktsaou
  33. # - syslog messages by @Ferroin
  34. # - Microsoft Team notification by @tioumen
  35. # - RocketChat notifications by @Hermsi1337 #3777
  36. # - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
  37. # -----------------------------------------------------------------------------
  38. # testing notifications
  39. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  40. if [ "${2}" = "test" ]; then
  41. recipient="${1}"
  42. else
  43. recipient="${2}"
  44. fi
  45. [ -z "${recipient}" ] && recipient="sysadmin"
  46. id=1
  47. last="CLEAR"
  48. test_res=0
  49. for x in "WARNING" "CRITICAL" "CLEAR"; do
  50. echo >&2
  51. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  52. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  53. #shellcheck disable=SC2181
  54. if [ $? -ne 0 ]; then
  55. echo >&2 "# FAILED"
  56. test_res=1
  57. else
  58. echo >&2 "# OK"
  59. fi
  60. last="${x}"
  61. id=$((id + 1))
  62. done
  63. exit $test_res
  64. fi
  65. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  66. export LC_ALL=C
  67. # -----------------------------------------------------------------------------
  68. PROGRAM_NAME="$(basename "${0}")"
  69. logdate() {
  70. date "+%Y-%m-%d %H:%M:%S"
  71. }
  72. log() {
  73. local status="${1}"
  74. shift
  75. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  76. }
  77. warning() {
  78. log WARNING "${@}"
  79. }
  80. error() {
  81. log ERROR "${@}"
  82. }
  83. info() {
  84. log INFO "${@}"
  85. }
  86. fatal() {
  87. log FATAL "${@}"
  88. exit 1
  89. }
  90. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  91. debug() {
  92. [ "${debug}" = "1" ] && log DEBUG "${@}"
  93. }
  94. docurl() {
  95. if [ -z "${curl}" ]; then
  96. error "${curl} is unset."
  97. return 1
  98. fi
  99. if [ "${debug}" = "1" ]; then
  100. echo >&2 "--- BEGIN curl command ---"
  101. printf >&2 "%q " ${curl} "${@}"
  102. echo >&2
  103. echo >&2 "--- END curl command ---"
  104. local out code ret
  105. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  106. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  107. ret=$?
  108. echo >&2 "--- BEGIN received response ---"
  109. cat >&2 "${out}"
  110. echo >&2
  111. echo >&2 "--- END received response ---"
  112. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  113. rm "${out}"
  114. echo "${code}"
  115. return ${ret}
  116. fi
  117. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  118. return $?
  119. }
  120. # -----------------------------------------------------------------------------
  121. # List of all the notification mechanisms we support.
  122. # Used in a couple of places to write more compact code.
  123. method_names="
  124. email
  125. pushover
  126. pushbullet
  127. telegram
  128. slack
  129. alerta
  130. flock
  131. discord
  132. hipchat
  133. twilio
  134. messagebird
  135. pd
  136. fleep
  137. syslog
  138. custom
  139. msteam
  140. kavenegar
  141. prowl
  142. irc
  143. awssns
  144. rocketchat
  145. sms
  146. hangouts
  147. "
  148. # -----------------------------------------------------------------------------
  149. # this is to be overwritten by the config file
  150. custom_sender() {
  151. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  152. }
  153. # -----------------------------------------------------------------------------
  154. # check for BASH v4+ (required for associative arrays)
  155. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  156. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  157. fi
  158. # -----------------------------------------------------------------------------
  159. # defaults to allow running this script by hand
  160. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  161. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  162. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  163. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  164. [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://netdata.cloud"
  165. # -----------------------------------------------------------------------------
  166. # parse command line parameters
  167. if [[ ${1} = "unittest" ]]; then
  168. unittest=1 # enable unit testing mode
  169. roles="${2}" # the role that should be used for unit testing
  170. cfgfile="${3}" # the location of the config file to use for unit testing
  171. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  172. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  173. else
  174. roles="${1}" # the roles that should be notified for this event
  175. args_host="${2}" # the host generated this event
  176. unique_id="${3}" # the unique id of this event
  177. alarm_id="${4}" # the unique id of the alarm that generated this event
  178. event_id="${5}" # the incremental id of the event, for this alarm id
  179. when="${6}" # the timestamp this event occurred
  180. name="${7}" # the name of the alarm, as given in netdata health.d entries
  181. chart="${8}" # the name of the chart (type.id)
  182. family="${9}" # the family of the chart
  183. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  184. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  185. value="${12}" # the current value of the alarm
  186. old_value="${13}" # the previous value of the alarm
  187. src="${14}" # the line number and file the alarm has been configured
  188. duration="${15}" # the duration in seconds of the previous alarm state
  189. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  190. units="${17}" # the units of the value
  191. info="${18}" # a short description of the alarm
  192. value_string="${19}" # friendly value (with units)
  193. # shellcheck disable=SC2034
  194. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  195. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  196. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  197. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  198. total_warnings="${23}" # Total number of alarms in WARNING state
  199. total_critical="${24}" # Total number of alarms in CRITICAL state
  200. fi
  201. # -----------------------------------------------------------------------------
  202. # find a suitable hostname to use, if netdata did not supply a hostname
  203. if [ -z ${args_host} ]; then
  204. this_host=$(hostname -s 2>/dev/null)
  205. host="${this_host}"
  206. args_host="${this_host}"
  207. else
  208. host="${args_host}"
  209. fi
  210. # -----------------------------------------------------------------------------
  211. # screen statuses we don't need to send a notification
  212. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  213. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  214. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  215. exit 1
  216. fi
  217. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  218. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  219. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  220. exit 1
  221. fi
  222. # -----------------------------------------------------------------------------
  223. # load configuration
  224. # By default fetch images from the global public registry.
  225. # This is required by default, since all notification methods need to download
  226. # images via the Internet, and private registries might not be reachable.
  227. # This can be overwritten at the configuration file.
  228. images_base_url="https://registry.my-netdata.io"
  229. # curl options to use
  230. curl_options=""
  231. # hostname handling
  232. use_fqdn="NO"
  233. # needed commands
  234. # if empty they will be searched in the system path
  235. curl=
  236. sendmail=
  237. # enable / disable features
  238. for method_name in ${method_names^^}; do
  239. declare SEND_${method_name}="YES"
  240. declare DEFAULT_RECIPIENT_${method_name}
  241. done
  242. for method_name in ${method_names}; do
  243. declare -A role_recipients_${method_name}
  244. done
  245. # slack configs
  246. SLACK_WEBHOOK_URL=
  247. # Microsoft Team configs
  248. MSTEAM_WEBHOOK_URL=
  249. # rocketchat configs
  250. ROCKETCHAT_WEBHOOK_URL=
  251. # alerta configs
  252. ALERTA_WEBHOOK_URL=
  253. ALERTA_API_KEY=
  254. # flock configs
  255. FLOCK_WEBHOOK_URL=
  256. # discord configs
  257. DISCORD_WEBHOOK_URL=
  258. # pushover configs
  259. PUSHOVER_APP_TOKEN=
  260. # pushbullet configs
  261. PUSHBULLET_ACCESS_TOKEN=
  262. PUSHBULLET_SOURCE_DEVICE=
  263. # twilio configs
  264. TWILIO_ACCOUNT_SID=
  265. TWILIO_ACCOUNT_TOKEN=
  266. TWILIO_NUMBER=
  267. # hipchat configs
  268. HIPCHAT_SERVER=
  269. HIPCHAT_AUTH_TOKEN=
  270. # messagebird configs
  271. MESSAGEBIRD_ACCESS_KEY=
  272. MESSAGEBIRD_NUMBER=
  273. # kavenegar configs
  274. KAVENEGAR_API_KEY=
  275. KAVENEGAR_SENDER=
  276. # telegram configs
  277. TELEGRAM_BOT_TOKEN=
  278. # kafka configs
  279. SEND_KAFKA="YES"
  280. KAFKA_URL=
  281. KAFKA_SENDER_IP=
  282. # pagerduty.com configs
  283. PD_SERVICE_KEY=
  284. # fleep.io configs
  285. FLEEP_SENDER="${host}"
  286. # Amazon SNS configs
  287. AWSSNS_MESSAGE_FORMAT=
  288. # syslog configs
  289. SYSLOG_FACILITY=
  290. # email configs
  291. EMAIL_SENDER=
  292. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  293. EMAIL_THREADING=
  294. EMAIL_PLAINTEXT_ONLY=
  295. # irc configs
  296. IRC_NICKNAME=
  297. IRC_REALNAME=
  298. IRC_NETWORK=
  299. # hangouts configs
  300. declare -A HANGOUTS_WEBHOOK_URI
  301. # load the stock and user configuration files
  302. # these will overwrite the variables above
  303. if [ ${unittest} ]; then
  304. if source "${cfgfile}"; then
  305. error "Failed to load requested config file."
  306. exit 1
  307. fi
  308. else
  309. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  310. if [ -f "${CONFIG}" ]; then
  311. debug "Loading config file '${CONFIG}'..."
  312. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  313. else
  314. warning "Cannot find file '${CONFIG}'."
  315. fi
  316. done
  317. fi
  318. # If we didn't autodetect the character set for e-mail and it wasn't
  319. # set by the user, we need to set it to a reasonable default. UTF-8
  320. # should be correct for almost all modern UNIX systems.
  321. if [ -z ${EMAIL_CHARSET} ]; then
  322. EMAIL_CHARSET="UTF-8"
  323. fi
  324. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  325. # unless we're sending an alarm for a slave system which we can't get the
  326. # FQDN of easily.
  327. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  328. host="$(hostname -f 2>/dev/null)"
  329. fi
  330. # -----------------------------------------------------------------------------
  331. # filter a recipient based on alarm event severity
  332. filter_recipient_by_criticality() {
  333. local method="${1}" x="${2}" r s
  334. shift
  335. r="${x/|*/}" # the recipient
  336. s="${x/*|/}" # the severity required for notifying this recipient
  337. # no severity filtering for this person
  338. [ "${r}" = "${s}" ] && return 0
  339. # the severity is invalid
  340. s="${s^^}"
  341. if [ "${s}" != "CRITICAL" ]; then
  342. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  343. return 0
  344. fi
  345. # create the status tracking directory for this user
  346. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  347. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  348. case "${status}" in
  349. CRITICAL)
  350. # make sure he will get future notifications for this alarm too
  351. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  352. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  353. return 0
  354. ;;
  355. WARNING)
  356. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  357. # we do not remove the file, so that he will get future notifications of this alarm
  358. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  359. return 0
  360. fi
  361. ;;
  362. *)
  363. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  364. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  365. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  366. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  367. return 0
  368. fi
  369. ;;
  370. esac
  371. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  372. return 1
  373. }
  374. # -----------------------------------------------------------------------------
  375. # verify the delivery methods supported
  376. # check slack
  377. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  378. # check rocketchat
  379. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  380. # check alerta
  381. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  382. # check flock
  383. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  384. # check discord
  385. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  386. # check pushover
  387. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  388. # check pushbullet
  389. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  390. # check twilio
  391. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  392. # check hipchat
  393. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  394. # check messagebird
  395. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  396. # check kavenegar
  397. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  398. # check telegram
  399. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  400. # check kafka
  401. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  402. # check irc
  403. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  404. # check hangouts
  405. [ ${#HANGOUTS_WEBHOOK_URI[@]} -eq 0 ] && SEND_HANGOUTS="NO"
  406. # check fleep
  407. #shellcheck disable=SC2153
  408. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  409. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  410. [ "${SEND_SLACK}" = "YES" ] ||
  411. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  412. [ "${SEND_ALERTA}" = "YES" ] ||
  413. [ "${SEND_PD}" = "YES" ] ||
  414. [ "${SEND_FLOCK}" = "YES" ] ||
  415. [ "${SEND_DISCORD}" = "YES" ] ||
  416. [ "${SEND_HIPCHAT}" = "YES" ] ||
  417. [ "${SEND_TWILIO}" = "YES" ] ||
  418. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  419. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  420. [ "${SEND_TELEGRAM}" = "YES" ] ||
  421. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  422. [ "${SEND_KAFKA}" = "YES" ] ||
  423. [ "${SEND_FLEEP}" = "YES" ] ||
  424. [ "${SEND_PROWL}" = "YES" ] ||
  425. [ "${SEND_HANGOUTS}" = "YES" ] ||
  426. [ "${SEND_CUSTOM}" = "YES" ] ||
  427. [ "${SEND_MSTEAM}" = "YES" ]; then
  428. # if we need curl, check for the curl command
  429. if [ -z "${curl}" ]; then
  430. curl="$(command -v curl 2>/dev/null)"
  431. fi
  432. if [ -z "${curl}" ]; then
  433. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  434. SEND_PUSHOVER="NO"
  435. SEND_PUSHBULLET="NO"
  436. SEND_TELEGRAM="NO"
  437. SEND_SLACK="NO"
  438. SEND_MSTEAM="NO"
  439. SEND_ROCKETCHAT="NO"
  440. SEND_ALERTA="NO"
  441. SEND_PD="NO"
  442. SEND_FLOCK="NO"
  443. SEND_DISCORD="NO"
  444. SEND_TWILIO="NO"
  445. SEND_HIPCHAT="NO"
  446. SEND_MESSAGEBIRD="NO"
  447. SEND_KAVENEGAR="NO"
  448. SEND_KAFKA="NO"
  449. SEND_FLEEP="NO"
  450. SEND_PROWL="NO"
  451. SEND_HANGOUTS="NO"
  452. SEND_CUSTOM="NO"
  453. fi
  454. fi
  455. if [ "${SEND_SMS}" = "YES" ]; then
  456. if [ -z "${sendsms}" ]; then
  457. sendsms="$(command -v sendsms 2>/dev/null)"
  458. fi
  459. if [ -z "${sendsms}" ]; then
  460. SEND_SMS="NO"
  461. fi
  462. fi
  463. # if we need sendmail, check for the sendmail command
  464. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  465. sendmail="$(command -v sendmail 2>/dev/null)"
  466. if [ -z "${sendmail}" ]; then
  467. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  468. SEND_EMAIL="NO"
  469. fi
  470. fi
  471. # if we need logger, check for the logger command
  472. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  473. logger="$(command -v logger 2>/dev/null)"
  474. if [ -z "${logger}" ]; then
  475. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  476. SEND_SYSLOG="NO"
  477. fi
  478. fi
  479. # if we need aws, check for the aws command
  480. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  481. aws="$(command -v aws 2>/dev/null)"
  482. if [ -z "${aws}" ]; then
  483. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  484. SEND_AWSSNS="NO"
  485. fi
  486. fi
  487. # -----------------------------------------------------------------------------
  488. # find the recipients' addresses per method
  489. # netdata may call us with multiple roles, and roles may have multiple but
  490. # overlapping recipients - so, here we find the unique recipients.
  491. for method_name in ${method_names}; do
  492. send_var="SEND_${method_name^^}"
  493. if [ "${!send_var}" = "NO" ]; then
  494. continue
  495. fi
  496. declare -A arr_var=()
  497. for x in ${roles//,/ }; do
  498. # the roles 'silent' and 'disabled' mean:
  499. # don't send a notification for this role
  500. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  501. continue
  502. fi
  503. role_recipients="role_recipients_${method_name}[$x]"
  504. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  505. a="${!role_recipients}"
  506. [ -z "${a}" ] && a="${!default_recipient_var}"
  507. for r in ${a//,/ }; do
  508. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  509. done
  510. done
  511. # build the list of recipients
  512. to_var="to_${method_name}"
  513. declare to_${method_name}="${!arr_var[*]}"
  514. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  515. done
  516. # -----------------------------------------------------------------------------
  517. # handle fixup of the email recipient list.
  518. fix_to_email() {
  519. to_email=
  520. while [ -n "${1}" ]; do
  521. [ -n "${to_email}" ] && to_email="${to_email}, "
  522. to_email="${to_email}${1}"
  523. shift 1
  524. done
  525. }
  526. # ${to_email} without quotes here
  527. fix_to_email ${to_email}
  528. # -----------------------------------------------------------------------------
  529. # handle output if we're running in unit test mode
  530. if [ ${unittest} ]; then
  531. for method_name in ${method_names}; do
  532. to_var="to_${method_name}"
  533. echo "results: ${method_name}: ${!to_var}"
  534. done
  535. exit 0
  536. fi
  537. # -----------------------------------------------------------------------------
  538. # check that we have at least a method enabled
  539. proceed=0
  540. for method in "${SEND_EMAIL}" \
  541. "${SEND_PUSHOVER}" \
  542. "${SEND_TELEGRAM}" \
  543. "${SEND_SLACK}" \
  544. "${SEND_ROCKETCHAT}" \
  545. "${SEND_ALERTA}" \
  546. "${SEND_FLOCK}" \
  547. "${SEND_DISCORD}" \
  548. "${SEND_TWILIO}" \
  549. "${SEND_HIPCHAT}" \
  550. "${SEND_MESSAGEBIRD}" \
  551. "${SEND_KAVENEGAR}" \
  552. "${SEND_PUSHBULLET}" \
  553. "${SEND_KAFKA}" \
  554. "${SEND_PD}" \
  555. "${SEND_FLEEP}" \
  556. "${SEND_PROWL}" \
  557. "${SEND_CUSTOM}" \
  558. "${SEND_IRC}" \
  559. "${SEND_HANGOUTS}" \
  560. "${SEND_AWSSNS}" \
  561. "${SEND_SYSLOG}" \
  562. "${SEND_SMS}" \
  563. "${SEND_MSTEAM}"; do
  564. if [ "${method}" == "YES" ]; then
  565. proceed=1
  566. break
  567. fi
  568. done
  569. if [ "$proceed" -eq 0 ]; then
  570. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  571. fi
  572. # -----------------------------------------------------------------------------
  573. # get the date the alarm happened
  574. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  575. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  576. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  577. [ -z "${date}" ] && date=$(date 2>/dev/null)
  578. # ----------------------------------------------------------------------------
  579. # prepare some extra headers if we've been asked to thread e-mails
  580. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  581. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
  582. else
  583. email_thread_headers=
  584. fi
  585. # -----------------------------------------------------------------------------
  586. # function to URL encode a string
  587. urlencode() {
  588. local string="${1}" strlen encoded pos c o
  589. strlen=${#string}
  590. for ((pos = 0; pos < strlen; pos++)); do
  591. c=${string:pos:1}
  592. case "${c}" in
  593. [-_.~a-zA-Z0-9])
  594. o="${c}"
  595. ;;
  596. *)
  597. printf -v o '%%%02x' "'${c}"
  598. ;;
  599. esac
  600. encoded+="${o}"
  601. done
  602. REPLY="${encoded}"
  603. echo "${REPLY}"
  604. }
  605. # -----------------------------------------------------------------------------
  606. # function to convert a duration in seconds, to a human readable duration
  607. # using DAYS, MINUTES, SECONDS
  608. duration4human() {
  609. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  610. d=$((s / 86400))
  611. s=$((s - (d * 86400)))
  612. h=$((s / 3600))
  613. s=$((s - (h * 3600)))
  614. m=$((s / 60))
  615. s=$((s - (m * 60)))
  616. if [ ${d} -gt 0 ]; then
  617. [ ${m} -ge 30 ] && h=$((h + 1))
  618. [ ${d} -gt 1 ] && ds="days"
  619. [ ${h} -gt 1 ] && hs="hours"
  620. if [ ${h} -gt 0 ]; then
  621. ret="${d} ${ds} and ${h} ${hs}"
  622. else
  623. ret="${d} ${ds}"
  624. fi
  625. elif [ ${h} -gt 0 ]; then
  626. [ ${s} -ge 30 ] && m=$((m + 1))
  627. [ ${h} -gt 1 ] && hs="hours"
  628. [ ${m} -gt 1 ] && ms="minutes"
  629. if [ ${m} -gt 0 ]; then
  630. ret="${h} ${hs} and ${m} ${ms}"
  631. else
  632. ret="${h} ${hs}"
  633. fi
  634. elif [ ${m} -gt 0 ]; then
  635. [ ${m} -gt 1 ] && ms="minutes"
  636. [ ${s} -gt 1 ] && ss="seconds"
  637. if [ ${s} -gt 0 ]; then
  638. ret="${m} ${ms} and ${s} ${ss}"
  639. else
  640. ret="${m} ${ms}"
  641. fi
  642. else
  643. [ ${s} -gt 1 ] && ss="seconds"
  644. ret="${s} ${ss}"
  645. fi
  646. REPLY="${ret}"
  647. echo "${REPLY}"
  648. }
  649. # -----------------------------------------------------------------------------
  650. # email sender
  651. send_email() {
  652. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  653. if [ "${SEND_EMAIL}" = "YES" ]; then
  654. if [ -n "${EMAIL_SENDER}" ]; then
  655. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  656. # the name includes double quotes
  657. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  658. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  659. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  660. # the name includes single quotes
  661. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  662. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  663. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  664. # the name does not have any quotes
  665. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  666. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  667. fi
  668. fi
  669. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  670. [ -n "${sender_name}" ] && opts+=(-F "${sender_name}")
  671. if [ "${debug}" = "1" ]; then
  672. echo >&2 "--- BEGIN sendmail command ---"
  673. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  674. echo >&2
  675. echo >&2 "--- END sendmail command ---"
  676. fi
  677. "${sendmail}" -t "${opts[@]}"
  678. ret=$?
  679. if [ ${ret} -eq 0 ]; then
  680. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  681. return 0
  682. else
  683. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  684. return 1
  685. fi
  686. fi
  687. return 1
  688. }
  689. # -----------------------------------------------------------------------------
  690. # pushover sender
  691. send_pushover() {
  692. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  693. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  694. # https://pushover.net/api
  695. priority=-2
  696. case "${status}" in
  697. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  698. WARNING) priority=0 ;; # normal priority: respect quiet hours
  699. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  700. *) priority=-2 ;; # lowest priority: no notification at all
  701. esac
  702. for user in ${usertokens}; do
  703. httpcode=$(docurl \
  704. --form-string "token=${apptoken}" \
  705. --form-string "user=${user}" \
  706. --form-string "html=1" \
  707. --form-string "title=${title}" \
  708. --form-string "message=${message}" \
  709. --form-string "timestamp=${when}" \
  710. --form-string "url=${url}" \
  711. --form-string "url_title=Open netdata dashboard to view the alarm" \
  712. --form-string "priority=${priority}" \
  713. https://api.pushover.net/1/messages.json)
  714. if [ "${httpcode}" = "200" ]; then
  715. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  716. sent=$((sent + 1))
  717. else
  718. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  719. fi
  720. done
  721. [ ${sent} -gt 0 ] && return 0
  722. fi
  723. return 1
  724. }
  725. # -----------------------------------------------------------------------------
  726. # pushbullet sender
  727. send_pushbullet() {
  728. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  729. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  730. #https://docs.pushbullet.com/#create-push
  731. for user in ${recipients}; do
  732. httpcode=$(docurl \
  733. --header 'Access-Token: '${userapikey}'' \
  734. --header 'Content-Type: application/json' \
  735. --data-binary @<(
  736. cat <<EOF
  737. {"title": "${title}",
  738. "type": "link",
  739. "email": "${user}",
  740. "body": "$(echo -n ${message})",
  741. "url": "${url}",
  742. "source_device_iden": "${source_device}"}
  743. EOF
  744. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  745. if [ "${httpcode}" = "200" ]; then
  746. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  747. sent=$((sent + 1))
  748. else
  749. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  750. fi
  751. done
  752. [ ${sent} -gt 0 ] && return 0
  753. fi
  754. return 1
  755. }
  756. # -----------------------------------------------------------------------------
  757. # kafka sender
  758. send_kafka() {
  759. local httpcode sent=0
  760. if [ "${SEND_KAFKA}" = "YES" ]; then
  761. httpcode=$(docurl -X POST \
  762. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  763. "${KAFKA_URL}")
  764. if [ "${httpcode}" = "204" ]; then
  765. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  766. sent=$((sent + 1))
  767. else
  768. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  769. fi
  770. [ ${sent} -gt 0 ] && return 0
  771. fi
  772. return 1
  773. }
  774. # -----------------------------------------------------------------------------
  775. # pagerduty.com sender
  776. send_pd() {
  777. local recipients="${1}" sent=0
  778. unset t
  779. case ${status} in
  780. CLEAR) t='resolve' ;;
  781. WARNING) t='trigger' ;;
  782. CRITICAL) t='trigger' ;;
  783. esac
  784. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  785. for PD_SERVICE_KEY in ${recipients}; do
  786. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  787. payload="$(
  788. cat <<EOF
  789. {
  790. "service_key": "${PD_SERVICE_KEY}",
  791. "event_type": "${t}",
  792. "incident_key" : "${alarm_id}",
  793. "description": "${d}",
  794. "details": {
  795. "value_w_units": "${value_string}",
  796. "when": "${when}",
  797. "duration" : "${duration}",
  798. "roles": "${roles}",
  799. "alarm_id" : "${alarm_id}",
  800. "name" : "${name}",
  801. "chart" : "${chart}",
  802. "family" : "${family}",
  803. "status" : "${status}",
  804. "old_status" : "${old_status}",
  805. "value" : "${value}",
  806. "old_value" : "${old_value}",
  807. "src" : "${src}",
  808. "non_clear_duration" : "${non_clear_duration}",
  809. "units" : "${units}",
  810. "info" : "${info}"
  811. }
  812. }
  813. EOF
  814. )"
  815. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  816. if [ "${httpcode}" = "200" ]; then
  817. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  818. sent=$((sent + 1))
  819. else
  820. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  821. fi
  822. done
  823. [ ${sent} -gt 0 ] && return 0
  824. fi
  825. return 1
  826. }
  827. # -----------------------------------------------------------------------------
  828. # twilio sender
  829. send_twilio() {
  830. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  831. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  832. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  833. for user in ${recipients}; do
  834. httpcode=$(docurl -X POST \
  835. --data-urlencode "From=${twilionumber}" \
  836. --data-urlencode "To=${user}" \
  837. --data-urlencode "Body=${title} ${message}" \
  838. -u "${accountsid}:${accounttoken}" \
  839. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  840. if [ "${httpcode}" = "201" ]; then
  841. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  842. sent=$((sent + 1))
  843. else
  844. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  845. fi
  846. done
  847. [ ${sent} -gt 0 ] && return 0
  848. fi
  849. return 1
  850. }
  851. # -----------------------------------------------------------------------------
  852. # hipchat sender
  853. send_hipchat() {
  854. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  855. # remove <small></small> from the message
  856. message="${message//<small>/}"
  857. message="${message//<\/small>/}"
  858. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  859. # Valid values: html, text.
  860. # Defaults to 'html'.
  861. msg_format="html"
  862. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  863. case "${status}" in
  864. WARNING) color="yellow" ;;
  865. CRITICAL) color="red" ;;
  866. CLEAR) color="green" ;;
  867. *) color="gray" ;;
  868. esac
  869. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  870. # Each recipient's notification preferences are taken into account.
  871. # Defaults to false.
  872. notify="true"
  873. for room in ${recipients}; do
  874. httpcode=$(docurl -X POST \
  875. -H "Content-type: application/json" \
  876. -H "Authorization: Bearer ${authtoken}" \
  877. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  878. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  879. if [ "${httpcode}" = "204" ]; then
  880. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  881. sent=$((sent + 1))
  882. else
  883. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  884. fi
  885. done
  886. [ ${sent} -gt 0 ] && return 0
  887. fi
  888. return 1
  889. }
  890. # -----------------------------------------------------------------------------
  891. # messagebird sender
  892. send_messagebird() {
  893. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  894. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  895. #https://developers.messagebird.com/docs/messaging
  896. for user in ${recipients}; do
  897. httpcode=$(docurl -X POST \
  898. --data-urlencode "originator=${messagebirdnumber}" \
  899. --data-urlencode "recipients=${user}" \
  900. --data-urlencode "body=${title} ${message}" \
  901. --data-urlencode "datacoding=auto" \
  902. -H "Authorization: AccessKey ${accesskey}" \
  903. "https://rest.messagebird.com/messages")
  904. if [ "${httpcode}" = "201" ]; then
  905. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  906. sent=$((sent + 1))
  907. else
  908. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  909. fi
  910. done
  911. [ ${sent} -gt 0 ] && return 0
  912. fi
  913. return 1
  914. }
  915. # -----------------------------------------------------------------------------
  916. # kavenegar sender
  917. send_kavenegar() {
  918. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  919. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  920. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  921. for user in ${recipients}; do
  922. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  923. --data-urlencode "sender=${kavenegarsender}" \
  924. --data-urlencode "receptor=${user}" \
  925. --data-urlencode "message=${title} ${message}")
  926. if [ "${httpcode}" = "200" ]; then
  927. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  928. sent=$((sent + 1))
  929. else
  930. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  931. fi
  932. done
  933. [ ${sent} -gt 0 ] && return 0
  934. fi
  935. return 1
  936. }
  937. # -----------------------------------------------------------------------------
  938. # telegram sender
  939. send_telegram() {
  940. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  941. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  942. case "${status}" in
  943. WARNING) emoji="⚠️" ;;
  944. CRITICAL) emoji="🔴" ;;
  945. CLEAR) emoji="✅" ;;
  946. *) emoji="⚪️" ;;
  947. esac
  948. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  949. for chatid in ${chatids}; do
  950. # https://core.telegram.org/bots/api#sendmessage
  951. httpcode=$(docurl ${disableNotification} \
  952. --data-urlencode "parse_mode=HTML" \
  953. --data-urlencode "disable_web_page_preview=true" \
  954. --data-urlencode "text=${emoji} ${message}" \
  955. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  956. if [ "${httpcode}" = "200" ]; then
  957. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  958. sent=$((sent + 1))
  959. elif [ "${httpcode}" = "401" ]; then
  960. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  961. else
  962. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  963. fi
  964. done
  965. [ ${sent} -gt 0 ] && return 0
  966. fi
  967. return 1
  968. }
  969. # -----------------------------------------------------------------------------
  970. # Microsoft Team sender
  971. send_msteam() {
  972. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  973. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  974. case "${status}" in
  975. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}" ;;
  976. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}" ;;
  977. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}" ;;
  978. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}" ;;
  979. esac
  980. for channel in ${channels}; do
  981. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  982. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  983. payload="$(
  984. cat <<EOF
  985. {
  986. "@context": "http://schema.org/extensions",
  987. "@type": "MessageCard",
  988. "themeColor": "${color}",
  989. "title": "$icon Alert ${status} from netdata for ${host}",
  990. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  991. "potentialAction": [
  992. {
  993. "@type": "OpenUri",
  994. "name": "Netdata",
  995. "targets": [
  996. { "os": "default", "uri": "${goto_url}" }
  997. ]
  998. }
  999. ]
  1000. }
  1001. EOF
  1002. )"
  1003. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1004. webhook="${webhook//CHANNEL/${channel}}"
  1005. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  1006. if [ "${httpcode}" = "200" ]; then
  1007. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  1008. sent=$((sent + 1))
  1009. else
  1010. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  1011. fi
  1012. done
  1013. [ ${sent} -gt 0 ] && return 0
  1014. return 1
  1015. }
  1016. # slack sender
  1017. send_slack() {
  1018. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1019. [ "${SEND_SLACK}" != "YES" ] && return 1
  1020. case "${status}" in
  1021. WARNING) color="warning" ;;
  1022. CRITICAL) color="danger" ;;
  1023. CLEAR) color="good" ;;
  1024. *) color="#777777" ;;
  1025. esac
  1026. for channel in ${channels}; do
  1027. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1028. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1029. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1030. if [ "${channel}" = "#" ]; then
  1031. ch=""
  1032. chstr="without specifying a channel"
  1033. else
  1034. ch="\"channel\": \"${channel}\","
  1035. chstr="to '${channel}'"
  1036. fi
  1037. payload="$(
  1038. cat <<EOF
  1039. {
  1040. $ch
  1041. "username": "netdata on ${host}",
  1042. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1043. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1044. "attachments": [
  1045. {
  1046. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1047. "color": "${color}",
  1048. "title": "${alarm}",
  1049. "title_link": "${goto_url}",
  1050. "text": "${info}",
  1051. "fields": [
  1052. {
  1053. "title": "${chart}",
  1054. "short": true
  1055. },
  1056. {
  1057. "title": "${family}",
  1058. "short": true
  1059. }
  1060. ],
  1061. "thumb_url": "${image}",
  1062. "footer": "by ${host}",
  1063. "ts": ${when}
  1064. }
  1065. ]
  1066. }
  1067. EOF
  1068. )"
  1069. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1070. if [ "${httpcode}" = "200" ]; then
  1071. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1072. sent=$((sent + 1))
  1073. else
  1074. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1075. fi
  1076. done
  1077. [ ${sent} -gt 0 ] && return 0
  1078. return 1
  1079. }
  1080. # -----------------------------------------------------------------------------
  1081. # rocketchat sender
  1082. send_rocketchat() {
  1083. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1084. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1085. case "${status}" in
  1086. WARNING) color="warning" ;;
  1087. CRITICAL) color="danger" ;;
  1088. CLEAR) color="good" ;;
  1089. *) color="#777777" ;;
  1090. esac
  1091. for channel in ${channels}; do
  1092. payload="$(
  1093. cat <<EOF
  1094. {
  1095. "channel": "#${channel}",
  1096. "alias": "netdata on ${host}",
  1097. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1098. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1099. "attachments": [
  1100. {
  1101. "color": "${color}",
  1102. "title": "${alarm}",
  1103. "title_link": "${goto_url}",
  1104. "text": "${info}",
  1105. "fields": [
  1106. {
  1107. "title": "${chart}",
  1108. "short": true,
  1109. "value": "chart"
  1110. },
  1111. {
  1112. "title": "${family}",
  1113. "short": true,
  1114. "value": "family"
  1115. }
  1116. ],
  1117. "thumb_url": "${image}",
  1118. "ts": "${when}"
  1119. }
  1120. ]
  1121. }
  1122. EOF
  1123. )"
  1124. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1125. if [ "${httpcode}" = "200" ]; then
  1126. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1127. sent=$((sent + 1))
  1128. else
  1129. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1130. fi
  1131. done
  1132. [ ${sent} -gt 0 ] && return 0
  1133. return 1
  1134. }
  1135. # -----------------------------------------------------------------------------
  1136. # alerta sender
  1137. send_alerta() {
  1138. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1139. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1140. case "${status}" in
  1141. CRITICAL) severity="critical" ;;
  1142. WARNING) severity="warning" ;;
  1143. CLEAR) severity="cleared" ;;
  1144. *) severity="indeterminate" ;;
  1145. esac
  1146. if [[ ${chart} == httpcheck* ]]; then
  1147. resource=$chart
  1148. event=$name
  1149. else
  1150. resource="${host}:${family}"
  1151. event="${chart}.${name}"
  1152. fi
  1153. for channel in ${channels}; do
  1154. payload="$(
  1155. cat <<EOF
  1156. {
  1157. "resource": "${resource}",
  1158. "event": "${event}",
  1159. "environment": "${channel}",
  1160. "severity": "${severity}",
  1161. "service": ["Netdata"],
  1162. "group": "Performance",
  1163. "value": "${value_string}",
  1164. "text": "${info}",
  1165. "tags": ["alarm_id:${alarm_id}"],
  1166. "attributes": {
  1167. "roles": "${roles}",
  1168. "name": "${name}",
  1169. "chart": "${chart}",
  1170. "family": "${family}",
  1171. "source": "${src}",
  1172. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1173. },
  1174. "origin": "netdata/${host}",
  1175. "type": "netdataAlarm",
  1176. "rawData": "${BASH_ARGV[@]}"
  1177. }
  1178. EOF
  1179. )"
  1180. if [ -n "${ALERTA_API_KEY}" ]; then
  1181. auth="Key ${ALERTA_API_KEY}"
  1182. fi
  1183. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1184. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1185. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1186. sent=$((sent + 1))
  1187. elif [ "${httpcode}" = "202" ]; then
  1188. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1189. else
  1190. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1191. fi
  1192. done
  1193. [ ${sent} -gt 0 ] && return 0
  1194. return 1
  1195. }
  1196. # -----------------------------------------------------------------------------
  1197. # flock sender
  1198. send_flock() {
  1199. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1200. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1201. case "${status}" in
  1202. WARNING) color="warning" ;;
  1203. CRITICAL) color="danger" ;;
  1204. CLEAR) color="good" ;;
  1205. *) color="#777777" ;;
  1206. esac
  1207. for channel in ${channels}; do
  1208. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1209. \"sendAs\": {
  1210. \"name\" : \"netdata on ${host}\",
  1211. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1212. },
  1213. \"text\": \"${host} *${status_message}*\",
  1214. \"timestamp\": \"${when}\",
  1215. \"attachments\": [
  1216. {
  1217. \"description\": \"${chart} (${family}) - ${info}\",
  1218. \"color\": \"${color}\",
  1219. \"title\": \"${alarm}\",
  1220. \"url\": \"${goto_url}\",
  1221. \"text\": \"${info}\",
  1222. \"views\": {
  1223. \"image\": {
  1224. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1225. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1226. \"filename\": \"${image}\"
  1227. }
  1228. }
  1229. }
  1230. ]
  1231. }")
  1232. if [ "${httpcode}" = "200" ]; then
  1233. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1234. sent=$((sent + 1))
  1235. else
  1236. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1237. fi
  1238. done
  1239. [ ${sent} -gt 0 ] && return 0
  1240. return 1
  1241. }
  1242. # -----------------------------------------------------------------------------
  1243. # discord sender
  1244. send_discord() {
  1245. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1246. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1247. case "${status}" in
  1248. WARNING) color="warning" ;;
  1249. CRITICAL) color="danger" ;;
  1250. CLEAR) color="good" ;;
  1251. *) color="#777777" ;;
  1252. esac
  1253. for channel in ${channels}; do
  1254. username="netdata on ${host}"
  1255. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1256. payload="$(
  1257. cat <<EOF
  1258. {
  1259. "channel": "#${channel}",
  1260. "username": "${username}",
  1261. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1262. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1263. "attachments": [
  1264. {
  1265. "color": "${color}",
  1266. "title": "${alarm}",
  1267. "title_link": "${goto_url}",
  1268. "text": "${info}",
  1269. "fields": [
  1270. {
  1271. "title": "${chart}",
  1272. "value": "${family}"
  1273. }
  1274. ],
  1275. "thumb_url": "${image}",
  1276. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1277. "footer": "${host}",
  1278. "ts": ${when}
  1279. }
  1280. ]
  1281. }
  1282. EOF
  1283. )"
  1284. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1285. if [ "${httpcode}" = "200" ]; then
  1286. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1287. sent=$((sent + 1))
  1288. else
  1289. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1290. fi
  1291. done
  1292. [ ${sent} -gt 0 ] && return 0
  1293. return 1
  1294. }
  1295. # -----------------------------------------------------------------------------
  1296. # fleep sender
  1297. send_fleep() {
  1298. local httpcode sent=0 webhooks="${1}" data message
  1299. if [ "${SEND_FLEEP}" = "YES" ]; then
  1300. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1301. for hook in ${webhooks}; do
  1302. data="{ "
  1303. data="${data} 'message': '${message}', "
  1304. data="${data} 'user': '${FLEEP_SENDER}' "
  1305. data="${data} }"
  1306. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1307. if [ "${httpcode}" = "200" ]; then
  1308. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1309. sent=$((sent + 1))
  1310. else
  1311. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1312. fi
  1313. done
  1314. [ ${sent} -gt 0 ] && return 0
  1315. fi
  1316. return 1
  1317. }
  1318. # -----------------------------------------------------------------------------
  1319. # Prowl sender
  1320. send_prowl() {
  1321. local httpcode sent=0 data message keys prio=0 alarm_url event
  1322. if [ "${SEND_PROWL}" = "YES" ]; then
  1323. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1324. message="description=${message}"
  1325. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1326. keys="apikey=${keys}"
  1327. app="application=Netdata"
  1328. case "${status}" in
  1329. CRITICAL)
  1330. prio=2
  1331. ;;
  1332. WARNING)
  1333. prio=1
  1334. ;;
  1335. esac
  1336. prio="priority=${prio}"
  1337. alarm_url="$(urlencode ${goto_url})"
  1338. alarm_url="url=${alarm_url}"
  1339. event="$(urlencode "${host} ${status_message}")"
  1340. event="event=${event}"
  1341. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1342. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1343. if [ "${httpcode}" = "200" ]; then
  1344. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1345. sent=1
  1346. else
  1347. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1348. fi
  1349. [ ${sent} -gt 0 ] && return 0
  1350. fi
  1351. return 1
  1352. }
  1353. # -----------------------------------------------------------------------------
  1354. # irc sender
  1355. send_irc() {
  1356. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1357. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ]; then
  1358. case "${status}" in
  1359. WARNING) color="warning" ;;
  1360. CRITICAL) color="danger" ;;
  1361. CLEAR) color="good" ;;
  1362. *) color="#777777" ;;
  1363. esac
  1364. SNDMESSAGE="${MESSAGE//$'\n'/", "}"
  1365. for CHANNEL in ${CHANNELS}; do
  1366. error=0
  1367. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" 6667)
  1368. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1369. for code in ${reply_codes}; do
  1370. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1371. error=1
  1372. break
  1373. fi
  1374. done
  1375. if [ "${error}" -eq 0 ]; then
  1376. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1377. sent=$((sent + 1))
  1378. else
  1379. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1380. fi
  1381. done
  1382. fi
  1383. [ ${sent} -gt 0 ] && return 0
  1384. return 1
  1385. }
  1386. # -----------------------------------------------------------------------------
  1387. # Amazon SNS sender
  1388. send_awssns() {
  1389. local targets="${1}" message='' sent=0 region=''
  1390. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1391. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1392. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1393. for target in ${targets}; do
  1394. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1395. region="$(echo ${target} | cut -f 4 -d ':')"
  1396. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1397. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1398. sent=$((sent + 1))
  1399. else
  1400. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1401. fi
  1402. done
  1403. [ ${sent} -gt 0 ] && return 0
  1404. return 1
  1405. }
  1406. # -----------------------------------------------------------------------------
  1407. # syslog sender
  1408. send_syslog() {
  1409. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1410. local priority='' message='' host='' port='' prefix=''
  1411. local temp1='' temp2=''
  1412. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1413. if [ "${status}" = "CRITICAL" ]; then
  1414. level='crit'
  1415. elif [ "${status}" = "WARNING" ]; then
  1416. level='warning'
  1417. fi
  1418. for target in ${targets}; do
  1419. priority="${facility}.${level}"
  1420. message=''
  1421. host=''
  1422. port=''
  1423. prefix=''
  1424. temp1=''
  1425. temp2=''
  1426. prefix=$(echo ${target} | cut -d '/' -f 2)
  1427. temp1=$(echo ${target} | cut -d '/' -f 1)
  1428. if [ ${prefix} != ${temp1} ]; then
  1429. if (echo ${temp1} | grep -q '@'); then
  1430. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1431. host=$(echo ${temp1} | cut -d '@' -f 2)
  1432. if [ ${temp2} != ${host} ]; then
  1433. priority=${temp2}
  1434. fi
  1435. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1436. if (echo ${host} | grep -E -q '\[.*\]'); then
  1437. if (echo ${port} | grep -q ']'); then
  1438. port=''
  1439. else
  1440. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1441. fi
  1442. else
  1443. if [ ${port} = ${host} ]; then
  1444. port=''
  1445. else
  1446. host=$(echo ${host} | cut -d ':' -f 1)
  1447. fi
  1448. fi
  1449. else
  1450. priority=${temp1}
  1451. fi
  1452. fi
  1453. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1454. if [ ${host} ]; then
  1455. logger_options="${logger_options} -n ${host}"
  1456. if [ ${port} ]; then
  1457. logger_options="${logger_options} -P ${port}"
  1458. fi
  1459. fi
  1460. ${logger} -p ${priority} ${logger_options} "${message}"
  1461. done
  1462. return $?
  1463. }
  1464. # -----------------------------------------------------------------------------
  1465. # SMS sender
  1466. send_sms() {
  1467. local recipients="${1}" errcode errmessage sent=0
  1468. # Human readable SMS
  1469. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1470. # limit it to 160 characters
  1471. msg="${msg:0:160}"
  1472. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1473. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1474. for phone in ${recipients}; do
  1475. errmessage=$($sendsms $phone "$msg" 2>&1)
  1476. errcode=$?
  1477. if [ ${errcode} -eq 0 ]; then
  1478. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1479. sent=$((sent + 1))
  1480. else
  1481. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1482. fi
  1483. done
  1484. [ ${sent} -gt 0 ] && return 0
  1485. fi
  1486. return 1
  1487. }
  1488. # -----------------------------------------------------------------------------
  1489. # hangouts sender
  1490. send_hangouts() {
  1491. local rooms="${1}" httpcode sent=0 room color payload webhook
  1492. [ "${SEND_HANGOUTS}" != "YES" ] && return 1
  1493. case "${status}" in
  1494. WARNING) color="#ffa700" ;;
  1495. CRITICAL) color="#d62d20" ;;
  1496. CLEAR) color="#008744" ;;
  1497. *) color="#777777" ;;
  1498. esac
  1499. for room in ${rooms}; do
  1500. if [ -z "${HANGOUTS_WEBHOOK_URI[$room]}" ] ; then
  1501. info "Can't send Hangouts notification for: ${host} ${chart}.${name} to room ${room}. HANGOUTS_WEBHOOK_URI[$room] not defined"
  1502. else
  1503. webhook="${HANGOUTS_WEBHOOK_URI[$room]}"
  1504. payload="$(
  1505. cat <<EOF
  1506. {
  1507. "cards": [
  1508. {
  1509. "header": {
  1510. "title": "Netdata on ${host}",
  1511. "imageUrl": "${images_base_url}/images/banner-icon-144x144.png",
  1512. "imageStyle": "IMAGE"
  1513. },
  1514. "sections": [
  1515. {
  1516. "header": "<b>${host}</b>",
  1517. "widgets": [
  1518. {
  1519. "keyValue": {
  1520. "topLabel": "Status Message",
  1521. "content": "<b>${status_message}</b>",
  1522. "contentMultiline": "true",
  1523. "iconUrl": "${image}",
  1524. "onClick": {
  1525. "openLink": {
  1526. "url": "${goto_url}"
  1527. }
  1528. }
  1529. }
  1530. },
  1531. {
  1532. "keyValue": {
  1533. "topLabel": "${chart} | ${family}",
  1534. "content": "<font color=${color}>${alarm}</font>",
  1535. "contentMultiline": "true"
  1536. }
  1537. }
  1538. ]
  1539. },
  1540. {
  1541. "widgets": [
  1542. {
  1543. "textParagraph": {
  1544. "text": "<font color=\"#0057e7\">@ ${date}\n<b>${info}</b></font>"
  1545. }
  1546. }
  1547. ]
  1548. },
  1549. {
  1550. "widgets": [
  1551. {
  1552. "buttons": [
  1553. {
  1554. "textButton": {
  1555. "text": "Go to ${host}",
  1556. "onClick": {
  1557. "openLink": {
  1558. "url": "${goto_url}"
  1559. }
  1560. }
  1561. }
  1562. }
  1563. ]
  1564. }
  1565. ]
  1566. }
  1567. ]
  1568. }
  1569. ]
  1570. }
  1571. EOF
  1572. )"
  1573. httpcode=$(docurl -H "Content-Type: application/json" -X POST -d "${payload}" "${webhook}")
  1574. if [ "${httpcode}" = "200" ]; then
  1575. info "sent hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1576. sent=$((sent + 1))
  1577. else
  1578. error "failed to send hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP error code ${httpcode}."
  1579. fi
  1580. fi
  1581. done
  1582. [ ${sent} -gt 0 ] && return 0
  1583. return 1
  1584. }
  1585. # -----------------------------------------------------------------------------
  1586. # prepare the content of the notification
  1587. # the url to send the user on click
  1588. urlencode "${args_host}" >/dev/null
  1589. url_host="${REPLY}"
  1590. urlencode "${chart}" >/dev/null
  1591. url_chart="${REPLY}"
  1592. urlencode "${family}" >/dev/null
  1593. url_family="${REPLY}"
  1594. urlencode "${name}" >/dev/null
  1595. url_name="${REPLY}"
  1596. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}"
  1597. GOTOCLOUD=0
  1598. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1599. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1600. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1601. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1602. fi
  1603. fi
  1604. if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1605. GOTOCLOUD=1
  1606. fi
  1607. fi
  1608. if [ ${GOTOCLOUD} -eq 0 ]; then
  1609. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  1610. else
  1611. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  1612. fi
  1613. # the severity of the alarm
  1614. severity="${status}"
  1615. # the time the alarm was raised
  1616. duration4human ${duration} >/dev/null
  1617. duration_txt="${REPLY}"
  1618. duration4human ${non_clear_duration} >/dev/null
  1619. non_clear_duration_txt="${REPLY}"
  1620. raised_for="(was ${old_status,,} for ${duration_txt})"
  1621. # the key status message
  1622. status_message="status unknown"
  1623. # the color of the alarm
  1624. color="grey"
  1625. # the alarm value
  1626. alarm="${name//_/ } = ${value_string}"
  1627. # the image of the alarm
  1628. image="${images_base_url}/images/banner-icon-144x144.png"
  1629. # prepare the title based on status
  1630. case "${status}" in
  1631. CRITICAL)
  1632. image="${images_base_url}/images/alert-128-red.png"
  1633. status_message="is critical"
  1634. color="#ca414b"
  1635. ;;
  1636. WARNING)
  1637. image="${images_base_url}/images/alert-128-orange.png"
  1638. status_message="needs attention"
  1639. color="#ffc107"
  1640. ;;
  1641. CLEAR)
  1642. image="${images_base_url}/images/check-mark-2-128-green.png"
  1643. status_message="recovered"
  1644. color="#77ca6d"
  1645. ;;
  1646. esac
  1647. if [ "${status}" = "CLEAR" ]; then
  1648. severity="Recovered from ${old_status}"
  1649. if [ ${non_clear_duration} -gt ${duration} ]; then
  1650. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1651. fi
  1652. # don't show the value when the status is CLEAR
  1653. # for certain alarms, this value might not have any meaning
  1654. alarm="${name//_/ } ${raised_for}"
  1655. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  1656. severity="Escalated to ${status}"
  1657. if [ ${non_clear_duration} -gt ${duration} ]; then
  1658. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1659. fi
  1660. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  1661. severity="Demoted to ${status}"
  1662. if [ ${non_clear_duration} -gt ${duration} ]; then
  1663. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1664. fi
  1665. else
  1666. raised_for=
  1667. fi
  1668. # prepare HTML versions of elements
  1669. info_html=
  1670. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  1671. raised_for_html=
  1672. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1673. # -----------------------------------------------------------------------------
  1674. # send the slack notification
  1675. # slack aggregates posts from the same username
  1676. # so we use "${host} ${status}" as the bot username, to make them diff
  1677. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1678. SENT_SLACK=$?
  1679. # -----------------------------------------------------------------------------
  1680. # send the hangouts notification
  1681. # hangouts aggregates posts from the same room
  1682. # so we use "${host} ${status}" as the room, to make them diff
  1683. send_hangouts "${to_hangouts}"
  1684. SENT_HANGOUTS=$?
  1685. # -----------------------------------------------------------------------------
  1686. # send the Microsoft notification
  1687. # Microsoft team aggregates posts from the same username
  1688. # so we use "${host} ${status}" as the bot username, to make them diff
  1689. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1690. SENT_MSTEAM=$?
  1691. # -----------------------------------------------------------------------------
  1692. # send the rocketchat notification
  1693. # rocketchat aggregates posts from the same username
  1694. # so we use "${host} ${status}" as the bot username, to make them diff
  1695. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1696. SENT_ROCKETCHAT=$?
  1697. # -----------------------------------------------------------------------------
  1698. # send the alerta notification
  1699. # alerta aggregates posts from the same username
  1700. # so we use "${host} ${status}" as the bot username, to make them diff
  1701. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1702. SENT_ALERTA=$?
  1703. # -----------------------------------------------------------------------------
  1704. # send the flock notification
  1705. # flock aggregates posts from the same username
  1706. # so we use "${host} ${status}" as the bot username, to make them diff
  1707. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1708. SENT_FLOCK=$?
  1709. # -----------------------------------------------------------------------------
  1710. # send the discord notification
  1711. # discord aggregates posts from the same username
  1712. # so we use "${host} ${status}" as the bot username, to make them diff
  1713. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1714. SENT_DISCORD=$?
  1715. # -----------------------------------------------------------------------------
  1716. # send the pushover notification
  1717. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1718. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1719. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1720. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1721. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1722. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1723. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1724. <small><small>The source of this alarm is line ${src}</small></small>
  1725. "
  1726. SENT_PUSHOVER=$?
  1727. # -----------------------------------------------------------------------------
  1728. # send the pushbullet notification
  1729. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  1730. Severity: ${severity}\\n
  1731. Chart: ${chart}\\n
  1732. Family: ${family}\\n
  1733. ${date}\\n
  1734. The source of this alarm is line ${src}"
  1735. SENT_PUSHBULLET=$?
  1736. # -----------------------------------------------------------------------------
  1737. # send the twilio SMS
  1738. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1739. Severity: ${severity}
  1740. Chart: ${chart}
  1741. Family: ${family}
  1742. ${info}"
  1743. SENT_TWILIO=$?
  1744. # -----------------------------------------------------------------------------
  1745. # send the messagebird SMS
  1746. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1747. Severity: ${severity}
  1748. Chart: ${chart}
  1749. Family: ${family}
  1750. ${info}"
  1751. SENT_MESSAGEBIRD=$?
  1752. # -----------------------------------------------------------------------------
  1753. # send the kavenegar SMS
  1754. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1755. Severity: ${severity}
  1756. Chart: ${chart}
  1757. Family: ${family}
  1758. ${info}"
  1759. SENT_KAVENEGAR=$?
  1760. # -----------------------------------------------------------------------------
  1761. # send the telegram.org message
  1762. # https://core.telegram.org/bots/api#formatting-options
  1763. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1764. ${chart} (${family})
  1765. <a href=\"${goto_url}\">${alarm}</a>
  1766. <i>${info}</i>"
  1767. SENT_TELEGRAM=$?
  1768. # -----------------------------------------------------------------------------
  1769. # send the kafka message
  1770. send_kafka
  1771. SENT_KAFKA=$?
  1772. # -----------------------------------------------------------------------------
  1773. # send the pagerduty.com message
  1774. send_pd "${to_pd}"
  1775. SENT_PD=$?
  1776. # -----------------------------------------------------------------------------
  1777. # send the fleep message
  1778. send_fleep "${to_fleep}"
  1779. SENT_FLEEP=$?
  1780. # -----------------------------------------------------------------------------
  1781. # send the Prowl message
  1782. send_prowl "${to_prowl}"
  1783. SENT_PROWL=$?
  1784. # -----------------------------------------------------------------------------
  1785. # send the irc message
  1786. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1787. Severity: ${severity}
  1788. Chart: ${chart}
  1789. Family: ${family}
  1790. ${info}"
  1791. SENT_IRC=$?
  1792. # -----------------------------------------------------------------------------
  1793. # send the SMS message with smstools3
  1794. send_sms "${to_sms}"
  1795. SENT_SMS=$?
  1796. # -----------------------------------------------------------------------------
  1797. # send the custom message
  1798. send_custom() {
  1799. # is it enabled?
  1800. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1801. # do we have any sender?
  1802. [ -z "${1}" ] && return 1
  1803. # call the custom_sender function
  1804. custom_sender "${@}"
  1805. }
  1806. send_custom "${to_custom}"
  1807. SENT_CUSTOM=$?
  1808. # -----------------------------------------------------------------------------
  1809. # send hipchat message
  1810. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1811. ${host} ${status_message}<br/> \
  1812. <b>${alarm}</b> ${info_html}<br/> \
  1813. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1814. <b>${date}${raised_for_html}</b><br/> \
  1815. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1816. (source of alarm ${src}) \
  1817. "
  1818. SENT_HIPCHAT=$?
  1819. # -----------------------------------------------------------------------------
  1820. # send the Amazon SNS message
  1821. send_awssns "${to_awssns}"
  1822. SENT_AWSSNS=$?
  1823. # -----------------------------------------------------------------------------
  1824. # send the syslog message
  1825. send_syslog "${to_syslog}"
  1826. SENT_SYSLOG=$?
  1827. # -----------------------------------------------------------------------------
  1828. # send the email
  1829. IFS='' read -r -d '' email_plaintext_part <<EOF
  1830. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1831. Content-Disposition: inline
  1832. Content-Transfer-Encoding: 8bit
  1833. ${host} ${status_message}
  1834. ${alarm} ${info}
  1835. ${raised_for}
  1836. Chart : ${chart}
  1837. Family : ${family}
  1838. Severity: ${severity}
  1839. URL : ${goto_url}
  1840. Source : ${src}
  1841. Date : ${date}
  1842. Notification generated on ${host}
  1843. Evaluated Expression : ${calc_expression}
  1844. Expression Variables : ${calc_param_values}
  1845. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1846. EOF
  1847. if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
  1848. send_email <<EOF
  1849. To: ${to_email}
  1850. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1851. MIME-Version: 1.0
  1852. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1853. ${email_thread_headers}
  1854. This is a MIME-encoded multipart message
  1855. --multipart-boundary
  1856. ${email_plaintext_part}
  1857. --multipart-boundary--
  1858. EOF
  1859. else
  1860. IFS='' read -r -d '' email_html_part <<EOF
  1861. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1862. Content-Disposition: inline
  1863. Content-Transfer-Encoding: 8bit
  1864. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1865. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1866. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1867. <table>
  1868. <tbody>
  1869. <tr>
  1870. <td style="vertical-align: top;" valign="top"></td>
  1871. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1872. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1873. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1874. <tbody>
  1875. <tr>
  1876. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1877. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1878. </td>
  1879. </tr>
  1880. <tr>
  1881. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1882. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1883. </td>
  1884. </tr>
  1885. <tr>
  1886. <td style="vertical-align: top;" valign="top">
  1887. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1888. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  1889. <tbody>
  1890. <tr>
  1891. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  1892. <span>${chart}</span>
  1893. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  1894. </td>
  1895. </tr>
  1896. <tr style="margin: 0; padding: 0;">
  1897. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1898. <span><b>${alarm}</b>${info_html}</span>
  1899. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  1900. </td>
  1901. </tr>
  1902. <tr>
  1903. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1904. <span>${family}</span>
  1905. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  1906. </td>
  1907. </tr>
  1908. <tr style="margin: 0; padding: 0;">
  1909. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1910. <span>${severity}</span>
  1911. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  1912. </td>
  1913. </tr>
  1914. <tr style="margin: 0; padding: 0;">
  1915. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  1916. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  1917. </td>
  1918. </tr>
  1919. <tr style="margin: 0; padding: 0;">
  1920. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1921. <span>${calc_expression}</span>
  1922. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
  1923. </td>
  1924. </tr>
  1925. <tr style="margin: 0; padding: 0;">
  1926. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1927. <span>${calc_param_values}</span>
  1928. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
  1929. </td>
  1930. </tr>
  1931. <tr style="margin: 0; padding: 0;">
  1932. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1933. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1934. </td>
  1935. </tr>
  1936. <tr style="margin: 0; padding: 0;">
  1937. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  1938. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  1939. </td>
  1940. </tr>
  1941. <tr style="text-align: center; margin: 0; padding: 0;">
  1942. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  1943. </td>
  1944. </tr>
  1945. <tr style="text-align: center; margin: 0; padding: 0;">
  1946. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  1947. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  1948. </td>
  1949. </tr>
  1950. </tbody>
  1951. </table>
  1952. </div>
  1953. </td>
  1954. </tr>
  1955. </tbody>
  1956. </table>
  1957. </div>
  1958. </td>
  1959. </tr>
  1960. </tbody>
  1961. </table>
  1962. </body>
  1963. </html>
  1964. EOF
  1965. send_email <<EOF
  1966. To: ${to_email}
  1967. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1968. MIME-Version: 1.0
  1969. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1970. ${email_thread_headers}
  1971. This is a MIME-encoded multipart message
  1972. --multipart-boundary
  1973. ${email_plaintext_part}
  1974. --multipart-boundary
  1975. ${email_html_part}
  1976. --multipart-boundary--
  1977. EOF
  1978. fi
  1979. SENT_EMAIL=$?
  1980. # -----------------------------------------------------------------------------
  1981. # let netdata know
  1982. for state in "${SENT_EMAIL}" \
  1983. "${SENT_PUSHOVER}" \
  1984. "${SENT_TELEGRAM}" \
  1985. "${SENT_SLACK}" \
  1986. "${SENT_HANGOUTS}" \
  1987. "${SENT_ROCKETCHAT}" \
  1988. "${SENT_ALERTA}" \
  1989. "${SENT_FLOCK}" \
  1990. "${SENT_DISCORD}" \
  1991. "${SENT_TWILIO}" \
  1992. "${SENT_HIPCHAT}" \
  1993. "${SENT_MESSAGEBIRD}" \
  1994. "${SENT_KAVENEGAR}" \
  1995. "${SENT_PUSHBULLET}" \
  1996. "${SENT_KAFKA}" \
  1997. "${SENT_PD}" \
  1998. "${SENT_FLEEP}" \
  1999. "${SENT_PROWL}" \
  2000. "${SENT_CUSTOM}" \
  2001. "${SENT_IRC}" \
  2002. "${SENT_AWSSNS}" \
  2003. "${SENT_SYSLOG}" \
  2004. "${SENT_SMS}" \
  2005. "${SENT_MSTEAM}"; do
  2006. if [ "${state}" -eq 0 ]; then
  2007. # we sent something
  2008. exit 0
  2009. fi
  2010. done
  2011. # we did not send anything
  2012. exit 1