alarm-notify.sh.in 75 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200
  1. #!/usr/bin/env bash
  2. # netdata
  3. # real-time performance and health monitoring, done right!
  4. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  5. # SPDX-License-Identifier: GPL-3.0-or-later
  6. #
  7. # Script to send alarm notifications for netdata
  8. #
  9. # Features:
  10. # - multiple notification methods
  11. # - multiple roles per alarm
  12. # - multiple recipients per role
  13. # - severity filtering per recipient
  14. #
  15. # Supported notification methods:
  16. # - emails by @ktsaou
  17. # - slack.com notifications by @ktsaou
  18. # - alerta.io notifications by @kattunga
  19. # - discordapp.com notifications by @lowfive
  20. # - pushover.net notifications by @ktsaou
  21. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  22. # - telegram.org notifications by @hashworks #1002
  23. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  24. # - kafka notifications by @ktsaou #1342
  25. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  26. # - messagebird.com notifications by @tech_no_logical #1453
  27. # - hipchat notifications by @ktsaou #1561
  28. # - fleep notifications by @Ferroin
  29. # - custom notifications by @ktsaou
  30. # - syslog messages by @Ferroin
  31. # - Microsoft Team notification by @tioumen
  32. # -----------------------------------------------------------------------------
  33. # testing notifications
  34. if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
  35. then
  36. if [ "${2}" = "test" ]
  37. then
  38. recipient="${1}"
  39. else
  40. recipient="${2}"
  41. fi
  42. [ -z "${recipient}" ] && recipient="sysadmin"
  43. id=1
  44. last="CLEAR"
  45. test_res=0
  46. for x in "WARNING" "CRITICAL" "CLEAR"
  47. do
  48. echo >&2
  49. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  50. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value"
  51. if [ $? -ne 0 ]
  52. then
  53. echo >&2 "# FAILED"
  54. test_res=1
  55. else
  56. echo >&2 "# OK"
  57. fi
  58. last="${x}"
  59. id=$((id + 1))
  60. done
  61. exit $test_res
  62. fi
  63. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  64. export LC_ALL=C
  65. # -----------------------------------------------------------------------------
  66. PROGRAM_NAME="$(basename "${0}")"
  67. logdate() {
  68. date "+%Y-%m-%d %H:%M:%S"
  69. }
  70. log() {
  71. local status="${1}"
  72. shift
  73. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  74. }
  75. warning() {
  76. log WARNING "${@}"
  77. }
  78. error() {
  79. log ERROR "${@}"
  80. }
  81. info() {
  82. log INFO "${@}"
  83. }
  84. fatal() {
  85. log FATAL "${@}"
  86. exit 1
  87. }
  88. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  89. debug() {
  90. [ "${debug}" = "1" ] && log DEBUG "${@}"
  91. }
  92. docurl() {
  93. if [ -z "${curl}" ]
  94. then
  95. error "\${curl} is unset."
  96. return 1
  97. fi
  98. if [ "${debug}" = "1" ]
  99. then
  100. echo >&2 "--- BEGIN curl command ---"
  101. printf >&2 "%q " ${curl} "${@}"
  102. echo >&2
  103. echo >&2 "--- END curl command ---"
  104. local out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  105. local code=$(${curl} ${curl_options} --write-out %{http_code} --output "${out}" --silent --show-error "${@}")
  106. local ret=$?
  107. echo >&2 "--- BEGIN received response ---"
  108. cat >&2 "${out}"
  109. echo >&2
  110. echo >&2 "--- END received response ---"
  111. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  112. rm "${out}"
  113. echo "${code}"
  114. return ${ret}
  115. fi
  116. ${curl} ${curl_options} --write-out %{http_code} --output /dev/null --silent --show-error "${@}"
  117. return $?
  118. }
  119. # -----------------------------------------------------------------------------
  120. # List of all the notification mechanisms we support.
  121. # Used in a couple of places to write more compact code.
  122. method_names="
  123. email
  124. pushover
  125. pushbullet
  126. telegram
  127. slack
  128. alerta
  129. flock
  130. discord
  131. hipchat
  132. twilio
  133. messagebird
  134. pd
  135. fleep
  136. syslog
  137. custom
  138. msteam
  139. "
  140. # -----------------------------------------------------------------------------
  141. # this is to be overwritten by the config file
  142. custom_sender() {
  143. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  144. }
  145. # -----------------------------------------------------------------------------
  146. # check for BASH v4+ (required for associative arrays)
  147. [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  148. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  149. # -----------------------------------------------------------------------------
  150. # defaults to allow running this script by hand
  151. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  152. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  153. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  154. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  155. # -----------------------------------------------------------------------------
  156. # parse command line parameters
  157. if [ ${1} = "unittest" ] ; then
  158. unittest=1 # enable unit testing mode
  159. roles="${2}" # the role that should be used for unit testing
  160. cfgfile="${3}" # the location of the config file to use for unit testing
  161. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  162. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  163. else
  164. roles="${1}" # the roles that should be notified for this event
  165. args_host="${2}" # the host generated this event
  166. unique_id="${3}" # the unique id of this event
  167. alarm_id="${4}" # the unique id of the alarm that generated this event
  168. event_id="${5}" # the incremental id of the event, for this alarm id
  169. when="${6}" # the timestamp this event occurred
  170. name="${7}" # the name of the alarm, as given in netdata health.d entries
  171. chart="${8}" # the name of the chart (type.id)
  172. family="${9}" # the family of the chart
  173. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  174. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  175. value="${12}" # the current value of the alarm
  176. old_value="${13}" # the previous value of the alarm
  177. src="${14}" # the line number and file the alarm has been configured
  178. duration="${15}" # the duration in seconds of the previous alarm state
  179. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  180. units="${17}" # the units of the value
  181. info="${18}" # a short description of the alarm
  182. value_string="${19}" # friendly value (with units)
  183. old_value_string="${20}" # friendly old value (with units)
  184. fi
  185. # -----------------------------------------------------------------------------
  186. # find a suitable hostname to use, if netdata did not supply a hostname
  187. if [ -z ${args_host} ]
  188. then
  189. this_host=$(hostname -s 2>/dev/null)
  190. host="${this_host}"
  191. args_host="${this_host}"
  192. else
  193. host="${args_host}"
  194. fi
  195. # -----------------------------------------------------------------------------
  196. # screen statuses we don't need to send a notification
  197. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  198. if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  199. then
  200. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  201. exit 1
  202. fi
  203. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  204. if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  205. then
  206. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  207. exit 1
  208. fi
  209. # -----------------------------------------------------------------------------
  210. # load configuration
  211. # By default fetch images from the global public registry.
  212. # This is required by default, since all notification methods need to download
  213. # images via the Internet, and private registries might not be reachable.
  214. # This can be overwritten at the configuration file.
  215. images_base_url="https://registry.my-netdata.io"
  216. # curl options to use
  217. curl_options=""
  218. # hostname handling
  219. use_fqdn="NO"
  220. # needed commands
  221. # if empty they will be searched in the system path
  222. curl=
  223. sendmail=
  224. # enable / disable features
  225. for method_name in ${method_names^^} ; do
  226. declare SEND_${method_name}="YES"
  227. declare DEFAULT_RECIPIENT_${method_name}
  228. done
  229. for method_name in ${method_names} ; do
  230. declare -A role_recipients_${method_name}
  231. done
  232. # slack configs
  233. SLACK_WEBHOOK_URL=
  234. # Microsoft Team configs
  235. MSTEAM_WEBHOOK_URL=
  236. # rocketchat configs
  237. ROCKETCHAT_WEBHOOK_URL=
  238. # alerta configs
  239. ALERTA_WEBHOOK_URL=
  240. ALERTA_API_KEY=
  241. # flock configs
  242. FLOCK_WEBHOOK_URL=
  243. # discord configs
  244. DISCORD_WEBHOOK_URL=
  245. # pushover configs
  246. PUSHOVER_APP_TOKEN=
  247. # pushbullet configs
  248. PUSHBULLET_ACCESS_TOKEN=
  249. PUSHBULLET_SOURCE_DEVICE=
  250. # twilio configs
  251. TWILIO_ACCOUNT_SID=
  252. TWILIO_ACCOUNT_TOKEN=
  253. TWILIO_NUMBER=
  254. # hipchat configs
  255. HIPCHAT_SERVER=
  256. HIPCHAT_AUTH_TOKEN=
  257. # messagebird configs
  258. MESSAGEBIRD_ACCESS_KEY=
  259. MESSAGEBIRD_NUMBER=
  260. # kavenegar configs
  261. KAVENEGAR_API_KEY=""
  262. KAVENEGAR_SENDER=""
  263. # telegram configs
  264. TELEGRAM_BOT_TOKEN=
  265. # kafka configs
  266. SEND_KAFKA="YES"
  267. KAFKA_URL=
  268. KAFKA_SENDER_IP=
  269. # pagerduty.com configs
  270. PD_SERVICE_KEY=
  271. # fleep.io configs
  272. FLEEP_SENDER="${host}"
  273. # Amazon SNS configs
  274. DEFAULT_RECIPIENT_AWSSNS=
  275. AWSSNS_MESSAGE_FORMAT=
  276. declare -A role_recipients_awssns=()
  277. # syslog configs
  278. SYSLOG_FACILITY=
  279. # email configs
  280. EMAIL_SENDER=
  281. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  282. EMAIL_THREADING=
  283. DEFAULT_RECIPIENT_EMAIL="root"
  284. # irc configs
  285. IRC_NICKNAME=
  286. IRC_REALNAME=
  287. IRC_NETWORK=
  288. # load the stock and user configuration files
  289. # these will overwrite the variables above
  290. if [ ${unittest} ] ;
  291. then
  292. source "${cfgfile}"
  293. [ $? -ne 0 ] && error "Failed to load requested config file." && exit 1
  294. else
  295. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"
  296. do
  297. if [ -f "${CONFIG}" ]
  298. then
  299. debug "Loading config file '${CONFIG}'..."
  300. source "${CONFIG}"
  301. [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
  302. else
  303. warning "Cannot find file '${CONFIG}'."
  304. fi
  305. done
  306. fi
  307. # If we didn't autodetect the character set for e-mail and it wasn't
  308. # set by the user, we need to set it to a reasonable default. UTF-8
  309. # should be correct for almost all modern UNIX systems.
  310. if [ -z ${EMAIL_CHARSET} ]
  311. then
  312. EMAIL_CHARSET="UTF-8"
  313. fi
  314. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  315. # unless we're sending an alarm for a slave system which we can't get the
  316. # FQDN of easily.
  317. if [ "${use_fqdn}" = "YES" -a "${host}" = "$(hostname -s 2>/dev/null)" ]
  318. then
  319. host="$(hostname -f 2>/dev/null)"
  320. fi
  321. # -----------------------------------------------------------------------------
  322. # filter a recipient based on alarm event severity
  323. filter_recipient_by_criticality() {
  324. local method="${1}" x="${2}" r s
  325. shift
  326. r="${x/|*/}" # the recipient
  327. s="${x/*|/}" # the severity required for notifying this recipient
  328. # no severity filtering for this person
  329. [ "${r}" = "${s}" ] && return 0
  330. # the severity is invalid
  331. s="${s^^}"
  332. if [ "${s}" != "CRITICAL" ]
  333. then
  334. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  335. return 0
  336. fi
  337. # create the status tracking directory for this user
  338. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
  339. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  340. case "${status}" in
  341. CRITICAL)
  342. # make sure he will get future notifications for this alarm too
  343. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  344. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  345. return 0
  346. ;;
  347. WARNING)
  348. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
  349. then
  350. # we do not remove the file, so that he will get future notifications of this alarm
  351. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  352. return 0
  353. fi
  354. ;;
  355. *)
  356. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
  357. then
  358. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  359. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  360. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  361. return 0
  362. fi
  363. ;;
  364. esac
  365. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  366. return 1
  367. }
  368. # -----------------------------------------------------------------------------
  369. # verify the delivery methods supported
  370. # check slack
  371. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  372. # check rocketchat
  373. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  374. # check alerta
  375. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  376. # check flock
  377. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  378. # check discord
  379. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  380. # check pushover
  381. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  382. # check pushbullet
  383. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  384. # check twilio
  385. [ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
  386. # check hipchat
  387. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  388. # check messagebird
  389. [ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
  390. # check kavenegar
  391. [ -z "${KAVENEGAR_API_KEY}" -o -z "${KAVENEGAR_SENDER}" ] && SEND_KAVENEGAR="NO"
  392. # check telegram
  393. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  394. # check kafka
  395. [ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
  396. # check irc
  397. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  398. # check fleep
  399. [ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO"
  400. # if we need curl, check for the curl command
  401. if [ \( \
  402. "${SEND_PUSHOVER}" = "YES" \
  403. -o "${SEND_SLACK}" = "YES" \
  404. -o "${SEND_ROCKETCHAT}" = "YES" \
  405. -o "${SEND_ALERTA}" = "YES" \
  406. -o "${SEND_PD}" = "YES" \
  407. -o "${SEND_FLOCK}" = "YES" \
  408. -o "${SEND_DISCORD}" = "YES" \
  409. -o "${SEND_HIPCHAT}" = "YES" \
  410. -o "${SEND_TWILIO}" = "YES" \
  411. -o "${SEND_MESSAGEBIRD}" = "YES" \
  412. -o "${SEND_KAVENEGAR}" = "YES" \
  413. -o "${SEND_TELEGRAM}" = "YES" \
  414. -o "${SEND_PUSHBULLET}" = "YES" \
  415. -o "${SEND_KAFKA}" = "YES" \
  416. -o "${SEND_FLEEP}" = "YES" \
  417. -o "${SEND_CUSTOM}" = "YES" \
  418. -o "${SEND_MSTEAM}" = "YES" \
  419. \) -a -z "${curl}" ]
  420. then
  421. curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
  422. if [ -z "${curl}" ]
  423. then
  424. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  425. SEND_PUSHOVER="NO"
  426. SEND_PUSHBULLET="NO"
  427. SEND_TELEGRAM="NO"
  428. SEND_SLACK="NO"
  429. SEND_MSTEAM="NO"
  430. SEND_ROCKETCHAT="NO"
  431. SEND_ALERTA="NO"
  432. SEND_PD="NO"
  433. SEND_FLOCK="NO"
  434. SEND_DISCORD="NO"
  435. SEND_TWILIO="NO"
  436. SEND_HIPCHAT="NO"
  437. SEND_MESSAGEBIRD="NO"
  438. SEND_KAVENEGAR="NO"
  439. SEND_KAFKA="NO"
  440. SEND_FLEEP="NO"
  441. SEND_CUSTOM="NO"
  442. fi
  443. fi
  444. # if we need sendmail, check for the sendmail command
  445. if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
  446. then
  447. sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
  448. if [ -z "${sendmail}" ]
  449. then
  450. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  451. SEND_EMAIL="NO"
  452. fi
  453. fi
  454. # if we need logger, check for the logger command
  455. if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
  456. then
  457. logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)"
  458. if [ -z "${logger}" ]
  459. then
  460. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  461. SEND_SYSLOG="NO"
  462. fi
  463. fi
  464. # if we need aws, check for the aws command
  465. if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ]
  466. then
  467. aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)"
  468. if [ -z "${aws}" ]
  469. then
  470. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  471. SEND_AWSSNS="NO"
  472. fi
  473. fi
  474. # -----------------------------------------------------------------------------
  475. # find the recipients' addresses per method
  476. # netdata may call us with multiple roles, and roles may have multiple but
  477. # overlapping recipients - so, here we find the unique recipients.
  478. for method_name in ${method_names} ; do
  479. send_var="SEND_${method_name^^}"
  480. if [ ${!send_var} = "NO" ] ; then
  481. continue
  482. fi
  483. declare -A arr_var=()
  484. for x in ${roles//,/ } ; do
  485. # the roles 'silent' and 'disabled' mean:
  486. # don't send a notification for this role
  487. [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
  488. role_recipients="role_recipients_${method_name}[$x]"
  489. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  490. a="${!role_recipients}"
  491. [ -z "${a}" ] && a="${!default_recipient_var}"
  492. for r in ${a//,/ } ; do
  493. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  494. done
  495. done
  496. # build the list of recipients
  497. to_var="to_${method_name}"
  498. declare to_${method_name}="${!arr_var[*]}"
  499. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  500. done
  501. # -----------------------------------------------------------------------------
  502. # handle fixup of the email recipient list.
  503. fix_to_email() {
  504. to_email=
  505. while [ ! -z "${1}" ]
  506. do
  507. [ ! -z "${to_email}" ] && to_email="${to_email}, "
  508. to_email="${to_email}${1}"
  509. shift 1
  510. done
  511. }
  512. # ${to_email} without quotes here
  513. fix_to_email ${to_email}
  514. # -----------------------------------------------------------------------------
  515. # handle output if we're running in unit test mode
  516. if [ ${unittest} ] ; then
  517. for method_name in ${method_names} ; do
  518. to_var="to_${method_name}"
  519. echo "results: ${method_name}: ${!to_var}"
  520. done
  521. exit 0
  522. fi
  523. # -----------------------------------------------------------------------------
  524. # check that we have at least a method enabled
  525. if [ "${SEND_EMAIL}" != "YES" \
  526. -a "${SEND_PUSHOVER}" != "YES" \
  527. -a "${SEND_TELEGRAM}" != "YES" \
  528. -a "${SEND_SLACK}" != "YES" \
  529. -a "${SEND_ROCKETCHAT}" != "YES" \
  530. -a "${SEND_ALERTA}" != "YES" \
  531. -a "${SEND_FLOCK}" != "YES" \
  532. -a "${SEND_DISCORD}" != "YES" \
  533. -a "${SEND_TWILIO}" != "YES" \
  534. -a "${SEND_HIPCHAT}" != "YES" \
  535. -a "${SEND_MESSAGEBIRD}" != "YES" \
  536. -a "${SEND_KAVENEGAR}" != "YES" \
  537. -a "${SEND_PUSHBULLET}" != "YES" \
  538. -a "${SEND_KAFKA}" != "YES" \
  539. -a "${SEND_PD}" != "YES" \
  540. -a "${SEND_FLEEP}" != "YES" \
  541. -a "${SEND_CUSTOM}" != "YES" \
  542. -a "${SEND_IRC}" != "YES" \
  543. -a "${SEND_AWSSNS}" != "YES" \
  544. -a "${SEND_SYSLOG}" != "YES" \
  545. -a "${SEND_MSTEAM}" != "YES" \
  546. ]
  547. then
  548. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  549. fi
  550. # -----------------------------------------------------------------------------
  551. # get the date the alarm happened
  552. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  553. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  554. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  555. [ -z "${date}" ] && date=$(date 2>/dev/null)
  556. # ----------------------------------------------------------------------------
  557. # prepare some extra headers if we've been asked to thread e-mails
  558. if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" != "NO" ] ; then
  559. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
  560. else
  561. email_thread_headers=
  562. fi
  563. # -----------------------------------------------------------------------------
  564. # function to URL encode a string
  565. urlencode() {
  566. local string="${1}" strlen encoded pos c o
  567. strlen=${#string}
  568. for (( pos=0 ; pos<strlen ; pos++ ))
  569. do
  570. c=${string:${pos}:1}
  571. case "${c}" in
  572. [-_.~a-zA-Z0-9])
  573. o="${c}"
  574. ;;
  575. *)
  576. printf -v o '%%%02x' "'${c}"
  577. ;;
  578. esac
  579. encoded+="${o}"
  580. done
  581. REPLY="${encoded}"
  582. echo "${REPLY}"
  583. }
  584. # -----------------------------------------------------------------------------
  585. # function to convert a duration in seconds, to a human readable duration
  586. # using DAYS, MINUTES, SECONDS
  587. duration4human() {
  588. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  589. d=$(( s / 86400 ))
  590. s=$(( s - (d * 86400) ))
  591. h=$(( s / 3600 ))
  592. s=$(( s - (h * 3600) ))
  593. m=$(( s / 60 ))
  594. s=$(( s - (m * 60) ))
  595. if [ ${d} -gt 0 ]
  596. then
  597. [ ${m} -ge 30 ] && h=$(( h + 1 ))
  598. [ ${d} -gt 1 ] && ds="days"
  599. [ ${h} -gt 1 ] && hs="hours"
  600. if [ ${h} -gt 0 ]
  601. then
  602. ret="${d} ${ds} and ${h} ${hs}"
  603. else
  604. ret="${d} ${ds}"
  605. fi
  606. elif [ ${h} -gt 0 ]
  607. then
  608. [ ${s} -ge 30 ] && m=$(( m + 1 ))
  609. [ ${h} -gt 1 ] && hs="hours"
  610. [ ${m} -gt 1 ] && ms="minutes"
  611. if [ ${m} -gt 0 ]
  612. then
  613. ret="${h} ${hs} and ${m} ${ms}"
  614. else
  615. ret="${h} ${hs}"
  616. fi
  617. elif [ ${m} -gt 0 ]
  618. then
  619. [ ${m} -gt 1 ] && ms="minutes"
  620. [ ${s} -gt 1 ] && ss="seconds"
  621. if [ ${s} -gt 0 ]
  622. then
  623. ret="${m} ${ms} and ${s} ${ss}"
  624. else
  625. ret="${m} ${ms}"
  626. fi
  627. else
  628. [ ${s} -gt 1 ] && ss="seconds"
  629. ret="${s} ${ss}"
  630. fi
  631. REPLY="${ret}"
  632. echo "${REPLY}"
  633. }
  634. # -----------------------------------------------------------------------------
  635. # email sender
  636. send_email() {
  637. local ret= opts=() sender_email="${EMAIL_SENDER}" sender_name=
  638. if [ "${SEND_EMAIL}" = "YES" ]
  639. then
  640. if [ ! -z "${EMAIL_SENDER}" ]
  641. then
  642. if [[ "${EMAIL_SENDER}" =~ ^\".*\"\ \<.*\>$ ]]
  643. then
  644. # the name includes double quotes
  645. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  646. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  647. elif [[ "${EMAIL_SENDER}" =~ ^\'.*\'\ \<.*\>$ ]]
  648. then
  649. # the name includes single quotes
  650. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  651. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  652. elif [[ "${EMAIL_SENDER}" =~ ^.*\ \<.*\>$ ]]
  653. then
  654. # the name does not have any quotes
  655. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  656. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  657. fi
  658. fi
  659. [ ! -z "${sender_email}" ] && opts+=(-f "${sender_email}")
  660. [ ! -z "${sender_name}" ] && opts+=(-F "${sender_name}")
  661. if [ "${debug}" = "1" ]
  662. then
  663. echo >&2 "--- BEGIN sendmail command ---"
  664. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  665. echo >&2
  666. echo >&2 "--- END sendmail command ---"
  667. fi
  668. "${sendmail}" -t "${opts[@]}"
  669. ret=$?
  670. if [ ${ret} -eq 0 ]
  671. then
  672. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  673. return 0
  674. else
  675. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  676. return 1
  677. fi
  678. fi
  679. return 1
  680. }
  681. # -----------------------------------------------------------------------------
  682. # pushover sender
  683. send_pushover() {
  684. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  685. if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
  686. then
  687. # https://pushover.net/api
  688. priority=-2
  689. case "${status}" in
  690. CLEAR) priority=-1;; # low priority: no sound or vibration
  691. WARNING) priority=0;; # normal priority: respect quiet hours
  692. CRITICAL) priority=1;; # high priority: bypass quiet hours
  693. *) priority=-2;; # lowest priority: no notification at all
  694. esac
  695. for user in ${usertokens}
  696. do
  697. httpcode=$(docurl \
  698. --form-string "token=${apptoken}" \
  699. --form-string "user=${user}" \
  700. --form-string "html=1" \
  701. --form-string "title=${title}" \
  702. --form-string "message=${message}" \
  703. --form-string "timestamp=${when}" \
  704. --form-string "url=${url}" \
  705. --form-string "url_title=Open netdata dashboard to view the alarm" \
  706. --form-string "priority=${priority}" \
  707. https://api.pushover.net/1/messages.json)
  708. if [ "${httpcode}" = "200" ]
  709. then
  710. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  711. sent=$((sent + 1))
  712. else
  713. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  714. fi
  715. done
  716. [ ${sent} -gt 0 ] && return 0
  717. fi
  718. return 1
  719. }
  720. # -----------------------------------------------------------------------------
  721. # pushbullet sender
  722. send_pushbullet() {
  723. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  724. if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  725. then
  726. #https://docs.pushbullet.com/#create-push
  727. for user in ${recipients}
  728. do
  729. httpcode=$(docurl \
  730. --header 'Access-Token: '${userapikey}'' \
  731. --header 'Content-Type: application/json' \
  732. --data-binary @<(cat <<EOF
  733. {"title": "${title}",
  734. "type": "link",
  735. "email": "${user}",
  736. "body": "$( echo -n ${message})",
  737. "url": "${url}",
  738. "source_device_iden": "${source_device}"}
  739. EOF
  740. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  741. if [ "${httpcode}" = "200" ]
  742. then
  743. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  744. sent=$((sent + 1))
  745. else
  746. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  747. fi
  748. done
  749. [ ${sent} -gt 0 ] && return 0
  750. fi
  751. return 1
  752. }
  753. # -----------------------------------------------------------------------------
  754. # kafka sender
  755. send_kafka() {
  756. local httpcode sent=0
  757. if [ "${SEND_KAFKA}" = "YES" ]
  758. then
  759. httpcode=$(docurl -X POST \
  760. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  761. "${KAFKA_URL}")
  762. if [ "${httpcode}" = "204" ]
  763. then
  764. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  765. sent=$((sent + 1))
  766. else
  767. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  768. fi
  769. [ ${sent} -gt 0 ] && return 0
  770. fi
  771. return 1
  772. }
  773. # -----------------------------------------------------------------------------
  774. # pagerduty.com sender
  775. send_pd() {
  776. local recipients="${1}" sent=0
  777. unset t
  778. case ${status} in
  779. CLEAR) t='resolve';;
  780. WARNING) t='trigger';;
  781. CRITICAL) t='trigger';;
  782. esac
  783. if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
  784. then
  785. for PD_SERVICE_KEY in ${recipients}
  786. do
  787. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  788. payload="$(cat << EOF
  789. {
  790. "service_key": "${PD_SERVICE_KEY}",
  791. "event_type": "${t}",
  792. "incident_key" : "${alarm_id}",
  793. "description": "${d}",
  794. "details": {
  795. "value_w_units": "${value_string}",
  796. "when": "${when}",
  797. "duration" : "${duration}",
  798. "roles": "${roles}",
  799. "alarm_id" : "${alarm_id}",
  800. "name" : "${name}",
  801. "chart" : "${chart}",
  802. "family" : "${family}",
  803. "status" : "${status}",
  804. "old_status" : "${old_status}",
  805. "value" : "${value}",
  806. "old_value" : "${old_value}",
  807. "src" : "${src}",
  808. "non_clear_duration" : "${non_clear_duration}",
  809. "units" : "${units}",
  810. "info" : "${info}"
  811. }
  812. }
  813. EOF
  814. )"
  815. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  816. if [ "${httpcode}" = "200" ]
  817. then
  818. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  819. sent=$((sent + 1))
  820. else
  821. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  822. fi
  823. done
  824. [ ${sent} -gt 0 ] && return 0
  825. fi
  826. return 1
  827. }
  828. # -----------------------------------------------------------------------------
  829. # twilio sender
  830. send_twilio() {
  831. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  832. if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  833. then
  834. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  835. for user in ${recipients}
  836. do
  837. httpcode=$(docurl -X POST \
  838. --data-urlencode "From=${twilionumber}" \
  839. --data-urlencode "To=${user}" \
  840. --data-urlencode "Body=${title} ${message}" \
  841. -u "${accountsid}:${accounttoken}" \
  842. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  843. if [ "${httpcode}" = "201" ]
  844. then
  845. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  846. sent=$((sent + 1))
  847. else
  848. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  849. fi
  850. done
  851. [ ${sent} -gt 0 ] && return 0
  852. fi
  853. return 1
  854. }
  855. # -----------------------------------------------------------------------------
  856. # hipchat sender
  857. send_hipchat() {
  858. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
  859. # remove <small></small> from the message
  860. message="${message//<small>/}"
  861. message="${message//<\/small>/}"
  862. if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
  863. then
  864. # A label to be shown in addition to the sender's name
  865. # Valid length range: 0 - 64.
  866. sender="netdata"
  867. # Valid values: html, text.
  868. # Defaults to 'html'.
  869. msg_format="html"
  870. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  871. case "${status}" in
  872. WARNING) color="yellow" ;;
  873. CRITICAL) color="red" ;;
  874. CLEAR) color="green" ;;
  875. *) color="gray" ;;
  876. esac
  877. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  878. # Each recipient's notification preferences are taken into account.
  879. # Defaults to false.
  880. notify="true"
  881. for room in ${recipients}
  882. do
  883. httpcode=$(docurl -X POST \
  884. -H "Content-type: application/json" \
  885. -H "Authorization: Bearer ${authtoken}" \
  886. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  887. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  888. if [ "${httpcode}" = "204" ]
  889. then
  890. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  891. sent=$((sent + 1))
  892. else
  893. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  894. fi
  895. done
  896. [ ${sent} -gt 0 ] && return 0
  897. fi
  898. return 1
  899. }
  900. # -----------------------------------------------------------------------------
  901. # messagebird sender
  902. send_messagebird() {
  903. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  904. if [ "${SEND_MESSAGEBIRD}" = "YES" -a ! -z "${accesskey}" -a ! -z "${messagebirdnumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  905. then
  906. #https://developers.messagebird.com/docs/messaging
  907. for user in ${recipients}
  908. do
  909. httpcode=$(docurl -X POST \
  910. --data-urlencode "originator=${messagebirdnumber}" \
  911. --data-urlencode "recipients=${user}" \
  912. --data-urlencode "body=${title} ${message}" \
  913. --data-urlencode "datacoding=auto" \
  914. -H "Authorization: AccessKey ${accesskey}" \
  915. "https://rest.messagebird.com/messages")
  916. if [ "${httpcode}" = "201" ]
  917. then
  918. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  919. sent=$((sent + 1))
  920. else
  921. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  922. fi
  923. done
  924. [ ${sent} -gt 0 ] && return 0
  925. fi
  926. return 1
  927. }
  928. # -----------------------------------------------------------------------------
  929. # kavenegar sender
  930. send_kavenegar() {
  931. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  932. if [ "${SEND_KAVENEGAR}" = "YES" -a ! -z "${API_KEY}" -a ! -z "${kavenegarsender}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  933. then
  934. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  935. for user in ${recipients}
  936. do
  937. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  938. --data-urlencode "sender=${kavenegarsender}" \
  939. --data-urlencode "receptor=${user}" \
  940. --data-urlencode "message=${title} ${message}")
  941. if [ "${httpcode}" = "201" ]
  942. then
  943. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  944. sent=$((sent + 1))
  945. else
  946. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  947. fi
  948. done
  949. [ ${sent} -gt 0 ] && return 0
  950. fi
  951. return 1
  952. }
  953. # -----------------------------------------------------------------------------
  954. # telegram sender
  955. send_telegram() {
  956. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  957. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  958. case "${status}" in
  959. WARNING) emoji="⚠️" ;;
  960. CRITICAL) emoji="🔴" ;;
  961. CLEAR) emoji="✅" ;;
  962. *) emoji="⚪️" ;;
  963. esac
  964. if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
  965. then
  966. for chatid in ${chatids}
  967. do
  968. # https://core.telegram.org/bots/api#sendmessage
  969. httpcode=$(docurl ${disableNotification} \
  970. --data-urlencode "parse_mode=HTML" \
  971. --data-urlencode "disable_web_page_preview=true" \
  972. --data-urlencode "text=${emoji} ${message}" \
  973. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  974. if [ "${httpcode}" = "200" ]
  975. then
  976. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  977. sent=$((sent + 1))
  978. elif [ "${httpcode}" = "401" ]
  979. then
  980. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  981. else
  982. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  983. fi
  984. done
  985. [ ${sent} -gt 0 ] && return 0
  986. fi
  987. return 1
  988. }
  989. # -----------------------------------------------------------------------------
  990. # Microsoft Team sender
  991. send_msteam() {
  992. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  993. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  994. case "${status}" in
  995. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}";;
  996. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}";;
  997. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}";;
  998. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}";;
  999. esac
  1000. for channel in ${channels}
  1001. do
  1002. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  1003. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  1004. payload="$(cat <<EOF
  1005. {
  1006. "@context": "http://schema.org/extensions",
  1007. "@type": "MessageCard",
  1008. "themeColor": "${color}",
  1009. "title": "$icon Alert ${status} from netdata for ${host}",
  1010. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  1011. "potentialAction": [
  1012. {
  1013. "@type": "OpenUri",
  1014. "name": "Netdata",
  1015. "targets": [
  1016. { "os": "default", "uri": "${goto_url}" }
  1017. ]
  1018. }
  1019. ]
  1020. }
  1021. EOF
  1022. )"
  1023. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1024. webhook="${webhook//CHANNEL/${channel}}"
  1025. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  1026. if [ "${httpcode}" = "200" ]
  1027. then
  1028. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  1029. sent=$((sent + 1))
  1030. else
  1031. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  1032. fi
  1033. done
  1034. [ ${sent} -gt 0 ] && return 0
  1035. return 1
  1036. }
  1037. # slack sender
  1038. send_slack() {
  1039. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1040. [ "${SEND_SLACK}" != "YES" ] && return 1
  1041. case "${status}" in
  1042. WARNING) color="warning" ;;
  1043. CRITICAL) color="danger" ;;
  1044. CLEAR) color="good" ;;
  1045. *) color="#777777" ;;
  1046. esac
  1047. for channel in ${channels}
  1048. do
  1049. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1050. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ] ; then channel="#$channel"; fi
  1051. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1052. if [ "${channel}" = "#" ] ; then
  1053. ch=""
  1054. chstr="without specifying a channel"
  1055. else
  1056. ch="\"channel\": \"${channel}\","
  1057. chstr="to '${channel}'"
  1058. fi
  1059. payload="$(cat <<EOF
  1060. {
  1061. $ch
  1062. "username": "netdata on ${host}",
  1063. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1064. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1065. "attachments": [
  1066. {
  1067. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1068. "color": "${color}",
  1069. "title": "${alarm}",
  1070. "title_link": "${goto_url}",
  1071. "text": "${info}",
  1072. "fields": [
  1073. {
  1074. "title": "${chart}",
  1075. "short": true
  1076. },
  1077. {
  1078. "title": "${family}",
  1079. "short": true
  1080. }
  1081. ],
  1082. "thumb_url": "${image}",
  1083. "footer": "by <${goto_url}|${host}>",
  1084. "ts": ${when}
  1085. }
  1086. ]
  1087. }
  1088. EOF
  1089. )"
  1090. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1091. if [ "${httpcode}" = "200" ]
  1092. then
  1093. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1094. sent=$((sent + 1))
  1095. else
  1096. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1097. fi
  1098. done
  1099. [ ${sent} -gt 0 ] && return 0
  1100. return 1
  1101. }
  1102. # -----------------------------------------------------------------------------
  1103. # rocketchat sender
  1104. send_rocketchat() {
  1105. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1106. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1107. case "${status}" in
  1108. WARNING) color="warning" ;;
  1109. CRITICAL) color="danger" ;;
  1110. CLEAR) color="good" ;;
  1111. *) color="#777777" ;;
  1112. esac
  1113. for channel in ${channels}
  1114. do
  1115. payload="$(cat <<EOF
  1116. {
  1117. "channel": "#${channel}",
  1118. "alias": "netdata on ${host}",
  1119. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1120. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1121. "attachments": [
  1122. {
  1123. "color": "${color}",
  1124. "title": "${alarm}",
  1125. "title_link": "${goto_url}",
  1126. "text": "${info}",
  1127. "fields": [
  1128. {
  1129. "title": "${chart}",
  1130. "short": true,
  1131. "value": "chart"
  1132. },
  1133. {
  1134. "title": "${family}",
  1135. "short": true,
  1136. "value": "family"
  1137. }
  1138. ],
  1139. "thumb_url": "${image}",
  1140. "ts": "${when}"
  1141. }
  1142. ]
  1143. }
  1144. EOF
  1145. )"
  1146. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1147. if [ "${httpcode}" = "200" ]
  1148. then
  1149. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1150. sent=$((sent + 1))
  1151. else
  1152. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1153. fi
  1154. done
  1155. [ ${sent} -gt 0 ] && return 0
  1156. return 1
  1157. }
  1158. # -----------------------------------------------------------------------------
  1159. # alerta sender
  1160. send_alerta() {
  1161. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1162. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1163. case "${status}" in
  1164. CRITICAL) severity="critical" ;;
  1165. WARNING) severity="warning" ;;
  1166. CLEAR) severity="cleared" ;;
  1167. *) severity="indeterminate" ;;
  1168. esac
  1169. if [[ "${chart}" == httpcheck* ]]
  1170. then
  1171. resource=$chart
  1172. event=$name
  1173. else
  1174. resource="${host}:${family}"
  1175. event="${chart}.${name}"
  1176. fi
  1177. for channel in ${channels}
  1178. do
  1179. payload="$(cat <<EOF
  1180. {
  1181. "resource": "${resource}",
  1182. "event": "${event}",
  1183. "environment": "${channel}",
  1184. "severity": "${severity}",
  1185. "service": ["Netdata"],
  1186. "group": "Performance",
  1187. "value": "${value_string}",
  1188. "text": "${info}",
  1189. "tags": ["alarm_id:${alarm_id}"],
  1190. "attributes": {
  1191. "roles": "${roles}",
  1192. "name": "${name}",
  1193. "chart": "${chart}",
  1194. "family": "${family}",
  1195. "source": "${src}",
  1196. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1197. },
  1198. "origin": "netdata/${host}",
  1199. "type": "netdataAlarm",
  1200. "rawData": "${BASH_ARGV[@]}"
  1201. }
  1202. EOF
  1203. )"
  1204. if [[ -n "${ALERTA_API_KEY}" ]]
  1205. then
  1206. auth="Key ${ALERTA_API_KEY}"
  1207. fi
  1208. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1209. if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
  1210. then
  1211. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1212. sent=$((sent + 1))
  1213. elif [[ "${httpcode}" = "202" ]]
  1214. then
  1215. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1216. else
  1217. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1218. fi
  1219. done
  1220. [ ${sent} -gt 0 ] && return 0
  1221. return 1
  1222. }
  1223. # -----------------------------------------------------------------------------
  1224. # flock sender
  1225. send_flock() {
  1226. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1227. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1228. case "${status}" in
  1229. WARNING) color="warning" ;;
  1230. CRITICAL) color="danger" ;;
  1231. CLEAR) color="good" ;;
  1232. *) color="#777777" ;;
  1233. esac
  1234. for channel in ${channels}
  1235. do
  1236. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1237. \"sendAs\": {
  1238. \"name\" : \"netdata on ${host}\",
  1239. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1240. },
  1241. \"text\": \"${host} *${status_message}*\",
  1242. \"timestamp\": \"${when}\",
  1243. \"attachments\": [
  1244. {
  1245. \"description\": \"${chart} (${family}) - ${info}\",
  1246. \"color\": \"${color}\",
  1247. \"title\": \"${alarm}\",
  1248. \"url\": \"${goto_url}\",
  1249. \"text\": \"${info}\",
  1250. \"views\": {
  1251. \"image\": {
  1252. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1253. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1254. \"filename\": \"${image}\"
  1255. }
  1256. }
  1257. }
  1258. ]
  1259. }" )
  1260. if [ "${httpcode}" = "200" ]
  1261. then
  1262. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1263. sent=$((sent + 1))
  1264. else
  1265. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1266. fi
  1267. done
  1268. [ ${sent} -gt 0 ] && return 0
  1269. return 1
  1270. }
  1271. # -----------------------------------------------------------------------------
  1272. # discord sender
  1273. send_discord() {
  1274. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1275. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1276. case "${status}" in
  1277. WARNING) color="warning" ;;
  1278. CRITICAL) color="danger" ;;
  1279. CLEAR) color="good" ;;
  1280. *) color="#777777" ;;
  1281. esac
  1282. for channel in ${channels}
  1283. do
  1284. username="netdata on ${host}"
  1285. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1286. payload="$(cat <<EOF
  1287. {
  1288. "channel": "#${channel}",
  1289. "username": "${username}",
  1290. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1291. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1292. "attachments": [
  1293. {
  1294. "color": "${color}",
  1295. "title": "${alarm}",
  1296. "title_link": "${goto_url}",
  1297. "text": "${info}",
  1298. "fields": [
  1299. {
  1300. "title": "${chart}",
  1301. "value": "${family}"
  1302. }
  1303. ],
  1304. "thumb_url": "${image}",
  1305. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1306. "footer": "${host}",
  1307. "ts": ${when}
  1308. }
  1309. ]
  1310. }
  1311. EOF
  1312. )"
  1313. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1314. if [ "${httpcode}" = "200" ]
  1315. then
  1316. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1317. sent=$((sent + 1))
  1318. else
  1319. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1320. fi
  1321. done
  1322. [ ${sent} -gt 0 ] && return 0
  1323. return 1
  1324. }
  1325. # -----------------------------------------------------------------------------
  1326. # fleep sender
  1327. send_fleep() {
  1328. local httpcode sent=0 webhooks="${1}" data message
  1329. if [ "${SEND_FLEEP}" = "YES" ] ; then
  1330. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1331. for hook in "${webhooks}" ; do
  1332. data="{ "
  1333. data="${data} 'message': '${message}', "
  1334. data="${data} 'user': '${FLEEP_SENDER}' "
  1335. data="${data} }"
  1336. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1337. if [ "${httpcode}" = "200" ] ; then
  1338. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1339. sent=$((sent + 1))
  1340. else
  1341. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1342. fi
  1343. done
  1344. [ ${sent} -gt 0 ] && return 0
  1345. fi
  1346. return 1
  1347. }
  1348. # -----------------------------------------------------------------------------
  1349. # irc sender
  1350. send_irc() {
  1351. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1352. if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ]
  1353. then
  1354. case "${status}" in
  1355. WARNING) color="warning" ;;
  1356. CRITICAL) color="danger" ;;
  1357. CLEAR) color="good" ;;
  1358. *) color="#777777" ;;
  1359. esac
  1360. for CHANNEL in ${CHANNELS}
  1361. do
  1362. error=0
  1363. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\nNICK ${NICKNAME}\nJOIN ${CHANNEL}\nPRIVMSG ${CHANNEL} :${MESSAGE}\nQUIT\n" \ | nc ${NETWORK} 6667)
  1364. reply_codes=$(echo ${send_alarm} | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1365. for code in ${reply_codes}
  1366. do
  1367. [ "${code}" -ge 400 -a "${code}" -le 599 ] && error=1 && break
  1368. done
  1369. if [ "${error}" -eq 0 ]
  1370. then
  1371. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1372. sent=$((sent + 1))
  1373. else
  1374. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1375. fi
  1376. done
  1377. fi
  1378. [ ${sent} -gt 0 ] && return 0
  1379. return 1
  1380. }
  1381. # -----------------------------------------------------------------------------
  1382. # Amazon SNS sender
  1383. send_awssns() {
  1384. local targets="${1}" message='' sent=0 region=''
  1385. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1386. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1387. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1388. for target in ${targets} ; do
  1389. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1390. region="$(echo ${target} | cut -f 4 -d ':')"
  1391. ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null
  1392. if [ $? = 0 ]; then
  1393. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1394. sent=$((sent + 1))
  1395. else
  1396. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1397. fi
  1398. done
  1399. [ ${sent} -gt 0 ] && return 0
  1400. return 1
  1401. }
  1402. # -----------------------------------------------------------------------------
  1403. # syslog sender
  1404. send_syslog() {
  1405. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1406. local priority='' message='' host='' port='' prefix=''
  1407. local temp1='' temp2=''
  1408. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1409. if [ "${status}" = "CRITICAL" ] ; then
  1410. level='crit'
  1411. elif [ "${status}" = "WARNING" ] ; then
  1412. level='warning'
  1413. fi
  1414. for target in ${targets} ; do
  1415. priority="${facility}.${level}"
  1416. message=''
  1417. host=''
  1418. port=''
  1419. prefix=''
  1420. temp1=''
  1421. temp2=''
  1422. prefix=$(echo ${target} | cut -d '/' -f 2)
  1423. temp1=$(echo ${target} | cut -d '/' -f 1)
  1424. if [ ${prefix} != ${temp1} ] ; then
  1425. if (echo ${temp1} | grep -q '@' ) ; then
  1426. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1427. host=$(echo ${temp1} | cut -d '@' -f 2)
  1428. if [ ${temp2} != ${host} ] ; then
  1429. priority=${temp2}
  1430. fi
  1431. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1432. if ( echo ${host} | grep -E -q '\[.*\]' ) ; then
  1433. if ( echo ${port} | grep -q ']' ) ; then
  1434. port=''
  1435. else
  1436. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1437. fi
  1438. else
  1439. if [ ${port} = ${host} ] ; then
  1440. port=''
  1441. else
  1442. host=$(echo ${host} | cut -d ':' -f 1)
  1443. fi
  1444. fi
  1445. else
  1446. priority=${temp1}
  1447. fi
  1448. fi
  1449. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1450. if [ ${host} ] ; then
  1451. logger_options="${logger_options} -n ${host}"
  1452. if [ ${port} ] ; then
  1453. logger_options="${logger_options} -P ${port}"
  1454. fi
  1455. fi
  1456. ${logger} -p ${priority} ${logger_options} "${message}"
  1457. done
  1458. return $?
  1459. }
  1460. # -----------------------------------------------------------------------------
  1461. # prepare the content of the notification
  1462. # the url to send the user on click
  1463. urlencode "${args_host}" >/dev/null; url_host="${REPLY}"
  1464. urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
  1465. urlencode "${family}" >/dev/null; url_family="${REPLY}"
  1466. urlencode "${name}" >/dev/null; url_name="${REPLY}"
  1467. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
  1468. # the severity of the alarm
  1469. severity="${status}"
  1470. # the time the alarm was raised
  1471. duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
  1472. duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
  1473. raised_for="(was ${old_status,,} for ${duration_txt})"
  1474. # the key status message
  1475. status_message="status unknown"
  1476. # the color of the alarm
  1477. color="grey"
  1478. # the alarm value
  1479. alarm="${name//_/ } = ${value_string}"
  1480. # the image of the alarm
  1481. image="${images_base_url}/images/banner-icon-144x144.png"
  1482. # prepare the title based on status
  1483. case "${status}" in
  1484. CRITICAL)
  1485. image="${images_base_url}/images/alert-128-red.png"
  1486. status_message="is critical"
  1487. color="#ca414b"
  1488. ;;
  1489. WARNING)
  1490. image="${images_base_url}/images/alert-128-orange.png"
  1491. status_message="needs attention"
  1492. color="#ffc107"
  1493. ;;
  1494. CLEAR)
  1495. image="${images_base_url}/images/check-mark-2-128-green.png"
  1496. status_message="recovered"
  1497. color="#77ca6d"
  1498. ;;
  1499. esac
  1500. if [ "${status}" = "CLEAR" ]
  1501. then
  1502. severity="Recovered from ${old_status}"
  1503. if [ ${non_clear_duration} -gt ${duration} ]
  1504. then
  1505. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1506. fi
  1507. # don't show the value when the status is CLEAR
  1508. # for certain alarms, this value might not have any meaning
  1509. alarm="${name//_/ } ${raised_for}"
  1510. elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
  1511. then
  1512. severity="Escalated to ${status}"
  1513. if [ ${non_clear_duration} -gt ${duration} ]
  1514. then
  1515. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1516. fi
  1517. elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
  1518. then
  1519. severity="Demoted to ${status}"
  1520. if [ ${non_clear_duration} -gt ${duration} ]
  1521. then
  1522. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1523. fi
  1524. else
  1525. raised_for=
  1526. fi
  1527. # prepare HTML versions of elements
  1528. info_html=
  1529. [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
  1530. raised_for_html=
  1531. [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1532. # -----------------------------------------------------------------------------
  1533. # send the slack notification
  1534. # slack aggregates posts from the same username
  1535. # so we use "${host} ${status}" as the bot username, to make them diff
  1536. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1537. SENT_SLACK=$?
  1538. # -----------------------------------------------------------------------------
  1539. # send the Microsoft notification
  1540. # Microsoft team aggregates posts from the same username
  1541. # so we use "${host} ${status}" as the bot username, to make them diff
  1542. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1543. SENT_MSTEAM=$?
  1544. # -----------------------------------------------------------------------------
  1545. # send the rocketchat notification
  1546. # rocketchat aggregates posts from the same username
  1547. # so we use "${host} ${status}" as the bot username, to make them diff
  1548. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1549. SENT_ROCKETCHAT=$?
  1550. # -----------------------------------------------------------------------------
  1551. # send the alerta notification
  1552. # alerta aggregates posts from the same username
  1553. # so we use "${host} ${status}" as the bot username, to make them diff
  1554. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1555. SENT_ALERTA=$?
  1556. # -----------------------------------------------------------------------------
  1557. # send the flock notification
  1558. # flock aggregates posts from the same username
  1559. # so we use "${host} ${status}" as the bot username, to make them diff
  1560. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1561. SENT_FLOCK=$?
  1562. # -----------------------------------------------------------------------------
  1563. # send the discord notification
  1564. # discord aggregates posts from the same username
  1565. # so we use "${host} ${status}" as the bot username, to make them diff
  1566. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1567. SENT_DISCORD=$?
  1568. # -----------------------------------------------------------------------------
  1569. # send the pushover notification
  1570. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1571. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1572. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1573. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1574. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1575. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1576. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1577. <small><small>The source of this alarm is line ${src}</small></small>
  1578. "
  1579. SENT_PUSHOVER=$?
  1580. # -----------------------------------------------------------------------------
  1581. # send the pushbullet notification
  1582. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
  1583. Severity: ${severity}\n
  1584. Chart: ${chart}\n
  1585. Family: ${family}\n
  1586. $(date -d @${when})\n
  1587. The source of this alarm is line ${src}"
  1588. SENT_PUSHBULLET=$?
  1589. # -----------------------------------------------------------------------------
  1590. # send the twilio SMS
  1591. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1592. Severity: ${severity}
  1593. Chart: ${chart}
  1594. Family: ${family}
  1595. ${info}"
  1596. SENT_TWILIO=$?
  1597. # -----------------------------------------------------------------------------
  1598. # send the messagebird SMS
  1599. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1600. Severity: ${severity}
  1601. Chart: ${chart}
  1602. Family: ${family}
  1603. ${info}"
  1604. SENT_MESSAGEBIRD=$?
  1605. # -----------------------------------------------------------------------------
  1606. # send the kavenegar SMS
  1607. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1608. Severity: ${severity}
  1609. Chart: ${chart}
  1610. Family: ${family}
  1611. ${info}"
  1612. SENT_KAVENEGAR=$?
  1613. # -----------------------------------------------------------------------------
  1614. # send the telegram.org message
  1615. # https://core.telegram.org/bots/api#formatting-options
  1616. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1617. ${chart} (${family})
  1618. <a href=\"${goto_url}\">${alarm}</a>
  1619. <i>${info}</i>"
  1620. SENT_TELEGRAM=$?
  1621. # -----------------------------------------------------------------------------
  1622. # send the kafka message
  1623. send_kafka
  1624. SENT_KAFKA=$?
  1625. # -----------------------------------------------------------------------------
  1626. # send the pagerduty.com message
  1627. send_pd "${to_pd}"
  1628. SENT_PD=$?
  1629. # -----------------------------------------------------------------------------
  1630. # send the fleep message
  1631. send_fleep "${to_fleep}"
  1632. SENT_FLEEP=$?
  1633. # -----------------------------------------------------------------------------
  1634. # send the irc message
  1635. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1636. Severity: ${severity}
  1637. Chart: ${chart}
  1638. Family: ${family}
  1639. ${info}"
  1640. SENT_IRC=$?
  1641. # -----------------------------------------------------------------------------
  1642. # send the custom message
  1643. send_custom() {
  1644. # is it enabled?
  1645. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1646. # do we have any sender?
  1647. [ -z "${1}" ] && return 1
  1648. # call the custom_sender function
  1649. custom_sender "${@}"
  1650. }
  1651. send_custom "${to_custom}"
  1652. SENT_CUSTOM=$?
  1653. # -----------------------------------------------------------------------------
  1654. # send hipchat message
  1655. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1656. ${host} ${status_message}<br/> \
  1657. <b>${alarm}</b> ${info_html}<br/> \
  1658. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1659. <b>${date}${raised_for_html}</b><br/> \
  1660. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1661. (source of alarm ${src}) \
  1662. "
  1663. SENT_HIPCHAT=$?
  1664. # -----------------------------------------------------------------------------
  1665. # send the Amazon SNS message
  1666. send_awssns ${to_awssns}
  1667. SENT_AWSSNS=$?
  1668. # -----------------------------------------------------------------------------
  1669. # send the syslog message
  1670. send_syslog ${to_syslog}
  1671. SENT_SYSLOG=$?
  1672. # -----------------------------------------------------------------------------
  1673. # send the email
  1674. send_email <<EOF
  1675. To: ${to_email}
  1676. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1677. MIME-Version: 1.0
  1678. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1679. ${email_thread_headers}
  1680. This is a MIME-encoded multipart message
  1681. --multipart-boundary
  1682. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1683. Content-Disposition: inline
  1684. Content-Transfer-Encoding: 8bit
  1685. ${host} ${status_message}
  1686. ${alarm} ${info}
  1687. ${raised_for}
  1688. Chart : ${chart}
  1689. Family : ${family}
  1690. Severity: ${severity}
  1691. URL : ${goto_url}
  1692. Source : ${src}
  1693. Date : ${date}
  1694. Notification generated on ${host}
  1695. --multipart-boundary
  1696. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1697. Content-Disposition: inline
  1698. Content-Transfer-Encoding: 8bit
  1699. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1700. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1701. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1702. <table>
  1703. <tbody>
  1704. <tr>
  1705. <td style="vertical-align: top;" valign="top"></td>
  1706. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1707. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1708. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1709. <tbody>
  1710. <tr>
  1711. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1712. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1713. </td>
  1714. </tr>
  1715. <tr>
  1716. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1717. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1718. </td>
  1719. </tr>
  1720. <tr>
  1721. <td style="vertical-align: top;" valign="top">
  1722. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1723. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  1724. <tbody>
  1725. <tr>
  1726. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  1727. <span>${chart}</span>
  1728. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  1729. </td>
  1730. </tr>
  1731. <tr style="margin: 0; padding: 0;">
  1732. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1733. <span><b>${alarm}</b>${info_html}</span>
  1734. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  1735. </td>
  1736. </tr>
  1737. <tr>
  1738. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1739. <span>${family}</span>
  1740. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  1741. </td>
  1742. </tr>
  1743. <tr style="margin: 0; padding: 0;">
  1744. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1745. <span>${severity}</span>
  1746. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  1747. </td>
  1748. </tr>
  1749. <tr style="margin: 0; padding: 0;">
  1750. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  1751. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  1752. </td>
  1753. </tr>
  1754. <tr style="margin: 0; padding: 0;">
  1755. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  1756. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  1757. </td>
  1758. </tr>
  1759. <tr style="text-align: center; margin: 0; padding: 0;">
  1760. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  1761. </td>
  1762. </tr>
  1763. <tr style="text-align: center; margin: 0; padding: 0;">
  1764. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  1765. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  1766. </td>
  1767. </tr>
  1768. </tbody>
  1769. </table>
  1770. </div>
  1771. </td>
  1772. </tr>
  1773. </tbody>
  1774. </table>
  1775. </div>
  1776. </td>
  1777. </tr>
  1778. </tbody>
  1779. </table>
  1780. </body>
  1781. </html>
  1782. --multipart-boundary--
  1783. EOF
  1784. SENT_EMAIL=$?
  1785. # -----------------------------------------------------------------------------
  1786. # let netdata know
  1787. if [ ${SENT_EMAIL} -eq 0 \
  1788. -o ${SENT_PUSHOVER} -eq 0 \
  1789. -o ${SENT_TELEGRAM} -eq 0 \
  1790. -o ${SENT_SLACK} -eq 0 \
  1791. -o ${SENT_MSTEAM} -eq 0 \
  1792. -o ${SENT_ROCKETCHAT} -eq 0 \
  1793. -o ${SENT_ALERTA} -eq 0 \
  1794. -o ${SENT_FLOCK} -eq 0 \
  1795. -o ${SENT_DISCORD} -eq 0 \
  1796. -o ${SENT_TWILIO} -eq 0 \
  1797. -o ${SENT_HIPCHAT} -eq 0 \
  1798. -o ${SENT_MESSAGEBIRD} -eq 0 \
  1799. -o ${SENT_KAVENEGAR} -eq 0 \
  1800. -o ${SENT_PUSHBULLET} -eq 0 \
  1801. -o ${SENT_KAFKA} -eq 0 \
  1802. -o ${SENT_PD} -eq 0 \
  1803. -o ${SENT_FLEEP} -eq 0 \
  1804. -o ${SENT_IRC} -eq 0 \
  1805. -o ${SENT_AWSSNS} -eq 0 \
  1806. -o ${SENT_CUSTOM} -eq 0 \
  1807. -o ${SENT_SYSLOG} -eq 0 \
  1808. ]
  1809. then
  1810. # we did send something
  1811. exit 0
  1812. fi
  1813. # we did not send anything
  1814. exit 1