alarm-notify.sh.in 74 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - custom notifications by @ktsaou
  32. # - syslog messages by @Ferroin
  33. # - Microsoft Team notification by @tioumen
  34. # - RocketChat notifications by @Hermsi1337 #3777
  35. # -----------------------------------------------------------------------------
  36. # testing notifications
  37. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  38. if [ "${2}" = "test" ]; then
  39. recipient="${1}"
  40. else
  41. recipient="${2}"
  42. fi
  43. [ -z "${recipient}" ] && recipient="sysadmin"
  44. id=1
  45. last="CLEAR"
  46. test_res=0
  47. for x in "WARNING" "CRITICAL" "CLEAR"; do
  48. echo >&2
  49. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  50. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  51. #shellcheck disable=SC2181
  52. if [ $? -ne 0 ]; then
  53. echo >&2 "# FAILED"
  54. test_res=1
  55. else
  56. echo >&2 "# OK"
  57. fi
  58. last="${x}"
  59. id=$((id + 1))
  60. done
  61. exit $test_res
  62. fi
  63. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  64. export LC_ALL=C
  65. # -----------------------------------------------------------------------------
  66. PROGRAM_NAME="$(basename "${0}")"
  67. logdate() {
  68. date "+%Y-%m-%d %H:%M:%S"
  69. }
  70. log() {
  71. local status="${1}"
  72. shift
  73. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  74. }
  75. warning() {
  76. log WARNING "${@}"
  77. }
  78. error() {
  79. log ERROR "${@}"
  80. }
  81. info() {
  82. log INFO "${@}"
  83. }
  84. fatal() {
  85. log FATAL "${@}"
  86. exit 1
  87. }
  88. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  89. debug() {
  90. [ "${debug}" = "1" ] && log DEBUG "${@}"
  91. }
  92. docurl() {
  93. if [ -z "${curl}" ]; then
  94. error "${curl} is unset."
  95. return 1
  96. fi
  97. if [ "${debug}" = "1" ]; then
  98. echo >&2 "--- BEGIN curl command ---"
  99. printf >&2 "%q " ${curl} "${@}"
  100. echo >&2
  101. echo >&2 "--- END curl command ---"
  102. local out code ret
  103. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  104. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  105. ret=$?
  106. echo >&2 "--- BEGIN received response ---"
  107. cat >&2 "${out}"
  108. echo >&2
  109. echo >&2 "--- END received response ---"
  110. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  111. rm "${out}"
  112. echo "${code}"
  113. return ${ret}
  114. fi
  115. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  116. return $?
  117. }
  118. # -----------------------------------------------------------------------------
  119. # List of all the notification mechanisms we support.
  120. # Used in a couple of places to write more compact code.
  121. method_names="
  122. email
  123. pushover
  124. pushbullet
  125. telegram
  126. slack
  127. alerta
  128. flock
  129. discord
  130. hipchat
  131. twilio
  132. messagebird
  133. pd
  134. fleep
  135. syslog
  136. custom
  137. msteam
  138. kavenegar
  139. prowl
  140. awssns
  141. rocketchat
  142. sms
  143. "
  144. # -----------------------------------------------------------------------------
  145. # this is to be overwritten by the config file
  146. custom_sender() {
  147. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  148. }
  149. # -----------------------------------------------------------------------------
  150. # check for BASH v4+ (required for associative arrays)
  151. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  152. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  153. fi
  154. # -----------------------------------------------------------------------------
  155. # defaults to allow running this script by hand
  156. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  157. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  158. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  159. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  160. [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://netdata.cloud"
  161. # -----------------------------------------------------------------------------
  162. # parse command line parameters
  163. if [ ${1} = "unittest" ]; then
  164. unittest=1 # enable unit testing mode
  165. roles="${2}" # the role that should be used for unit testing
  166. cfgfile="${3}" # the location of the config file to use for unit testing
  167. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  168. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  169. else
  170. roles="${1}" # the roles that should be notified for this event
  171. args_host="${2}" # the host generated this event
  172. unique_id="${3}" # the unique id of this event
  173. alarm_id="${4}" # the unique id of the alarm that generated this event
  174. event_id="${5}" # the incremental id of the event, for this alarm id
  175. when="${6}" # the timestamp this event occurred
  176. name="${7}" # the name of the alarm, as given in netdata health.d entries
  177. chart="${8}" # the name of the chart (type.id)
  178. family="${9}" # the family of the chart
  179. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  180. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  181. value="${12}" # the current value of the alarm
  182. old_value="${13}" # the previous value of the alarm
  183. src="${14}" # the line number and file the alarm has been configured
  184. duration="${15}" # the duration in seconds of the previous alarm state
  185. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  186. units="${17}" # the units of the value
  187. info="${18}" # a short description of the alarm
  188. value_string="${19}" # friendly value (with units)
  189. # shellcheck disable=SC2034
  190. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  191. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  192. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  193. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  194. total_warnings="${23}" # Total number of alarms in WARNING state
  195. total_critical="${24}" # Total number of alarms in CRITICAL state
  196. fi
  197. # -----------------------------------------------------------------------------
  198. # find a suitable hostname to use, if netdata did not supply a hostname
  199. if [ -z ${args_host} ]; then
  200. this_host=$(hostname -s 2>/dev/null)
  201. host="${this_host}"
  202. args_host="${this_host}"
  203. else
  204. host="${args_host}"
  205. fi
  206. # -----------------------------------------------------------------------------
  207. # screen statuses we don't need to send a notification
  208. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  209. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  210. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  211. exit 1
  212. fi
  213. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  214. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  215. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  216. exit 1
  217. fi
  218. # -----------------------------------------------------------------------------
  219. # load configuration
  220. # By default fetch images from the global public registry.
  221. # This is required by default, since all notification methods need to download
  222. # images via the Internet, and private registries might not be reachable.
  223. # This can be overwritten at the configuration file.
  224. images_base_url="https://registry.my-netdata.io"
  225. # curl options to use
  226. curl_options=""
  227. # hostname handling
  228. use_fqdn="NO"
  229. # needed commands
  230. # if empty they will be searched in the system path
  231. curl=
  232. sendmail=
  233. # enable / disable features
  234. for method_name in ${method_names^^}; do
  235. declare SEND_${method_name}="YES"
  236. declare DEFAULT_RECIPIENT_${method_name}
  237. done
  238. for method_name in ${method_names}; do
  239. declare -A role_recipients_${method_name}
  240. done
  241. # slack configs
  242. SLACK_WEBHOOK_URL=
  243. # Microsoft Team configs
  244. MSTEAM_WEBHOOK_URL=
  245. # rocketchat configs
  246. ROCKETCHAT_WEBHOOK_URL=
  247. # alerta configs
  248. ALERTA_WEBHOOK_URL=
  249. ALERTA_API_KEY=
  250. # flock configs
  251. FLOCK_WEBHOOK_URL=
  252. # discord configs
  253. DISCORD_WEBHOOK_URL=
  254. # pushover configs
  255. PUSHOVER_APP_TOKEN=
  256. # pushbullet configs
  257. PUSHBULLET_ACCESS_TOKEN=
  258. PUSHBULLET_SOURCE_DEVICE=
  259. # twilio configs
  260. TWILIO_ACCOUNT_SID=
  261. TWILIO_ACCOUNT_TOKEN=
  262. TWILIO_NUMBER=
  263. # hipchat configs
  264. HIPCHAT_SERVER=
  265. HIPCHAT_AUTH_TOKEN=
  266. # messagebird configs
  267. MESSAGEBIRD_ACCESS_KEY=
  268. MESSAGEBIRD_NUMBER=
  269. # kavenegar configs
  270. KAVENEGAR_API_KEY=
  271. KAVENEGAR_SENDER=
  272. # telegram configs
  273. TELEGRAM_BOT_TOKEN=
  274. # kafka configs
  275. SEND_KAFKA="YES"
  276. KAFKA_URL=
  277. KAFKA_SENDER_IP=
  278. # pagerduty.com configs
  279. PD_SERVICE_KEY=
  280. # fleep.io configs
  281. FLEEP_SENDER="${host}"
  282. # Amazon SNS configs
  283. AWSSNS_MESSAGE_FORMAT=
  284. # syslog configs
  285. SYSLOG_FACILITY=
  286. # email configs
  287. EMAIL_SENDER=
  288. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  289. EMAIL_THREADING=
  290. # irc configs
  291. IRC_NICKNAME=
  292. IRC_REALNAME=
  293. IRC_NETWORK=
  294. # load the stock and user configuration files
  295. # these will overwrite the variables above
  296. if [ ${unittest} ]; then
  297. if source "${cfgfile}"; then
  298. error "Failed to load requested config file."
  299. exit 1
  300. fi
  301. else
  302. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  303. if [ -f "${CONFIG}" ]; then
  304. debug "Loading config file '${CONFIG}'..."
  305. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  306. else
  307. warning "Cannot find file '${CONFIG}'."
  308. fi
  309. done
  310. fi
  311. # If we didn't autodetect the character set for e-mail and it wasn't
  312. # set by the user, we need to set it to a reasonable default. UTF-8
  313. # should be correct for almost all modern UNIX systems.
  314. if [ -z ${EMAIL_CHARSET} ]; then
  315. EMAIL_CHARSET="UTF-8"
  316. fi
  317. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  318. # unless we're sending an alarm for a slave system which we can't get the
  319. # FQDN of easily.
  320. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  321. host="$(hostname -f 2>/dev/null)"
  322. fi
  323. # -----------------------------------------------------------------------------
  324. # filter a recipient based on alarm event severity
  325. filter_recipient_by_criticality() {
  326. local method="${1}" x="${2}" r s
  327. shift
  328. r="${x/|*/}" # the recipient
  329. s="${x/*|/}" # the severity required for notifying this recipient
  330. # no severity filtering for this person
  331. [ "${r}" = "${s}" ] && return 0
  332. # the severity is invalid
  333. s="${s^^}"
  334. if [ "${s}" != "CRITICAL" ]; then
  335. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  336. return 0
  337. fi
  338. # create the status tracking directory for this user
  339. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  340. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  341. case "${status}" in
  342. CRITICAL)
  343. # make sure he will get future notifications for this alarm too
  344. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  345. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  346. return 0
  347. ;;
  348. WARNING)
  349. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  350. # we do not remove the file, so that he will get future notifications of this alarm
  351. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  352. return 0
  353. fi
  354. ;;
  355. *)
  356. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  357. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  358. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  359. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  360. return 0
  361. fi
  362. ;;
  363. esac
  364. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  365. return 1
  366. }
  367. # -----------------------------------------------------------------------------
  368. # verify the delivery methods supported
  369. # check slack
  370. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  371. # check rocketchat
  372. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  373. # check alerta
  374. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  375. # check flock
  376. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  377. # check discord
  378. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  379. # check pushover
  380. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  381. # check pushbullet
  382. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  383. # check twilio
  384. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  385. # check hipchat
  386. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  387. # check messagebird
  388. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  389. # check kavenegar
  390. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  391. # check telegram
  392. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  393. # check kafka
  394. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  395. # check irc
  396. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  397. # check fleep
  398. #shellcheck disable=SC2153
  399. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  400. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  401. [ "${SEND_SLACK}" = "YES" ] ||
  402. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  403. [ "${SEND_ALERTA}" = "YES" ] ||
  404. [ "${SEND_PD}" = "YES" ] ||
  405. [ "${SEND_FLOCK}" = "YES" ] ||
  406. [ "${SEND_DISCORD}" = "YES" ] ||
  407. [ "${SEND_HIPCHAT}" = "YES" ] ||
  408. [ "${SEND_TWILIO}" = "YES" ] ||
  409. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  410. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  411. [ "${SEND_TELEGRAM}" = "YES" ] ||
  412. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  413. [ "${SEND_KAFKA}" = "YES" ] ||
  414. [ "${SEND_FLEEP}" = "YES" ] ||
  415. [ "${SEND_PROWL}" = "YES" ] ||
  416. [ "${SEND_CUSTOM}" = "YES" ] ||
  417. [ "${SEND_MSTEAM}" = "YES" ]; then
  418. # if we need curl, check for the curl command
  419. if [ -z "${curl}" ]; then
  420. curl="$(command -v curl 2>/dev/null)"
  421. fi
  422. if [ -z "${curl}" ]; then
  423. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  424. SEND_PUSHOVER="NO"
  425. SEND_PUSHBULLET="NO"
  426. SEND_TELEGRAM="NO"
  427. SEND_SLACK="NO"
  428. SEND_MSTEAM="NO"
  429. SEND_ROCKETCHAT="NO"
  430. SEND_ALERTA="NO"
  431. SEND_PD="NO"
  432. SEND_FLOCK="NO"
  433. SEND_DISCORD="NO"
  434. SEND_TWILIO="NO"
  435. SEND_HIPCHAT="NO"
  436. SEND_MESSAGEBIRD="NO"
  437. SEND_KAVENEGAR="NO"
  438. SEND_KAFKA="NO"
  439. SEND_FLEEP="NO"
  440. SEND_PROWL="NO"
  441. SEND_CUSTOM="NO"
  442. fi
  443. fi
  444. if [ "${SEND_SMS}" = "YES" ]; then
  445. if [ -z "${sendsms}" ]; then
  446. sendsms="$(command -v sendsms 2>/dev/null)"
  447. fi
  448. if [ -z "${sendsms}" ]; then
  449. SEND_SMS="NO"
  450. fi
  451. fi
  452. # if we need sendmail, check for the sendmail command
  453. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  454. sendmail="$(command -v sendmail 2>/dev/null)"
  455. if [ -z "${sendmail}" ]; then
  456. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  457. SEND_EMAIL="NO"
  458. fi
  459. fi
  460. # if we need logger, check for the logger command
  461. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  462. logger="$(command -v logger 2>/dev/null)"
  463. if [ -z "${logger}" ]; then
  464. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  465. SEND_SYSLOG="NO"
  466. fi
  467. fi
  468. # if we need aws, check for the aws command
  469. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  470. aws="$(command -v aws 2>/dev/null)"
  471. if [ -z "${aws}" ]; then
  472. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  473. SEND_AWSSNS="NO"
  474. fi
  475. fi
  476. # -----------------------------------------------------------------------------
  477. # find the recipients' addresses per method
  478. # netdata may call us with multiple roles, and roles may have multiple but
  479. # overlapping recipients - so, here we find the unique recipients.
  480. for method_name in ${method_names}; do
  481. send_var="SEND_${method_name^^}"
  482. if [ "${!send_var}" = "NO" ]; then
  483. continue
  484. fi
  485. declare -A arr_var=()
  486. for x in ${roles//,/ }; do
  487. # the roles 'silent' and 'disabled' mean:
  488. # don't send a notification for this role
  489. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  490. continue
  491. fi
  492. role_recipients="role_recipients_${method_name}[$x]"
  493. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  494. a="${!role_recipients}"
  495. [ -z "${a}" ] && a="${!default_recipient_var}"
  496. for r in ${a//,/ }; do
  497. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  498. done
  499. done
  500. # build the list of recipients
  501. to_var="to_${method_name}"
  502. declare to_${method_name}="${!arr_var[*]}"
  503. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  504. done
  505. # -----------------------------------------------------------------------------
  506. # handle fixup of the email recipient list.
  507. fix_to_email() {
  508. to_email=
  509. while [ -n "${1}" ]; do
  510. [ -n "${to_email}" ] && to_email="${to_email}, "
  511. to_email="${to_email}${1}"
  512. shift 1
  513. done
  514. }
  515. # ${to_email} without quotes here
  516. fix_to_email ${to_email}
  517. # -----------------------------------------------------------------------------
  518. # handle output if we're running in unit test mode
  519. if [ ${unittest} ]; then
  520. for method_name in ${method_names}; do
  521. to_var="to_${method_name}"
  522. echo "results: ${method_name}: ${!to_var}"
  523. done
  524. exit 0
  525. fi
  526. # -----------------------------------------------------------------------------
  527. # check that we have at least a method enabled
  528. proceed=0
  529. for method in "${SEND_EMAIL}" \
  530. "${SEND_PUSHOVER}" \
  531. "${SEND_TELEGRAM}" \
  532. "${SEND_SLACK}" \
  533. "${SEND_ROCKETCHAT}" \
  534. "${SEND_ALERTA}" \
  535. "${SEND_FLOCK}" \
  536. "${SEND_DISCORD}" \
  537. "${SEND_TWILIO}" \
  538. "${SEND_HIPCHAT}" \
  539. "${SEND_MESSAGEBIRD}" \
  540. "${SEND_KAVENEGAR}" \
  541. "${SEND_PUSHBULLET}" \
  542. "${SEND_KAFKA}" \
  543. "${SEND_PD}" \
  544. "${SEND_FLEEP}" \
  545. "${SEND_PROWL}" \
  546. "${SEND_CUSTOM}" \
  547. "${SEND_IRC}" \
  548. "${SEND_AWSSNS}" \
  549. "${SEND_SYSLOG}" \
  550. "${SEND_SMS}" \
  551. "${SEND_MSTEAM}"; do
  552. if [ "${method}" == "YES" ]; then
  553. proceed=1
  554. break
  555. fi
  556. done
  557. if [ "$proceed" -eq 0 ]; then
  558. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  559. fi
  560. # -----------------------------------------------------------------------------
  561. # get the date the alarm happened
  562. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  563. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  564. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  565. [ -z "${date}" ] && date=$(date 2>/dev/null)
  566. # ----------------------------------------------------------------------------
  567. # prepare some extra headers if we've been asked to thread e-mails
  568. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  569. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
  570. else
  571. email_thread_headers=
  572. fi
  573. # -----------------------------------------------------------------------------
  574. # function to URL encode a string
  575. urlencode() {
  576. local string="${1}" strlen encoded pos c o
  577. strlen=${#string}
  578. for ((pos = 0; pos < strlen; pos++)); do
  579. c=${string:pos:1}
  580. case "${c}" in
  581. [-_.~a-zA-Z0-9])
  582. o="${c}"
  583. ;;
  584. *)
  585. printf -v o '%%%02x' "'${c}"
  586. ;;
  587. esac
  588. encoded+="${o}"
  589. done
  590. REPLY="${encoded}"
  591. echo "${REPLY}"
  592. }
  593. # -----------------------------------------------------------------------------
  594. # function to convert a duration in seconds, to a human readable duration
  595. # using DAYS, MINUTES, SECONDS
  596. duration4human() {
  597. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  598. d=$((s / 86400))
  599. s=$((s - (d * 86400)))
  600. h=$((s / 3600))
  601. s=$((s - (h * 3600)))
  602. m=$((s / 60))
  603. s=$((s - (m * 60)))
  604. if [ ${d} -gt 0 ]; then
  605. [ ${m} -ge 30 ] && h=$((h + 1))
  606. [ ${d} -gt 1 ] && ds="days"
  607. [ ${h} -gt 1 ] && hs="hours"
  608. if [ ${h} -gt 0 ]; then
  609. ret="${d} ${ds} and ${h} ${hs}"
  610. else
  611. ret="${d} ${ds}"
  612. fi
  613. elif [ ${h} -gt 0 ]; then
  614. [ ${s} -ge 30 ] && m=$((m + 1))
  615. [ ${h} -gt 1 ] && hs="hours"
  616. [ ${m} -gt 1 ] && ms="minutes"
  617. if [ ${m} -gt 0 ]; then
  618. ret="${h} ${hs} and ${m} ${ms}"
  619. else
  620. ret="${h} ${hs}"
  621. fi
  622. elif [ ${m} -gt 0 ]; then
  623. [ ${m} -gt 1 ] && ms="minutes"
  624. [ ${s} -gt 1 ] && ss="seconds"
  625. if [ ${s} -gt 0 ]; then
  626. ret="${m} ${ms} and ${s} ${ss}"
  627. else
  628. ret="${m} ${ms}"
  629. fi
  630. else
  631. [ ${s} -gt 1 ] && ss="seconds"
  632. ret="${s} ${ss}"
  633. fi
  634. REPLY="${ret}"
  635. echo "${REPLY}"
  636. }
  637. # -----------------------------------------------------------------------------
  638. # email sender
  639. send_email() {
  640. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  641. if [ "${SEND_EMAIL}" = "YES" ]; then
  642. if [ -n "${EMAIL_SENDER}" ]; then
  643. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  644. # the name includes double quotes
  645. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  646. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  647. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  648. # the name includes single quotes
  649. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  650. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  651. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  652. # the name does not have any quotes
  653. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  654. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  655. fi
  656. fi
  657. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  658. [ -n "${sender_name}" ] && opts+=(-F "${sender_name}")
  659. if [ "${debug}" = "1" ]; then
  660. echo >&2 "--- BEGIN sendmail command ---"
  661. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  662. echo >&2
  663. echo >&2 "--- END sendmail command ---"
  664. fi
  665. "${sendmail}" -t "${opts[@]}"
  666. ret=$?
  667. if [ ${ret} -eq 0 ]; then
  668. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  669. return 0
  670. else
  671. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  672. return 1
  673. fi
  674. fi
  675. return 1
  676. }
  677. # -----------------------------------------------------------------------------
  678. # pushover sender
  679. send_pushover() {
  680. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  681. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  682. # https://pushover.net/api
  683. priority=-2
  684. case "${status}" in
  685. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  686. WARNING) priority=0 ;; # normal priority: respect quiet hours
  687. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  688. *) priority=-2 ;; # lowest priority: no notification at all
  689. esac
  690. for user in ${usertokens}; do
  691. httpcode=$(docurl \
  692. --form-string "token=${apptoken}" \
  693. --form-string "user=${user}" \
  694. --form-string "html=1" \
  695. --form-string "title=${title}" \
  696. --form-string "message=${message}" \
  697. --form-string "timestamp=${when}" \
  698. --form-string "url=${url}" \
  699. --form-string "url_title=Open netdata dashboard to view the alarm" \
  700. --form-string "priority=${priority}" \
  701. https://api.pushover.net/1/messages.json)
  702. if [ "${httpcode}" = "200" ]; then
  703. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  704. sent=$((sent + 1))
  705. else
  706. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  707. fi
  708. done
  709. [ ${sent} -gt 0 ] && return 0
  710. fi
  711. return 1
  712. }
  713. # -----------------------------------------------------------------------------
  714. # pushbullet sender
  715. send_pushbullet() {
  716. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  717. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  718. #https://docs.pushbullet.com/#create-push
  719. for user in ${recipients}; do
  720. httpcode=$(docurl \
  721. --header 'Access-Token: '${userapikey}'' \
  722. --header 'Content-Type: application/json' \
  723. --data-binary @<(
  724. cat <<EOF
  725. {"title": "${title}",
  726. "type": "link",
  727. "email": "${user}",
  728. "body": "$(echo -n ${message})",
  729. "url": "${url}",
  730. "source_device_iden": "${source_device}"}
  731. EOF
  732. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  733. if [ "${httpcode}" = "200" ]; then
  734. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  735. sent=$((sent + 1))
  736. else
  737. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  738. fi
  739. done
  740. [ ${sent} -gt 0 ] && return 0
  741. fi
  742. return 1
  743. }
  744. # -----------------------------------------------------------------------------
  745. # kafka sender
  746. send_kafka() {
  747. local httpcode sent=0
  748. if [ "${SEND_KAFKA}" = "YES" ]; then
  749. httpcode=$(docurl -X POST \
  750. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  751. "${KAFKA_URL}")
  752. if [ "${httpcode}" = "204" ]; then
  753. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  754. sent=$((sent + 1))
  755. else
  756. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  757. fi
  758. [ ${sent} -gt 0 ] && return 0
  759. fi
  760. return 1
  761. }
  762. # -----------------------------------------------------------------------------
  763. # pagerduty.com sender
  764. send_pd() {
  765. local recipients="${1}" sent=0
  766. unset t
  767. case ${status} in
  768. CLEAR) t='resolve' ;;
  769. WARNING) t='trigger' ;;
  770. CRITICAL) t='trigger' ;;
  771. esac
  772. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  773. for PD_SERVICE_KEY in ${recipients}; do
  774. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  775. payload="$(
  776. cat <<EOF
  777. {
  778. "service_key": "${PD_SERVICE_KEY}",
  779. "event_type": "${t}",
  780. "incident_key" : "${alarm_id}",
  781. "description": "${d}",
  782. "details": {
  783. "value_w_units": "${value_string}",
  784. "when": "${when}",
  785. "duration" : "${duration}",
  786. "roles": "${roles}",
  787. "alarm_id" : "${alarm_id}",
  788. "name" : "${name}",
  789. "chart" : "${chart}",
  790. "family" : "${family}",
  791. "status" : "${status}",
  792. "old_status" : "${old_status}",
  793. "value" : "${value}",
  794. "old_value" : "${old_value}",
  795. "src" : "${src}",
  796. "non_clear_duration" : "${non_clear_duration}",
  797. "units" : "${units}",
  798. "info" : "${info}"
  799. }
  800. }
  801. EOF
  802. )"
  803. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  804. if [ "${httpcode}" = "200" ]; then
  805. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  806. sent=$((sent + 1))
  807. else
  808. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  809. fi
  810. done
  811. [ ${sent} -gt 0 ] && return 0
  812. fi
  813. return 1
  814. }
  815. # -----------------------------------------------------------------------------
  816. # twilio sender
  817. send_twilio() {
  818. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  819. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  820. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  821. for user in ${recipients}; do
  822. httpcode=$(docurl -X POST \
  823. --data-urlencode "From=${twilionumber}" \
  824. --data-urlencode "To=${user}" \
  825. --data-urlencode "Body=${title} ${message}" \
  826. -u "${accountsid}:${accounttoken}" \
  827. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  828. if [ "${httpcode}" = "201" ]; then
  829. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  830. sent=$((sent + 1))
  831. else
  832. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  833. fi
  834. done
  835. [ ${sent} -gt 0 ] && return 0
  836. fi
  837. return 1
  838. }
  839. # -----------------------------------------------------------------------------
  840. # hipchat sender
  841. send_hipchat() {
  842. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  843. # remove <small></small> from the message
  844. message="${message//<small>/}"
  845. message="${message//<\/small>/}"
  846. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  847. # Valid values: html, text.
  848. # Defaults to 'html'.
  849. msg_format="html"
  850. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  851. case "${status}" in
  852. WARNING) color="yellow" ;;
  853. CRITICAL) color="red" ;;
  854. CLEAR) color="green" ;;
  855. *) color="gray" ;;
  856. esac
  857. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  858. # Each recipient's notification preferences are taken into account.
  859. # Defaults to false.
  860. notify="true"
  861. for room in ${recipients}; do
  862. httpcode=$(docurl -X POST \
  863. -H "Content-type: application/json" \
  864. -H "Authorization: Bearer ${authtoken}" \
  865. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  866. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  867. if [ "${httpcode}" = "204" ]; then
  868. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  869. sent=$((sent + 1))
  870. else
  871. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  872. fi
  873. done
  874. [ ${sent} -gt 0 ] && return 0
  875. fi
  876. return 1
  877. }
  878. # -----------------------------------------------------------------------------
  879. # messagebird sender
  880. send_messagebird() {
  881. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  882. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  883. #https://developers.messagebird.com/docs/messaging
  884. for user in ${recipients}; do
  885. httpcode=$(docurl -X POST \
  886. --data-urlencode "originator=${messagebirdnumber}" \
  887. --data-urlencode "recipients=${user}" \
  888. --data-urlencode "body=${title} ${message}" \
  889. --data-urlencode "datacoding=auto" \
  890. -H "Authorization: AccessKey ${accesskey}" \
  891. "https://rest.messagebird.com/messages")
  892. if [ "${httpcode}" = "201" ]; then
  893. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  894. sent=$((sent + 1))
  895. else
  896. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  897. fi
  898. done
  899. [ ${sent} -gt 0 ] && return 0
  900. fi
  901. return 1
  902. }
  903. # -----------------------------------------------------------------------------
  904. # kavenegar sender
  905. send_kavenegar() {
  906. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  907. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  908. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  909. for user in ${recipients}; do
  910. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  911. --data-urlencode "sender=${kavenegarsender}" \
  912. --data-urlencode "receptor=${user}" \
  913. --data-urlencode "message=${title} ${message}")
  914. if [ "${httpcode}" = "200" ]; then
  915. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  916. sent=$((sent + 1))
  917. else
  918. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  919. fi
  920. done
  921. [ ${sent} -gt 0 ] && return 0
  922. fi
  923. return 1
  924. }
  925. # -----------------------------------------------------------------------------
  926. # telegram sender
  927. send_telegram() {
  928. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  929. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  930. case "${status}" in
  931. WARNING) emoji="⚠️" ;;
  932. CRITICAL) emoji="🔴" ;;
  933. CLEAR) emoji="✅" ;;
  934. *) emoji="⚪️" ;;
  935. esac
  936. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  937. for chatid in ${chatids}; do
  938. # https://core.telegram.org/bots/api#sendmessage
  939. httpcode=$(docurl ${disableNotification} \
  940. --data-urlencode "parse_mode=HTML" \
  941. --data-urlencode "disable_web_page_preview=true" \
  942. --data-urlencode "text=${emoji} ${message}" \
  943. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  944. if [ "${httpcode}" = "200" ]; then
  945. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  946. sent=$((sent + 1))
  947. elif [ "${httpcode}" = "401" ]; then
  948. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  949. else
  950. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  951. fi
  952. done
  953. [ ${sent} -gt 0 ] && return 0
  954. fi
  955. return 1
  956. }
  957. # -----------------------------------------------------------------------------
  958. # Microsoft Team sender
  959. send_msteam() {
  960. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  961. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  962. case "${status}" in
  963. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}" ;;
  964. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}" ;;
  965. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}" ;;
  966. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}" ;;
  967. esac
  968. for channel in ${channels}; do
  969. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  970. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  971. payload="$(
  972. cat <<EOF
  973. {
  974. "@context": "http://schema.org/extensions",
  975. "@type": "MessageCard",
  976. "themeColor": "${color}",
  977. "title": "$icon Alert ${status} from netdata for ${host}",
  978. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  979. "potentialAction": [
  980. {
  981. "@type": "OpenUri",
  982. "name": "Netdata",
  983. "targets": [
  984. { "os": "default", "uri": "${goto_url}" }
  985. ]
  986. }
  987. ]
  988. }
  989. EOF
  990. )"
  991. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  992. webhook="${webhook//CHANNEL/${channel}}"
  993. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  994. if [ "${httpcode}" = "200" ]; then
  995. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  996. sent=$((sent + 1))
  997. else
  998. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  999. fi
  1000. done
  1001. [ ${sent} -gt 0 ] && return 0
  1002. return 1
  1003. }
  1004. # slack sender
  1005. send_slack() {
  1006. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1007. [ "${SEND_SLACK}" != "YES" ] && return 1
  1008. case "${status}" in
  1009. WARNING) color="warning" ;;
  1010. CRITICAL) color="danger" ;;
  1011. CLEAR) color="good" ;;
  1012. *) color="#777777" ;;
  1013. esac
  1014. for channel in ${channels}; do
  1015. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1016. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1017. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1018. if [ "${channel}" = "#" ]; then
  1019. ch=""
  1020. chstr="without specifying a channel"
  1021. else
  1022. ch="\"channel\": \"${channel}\","
  1023. chstr="to '${channel}'"
  1024. fi
  1025. payload="$(
  1026. cat <<EOF
  1027. {
  1028. $ch
  1029. "username": "netdata on ${host}",
  1030. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1031. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1032. "attachments": [
  1033. {
  1034. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1035. "color": "${color}",
  1036. "title": "${alarm}",
  1037. "title_link": "${goto_url}",
  1038. "text": "${info}",
  1039. "fields": [
  1040. {
  1041. "title": "${chart}",
  1042. "short": true
  1043. },
  1044. {
  1045. "title": "${family}",
  1046. "short": true
  1047. }
  1048. ],
  1049. "thumb_url": "${image}",
  1050. "footer": "by ${host}",
  1051. "ts": ${when}
  1052. }
  1053. ]
  1054. }
  1055. EOF
  1056. )"
  1057. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1058. if [ "${httpcode}" = "200" ]; then
  1059. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1060. sent=$((sent + 1))
  1061. else
  1062. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1063. fi
  1064. done
  1065. [ ${sent} -gt 0 ] && return 0
  1066. return 1
  1067. }
  1068. # -----------------------------------------------------------------------------
  1069. # rocketchat sender
  1070. send_rocketchat() {
  1071. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1072. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1073. case "${status}" in
  1074. WARNING) color="warning" ;;
  1075. CRITICAL) color="danger" ;;
  1076. CLEAR) color="good" ;;
  1077. *) color="#777777" ;;
  1078. esac
  1079. for channel in ${channels}; do
  1080. payload="$(
  1081. cat <<EOF
  1082. {
  1083. "channel": "#${channel}",
  1084. "alias": "netdata on ${host}",
  1085. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1086. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1087. "attachments": [
  1088. {
  1089. "color": "${color}",
  1090. "title": "${alarm}",
  1091. "title_link": "${goto_url}",
  1092. "text": "${info}",
  1093. "fields": [
  1094. {
  1095. "title": "${chart}",
  1096. "short": true,
  1097. "value": "chart"
  1098. },
  1099. {
  1100. "title": "${family}",
  1101. "short": true,
  1102. "value": "family"
  1103. }
  1104. ],
  1105. "thumb_url": "${image}",
  1106. "ts": "${when}"
  1107. }
  1108. ]
  1109. }
  1110. EOF
  1111. )"
  1112. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1113. if [ "${httpcode}" = "200" ]; then
  1114. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1115. sent=$((sent + 1))
  1116. else
  1117. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1118. fi
  1119. done
  1120. [ ${sent} -gt 0 ] && return 0
  1121. return 1
  1122. }
  1123. # -----------------------------------------------------------------------------
  1124. # alerta sender
  1125. send_alerta() {
  1126. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1127. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1128. case "${status}" in
  1129. CRITICAL) severity="critical" ;;
  1130. WARNING) severity="warning" ;;
  1131. CLEAR) severity="cleared" ;;
  1132. *) severity="indeterminate" ;;
  1133. esac
  1134. if [[ ${chart} == httpcheck* ]]; then
  1135. resource=$chart
  1136. event=$name
  1137. else
  1138. resource="${host}:${family}"
  1139. event="${chart}.${name}"
  1140. fi
  1141. for channel in ${channels}; do
  1142. payload="$(
  1143. cat <<EOF
  1144. {
  1145. "resource": "${resource}",
  1146. "event": "${event}",
  1147. "environment": "${channel}",
  1148. "severity": "${severity}",
  1149. "service": ["Netdata"],
  1150. "group": "Performance",
  1151. "value": "${value_string}",
  1152. "text": "${info}",
  1153. "tags": ["alarm_id:${alarm_id}"],
  1154. "attributes": {
  1155. "roles": "${roles}",
  1156. "name": "${name}",
  1157. "chart": "${chart}",
  1158. "family": "${family}",
  1159. "source": "${src}",
  1160. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1161. },
  1162. "origin": "netdata/${host}",
  1163. "type": "netdataAlarm",
  1164. "rawData": "${BASH_ARGV[@]}"
  1165. }
  1166. EOF
  1167. )"
  1168. if [ -n "${ALERTA_API_KEY}" ]; then
  1169. auth="Key ${ALERTA_API_KEY}"
  1170. fi
  1171. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1172. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1173. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1174. sent=$((sent + 1))
  1175. elif [ "${httpcode}" = "202" ]; then
  1176. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1177. else
  1178. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1179. fi
  1180. done
  1181. [ ${sent} -gt 0 ] && return 0
  1182. return 1
  1183. }
  1184. # -----------------------------------------------------------------------------
  1185. # flock sender
  1186. send_flock() {
  1187. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1188. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1189. case "${status}" in
  1190. WARNING) color="warning" ;;
  1191. CRITICAL) color="danger" ;;
  1192. CLEAR) color="good" ;;
  1193. *) color="#777777" ;;
  1194. esac
  1195. for channel in ${channels}; do
  1196. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1197. \"sendAs\": {
  1198. \"name\" : \"netdata on ${host}\",
  1199. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1200. },
  1201. \"text\": \"${host} *${status_message}*\",
  1202. \"timestamp\": \"${when}\",
  1203. \"attachments\": [
  1204. {
  1205. \"description\": \"${chart} (${family}) - ${info}\",
  1206. \"color\": \"${color}\",
  1207. \"title\": \"${alarm}\",
  1208. \"url\": \"${goto_url}\",
  1209. \"text\": \"${info}\",
  1210. \"views\": {
  1211. \"image\": {
  1212. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1213. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1214. \"filename\": \"${image}\"
  1215. }
  1216. }
  1217. }
  1218. ]
  1219. }")
  1220. if [ "${httpcode}" = "200" ]; then
  1221. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1222. sent=$((sent + 1))
  1223. else
  1224. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1225. fi
  1226. done
  1227. [ ${sent} -gt 0 ] && return 0
  1228. return 1
  1229. }
  1230. # -----------------------------------------------------------------------------
  1231. # discord sender
  1232. send_discord() {
  1233. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1234. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1235. case "${status}" in
  1236. WARNING) color="warning" ;;
  1237. CRITICAL) color="danger" ;;
  1238. CLEAR) color="good" ;;
  1239. *) color="#777777" ;;
  1240. esac
  1241. for channel in ${channels}; do
  1242. username="netdata on ${host}"
  1243. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1244. payload="$(
  1245. cat <<EOF
  1246. {
  1247. "channel": "#${channel}",
  1248. "username": "${username}",
  1249. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1250. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1251. "attachments": [
  1252. {
  1253. "color": "${color}",
  1254. "title": "${alarm}",
  1255. "title_link": "${goto_url}",
  1256. "text": "${info}",
  1257. "fields": [
  1258. {
  1259. "title": "${chart}",
  1260. "value": "${family}"
  1261. }
  1262. ],
  1263. "thumb_url": "${image}",
  1264. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1265. "footer": "${host}",
  1266. "ts": ${when}
  1267. }
  1268. ]
  1269. }
  1270. EOF
  1271. )"
  1272. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1273. if [ "${httpcode}" = "200" ]; then
  1274. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1275. sent=$((sent + 1))
  1276. else
  1277. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1278. fi
  1279. done
  1280. [ ${sent} -gt 0 ] && return 0
  1281. return 1
  1282. }
  1283. # -----------------------------------------------------------------------------
  1284. # fleep sender
  1285. send_fleep() {
  1286. local httpcode sent=0 webhooks="${1}" data message
  1287. if [ "${SEND_FLEEP}" = "YES" ]; then
  1288. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1289. for hook in ${webhooks}; do
  1290. data="{ "
  1291. data="${data} 'message': '${message}', "
  1292. data="${data} 'user': '${FLEEP_SENDER}' "
  1293. data="${data} }"
  1294. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1295. if [ "${httpcode}" = "200" ]; then
  1296. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1297. sent=$((sent + 1))
  1298. else
  1299. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1300. fi
  1301. done
  1302. [ ${sent} -gt 0 ] && return 0
  1303. fi
  1304. return 1
  1305. }
  1306. # -----------------------------------------------------------------------------
  1307. # Prowl sender
  1308. send_prowl() {
  1309. local httpcode sent=0 data message keys prio=0 alarm_url event
  1310. if [ "${SEND_PROWL}" = "YES" ]; then
  1311. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1312. message="description=${message}"
  1313. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1314. keys="apikey=${keys}"
  1315. app="application=Netdata"
  1316. case "${status}" in
  1317. CRITICAL)
  1318. prio=2
  1319. ;;
  1320. WARNING)
  1321. prio=1
  1322. ;;
  1323. esac
  1324. prio="priority=${prio}"
  1325. alarm_url="$(urlencode ${goto_url})"
  1326. alarm_url="url=${alarm_url}"
  1327. event="$(urlencode "${host} ${status_message}")"
  1328. event="event=${event}"
  1329. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1330. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1331. if [ "${httpcode}" = "200" ]; then
  1332. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1333. sent=1
  1334. else
  1335. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1336. fi
  1337. [ ${sent} -gt 0 ] && return 0
  1338. fi
  1339. return 1
  1340. }
  1341. # -----------------------------------------------------------------------------
  1342. # irc sender
  1343. send_irc() {
  1344. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1345. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ]; then
  1346. case "${status}" in
  1347. WARNING) color="warning" ;;
  1348. CRITICAL) color="danger" ;;
  1349. CLEAR) color="good" ;;
  1350. *) color="#777777" ;;
  1351. esac
  1352. for CHANNEL in ${CHANNELS}; do
  1353. error=0
  1354. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${MESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" 6667)
  1355. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1356. for code in ${reply_codes}; do
  1357. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1358. error=1
  1359. break
  1360. fi
  1361. done
  1362. if [ "${error}" -eq 0 ]; then
  1363. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1364. sent=$((sent + 1))
  1365. else
  1366. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1367. fi
  1368. done
  1369. fi
  1370. [ ${sent} -gt 0 ] && return 0
  1371. return 1
  1372. }
  1373. # -----------------------------------------------------------------------------
  1374. # Amazon SNS sender
  1375. send_awssns() {
  1376. local targets="${1}" message='' sent=0 region=''
  1377. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1378. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1379. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1380. for target in ${targets}; do
  1381. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1382. region="$(echo ${target} | cut -f 4 -d ':')"
  1383. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1384. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1385. sent=$((sent + 1))
  1386. else
  1387. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1388. fi
  1389. done
  1390. [ ${sent} -gt 0 ] && return 0
  1391. return 1
  1392. }
  1393. # -----------------------------------------------------------------------------
  1394. # syslog sender
  1395. send_syslog() {
  1396. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1397. local priority='' message='' host='' port='' prefix=''
  1398. local temp1='' temp2=''
  1399. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1400. if [ "${status}" = "CRITICAL" ]; then
  1401. level='crit'
  1402. elif [ "${status}" = "WARNING" ]; then
  1403. level='warning'
  1404. fi
  1405. for target in ${targets}; do
  1406. priority="${facility}.${level}"
  1407. message=''
  1408. host=''
  1409. port=''
  1410. prefix=''
  1411. temp1=''
  1412. temp2=''
  1413. prefix=$(echo ${target} | cut -d '/' -f 2)
  1414. temp1=$(echo ${target} | cut -d '/' -f 1)
  1415. if [ ${prefix} != ${temp1} ]; then
  1416. if (echo ${temp1} | grep -q '@'); then
  1417. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1418. host=$(echo ${temp1} | cut -d '@' -f 2)
  1419. if [ ${temp2} != ${host} ]; then
  1420. priority=${temp2}
  1421. fi
  1422. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1423. if (echo ${host} | grep -E -q '\[.*\]'); then
  1424. if (echo ${port} | grep -q ']'); then
  1425. port=''
  1426. else
  1427. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1428. fi
  1429. else
  1430. if [ ${port} = ${host} ]; then
  1431. port=''
  1432. else
  1433. host=$(echo ${host} | cut -d ':' -f 1)
  1434. fi
  1435. fi
  1436. else
  1437. priority=${temp1}
  1438. fi
  1439. fi
  1440. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1441. if [ ${host} ]; then
  1442. logger_options="${logger_options} -n ${host}"
  1443. if [ ${port} ]; then
  1444. logger_options="${logger_options} -P ${port}"
  1445. fi
  1446. fi
  1447. ${logger} -p ${priority} ${logger_options} "${message}"
  1448. done
  1449. return $?
  1450. }
  1451. # -----------------------------------------------------------------------------
  1452. # SMS sender
  1453. send_sms() {
  1454. local recipients="${1}" errcode errmessage sent=0
  1455. # Human readable SMS
  1456. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1457. # limit it to 160 characters
  1458. msg="${msg:0:160}"
  1459. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1460. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1461. for phone in ${recipients}; do
  1462. errmessage=$($sendsms $phone "$msg" 2>&1)
  1463. errcode=$?
  1464. if [ ${errcode} -eq 0 ]; then
  1465. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1466. sent=$((sent + 1))
  1467. else
  1468. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1469. fi
  1470. done
  1471. [ ${sent} -gt 0 ] && return 0
  1472. fi
  1473. return 1
  1474. }
  1475. # -----------------------------------------------------------------------------
  1476. # prepare the content of the notification
  1477. # the url to send the user on click
  1478. urlencode "${args_host}" >/dev/null
  1479. url_host="${REPLY}"
  1480. urlencode "${chart}" >/dev/null
  1481. url_chart="${REPLY}"
  1482. urlencode "${family}" >/dev/null
  1483. url_family="${REPLY}"
  1484. urlencode "${name}" >/dev/null
  1485. url_name="${REPLY}"
  1486. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
  1487. GOTOCLOUD=0
  1488. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1489. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1490. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1491. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1492. fi
  1493. fi
  1494. if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1495. GOTOCLOUD=1
  1496. fi
  1497. fi
  1498. if [ ${GOTOCLOUD} -eq 0 ]; then
  1499. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  1500. else
  1501. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  1502. fi
  1503. # the severity of the alarm
  1504. severity="${status}"
  1505. # the time the alarm was raised
  1506. duration4human ${duration} >/dev/null
  1507. duration_txt="${REPLY}"
  1508. duration4human ${non_clear_duration} >/dev/null
  1509. non_clear_duration_txt="${REPLY}"
  1510. raised_for="(was ${old_status,,} for ${duration_txt})"
  1511. # the key status message
  1512. status_message="status unknown"
  1513. # the color of the alarm
  1514. color="grey"
  1515. # the alarm value
  1516. alarm="${name//_/ } = ${value_string}"
  1517. # the image of the alarm
  1518. image="${images_base_url}/images/banner-icon-144x144.png"
  1519. # prepare the title based on status
  1520. case "${status}" in
  1521. CRITICAL)
  1522. image="${images_base_url}/images/alert-128-red.png"
  1523. status_message="is critical"
  1524. color="#ca414b"
  1525. ;;
  1526. WARNING)
  1527. image="${images_base_url}/images/alert-128-orange.png"
  1528. status_message="needs attention"
  1529. color="#ffc107"
  1530. ;;
  1531. CLEAR)
  1532. image="${images_base_url}/images/check-mark-2-128-green.png"
  1533. status_message="recovered"
  1534. color="#77ca6d"
  1535. ;;
  1536. esac
  1537. if [ "${status}" = "CLEAR" ]; then
  1538. severity="Recovered from ${old_status}"
  1539. if [ ${non_clear_duration} -gt ${duration} ]; then
  1540. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1541. fi
  1542. # don't show the value when the status is CLEAR
  1543. # for certain alarms, this value might not have any meaning
  1544. alarm="${name//_/ } ${raised_for}"
  1545. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  1546. severity="Escalated to ${status}"
  1547. if [ ${non_clear_duration} -gt ${duration} ]; then
  1548. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1549. fi
  1550. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  1551. severity="Demoted to ${status}"
  1552. if [ ${non_clear_duration} -gt ${duration} ]; then
  1553. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1554. fi
  1555. else
  1556. raised_for=
  1557. fi
  1558. # prepare HTML versions of elements
  1559. info_html=
  1560. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  1561. raised_for_html=
  1562. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1563. # -----------------------------------------------------------------------------
  1564. # send the slack notification
  1565. # slack aggregates posts from the same username
  1566. # so we use "${host} ${status}" as the bot username, to make them diff
  1567. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1568. SENT_SLACK=$?
  1569. # -----------------------------------------------------------------------------
  1570. # send the Microsoft notification
  1571. # Microsoft team aggregates posts from the same username
  1572. # so we use "${host} ${status}" as the bot username, to make them diff
  1573. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1574. SENT_MSTEAM=$?
  1575. # -----------------------------------------------------------------------------
  1576. # send the rocketchat notification
  1577. # rocketchat aggregates posts from the same username
  1578. # so we use "${host} ${status}" as the bot username, to make them diff
  1579. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1580. SENT_ROCKETCHAT=$?
  1581. # -----------------------------------------------------------------------------
  1582. # send the alerta notification
  1583. # alerta aggregates posts from the same username
  1584. # so we use "${host} ${status}" as the bot username, to make them diff
  1585. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1586. SENT_ALERTA=$?
  1587. # -----------------------------------------------------------------------------
  1588. # send the flock notification
  1589. # flock aggregates posts from the same username
  1590. # so we use "${host} ${status}" as the bot username, to make them diff
  1591. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1592. SENT_FLOCK=$?
  1593. # -----------------------------------------------------------------------------
  1594. # send the discord notification
  1595. # discord aggregates posts from the same username
  1596. # so we use "${host} ${status}" as the bot username, to make them diff
  1597. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1598. SENT_DISCORD=$?
  1599. # -----------------------------------------------------------------------------
  1600. # send the pushover notification
  1601. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1602. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1603. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1604. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1605. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1606. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1607. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1608. <small><small>The source of this alarm is line ${src}</small></small>
  1609. "
  1610. SENT_PUSHOVER=$?
  1611. # -----------------------------------------------------------------------------
  1612. # send the pushbullet notification
  1613. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  1614. Severity: ${severity}\\n
  1615. Chart: ${chart}\\n
  1616. Family: ${family}\\n
  1617. ${date}\\n
  1618. The source of this alarm is line ${src}"
  1619. SENT_PUSHBULLET=$?
  1620. # -----------------------------------------------------------------------------
  1621. # send the twilio SMS
  1622. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1623. Severity: ${severity}
  1624. Chart: ${chart}
  1625. Family: ${family}
  1626. ${info}"
  1627. SENT_TWILIO=$?
  1628. # -----------------------------------------------------------------------------
  1629. # send the messagebird SMS
  1630. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1631. Severity: ${severity}
  1632. Chart: ${chart}
  1633. Family: ${family}
  1634. ${info}"
  1635. SENT_MESSAGEBIRD=$?
  1636. # -----------------------------------------------------------------------------
  1637. # send the kavenegar SMS
  1638. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1639. Severity: ${severity}
  1640. Chart: ${chart}
  1641. Family: ${family}
  1642. ${info}"
  1643. SENT_KAVENEGAR=$?
  1644. # -----------------------------------------------------------------------------
  1645. # send the telegram.org message
  1646. # https://core.telegram.org/bots/api#formatting-options
  1647. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1648. ${chart} (${family})
  1649. <a href=\"${goto_url}\">${alarm}</a>
  1650. <i>${info}</i>"
  1651. SENT_TELEGRAM=$?
  1652. # -----------------------------------------------------------------------------
  1653. # send the kafka message
  1654. send_kafka
  1655. SENT_KAFKA=$?
  1656. # -----------------------------------------------------------------------------
  1657. # send the pagerduty.com message
  1658. send_pd "${to_pd}"
  1659. SENT_PD=$?
  1660. # -----------------------------------------------------------------------------
  1661. # send the fleep message
  1662. send_fleep "${to_fleep}"
  1663. SENT_FLEEP=$?
  1664. # -----------------------------------------------------------------------------
  1665. # send the Prowl message
  1666. send_prowl "${to_prowl}"
  1667. SENT_PROWL=$?
  1668. # -----------------------------------------------------------------------------
  1669. # send the irc message
  1670. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1671. Severity: ${severity}
  1672. Chart: ${chart}
  1673. Family: ${family}
  1674. ${info}"
  1675. SENT_IRC=$?
  1676. # -----------------------------------------------------------------------------
  1677. # send the SMS message with smstools3
  1678. send_sms "${to_sms}"
  1679. SENT_SMS=$?
  1680. # -----------------------------------------------------------------------------
  1681. # send the custom message
  1682. send_custom() {
  1683. # is it enabled?
  1684. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1685. # do we have any sender?
  1686. [ -z "${1}" ] && return 1
  1687. # call the custom_sender function
  1688. custom_sender "${@}"
  1689. }
  1690. send_custom "${to_custom}"
  1691. SENT_CUSTOM=$?
  1692. # -----------------------------------------------------------------------------
  1693. # send hipchat message
  1694. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1695. ${host} ${status_message}<br/> \
  1696. <b>${alarm}</b> ${info_html}<br/> \
  1697. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1698. <b>${date}${raised_for_html}</b><br/> \
  1699. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1700. (source of alarm ${src}) \
  1701. "
  1702. SENT_HIPCHAT=$?
  1703. # -----------------------------------------------------------------------------
  1704. # send the Amazon SNS message
  1705. send_awssns "${to_awssns}"
  1706. SENT_AWSSNS=$?
  1707. # -----------------------------------------------------------------------------
  1708. # send the syslog message
  1709. send_syslog "${to_syslog}"
  1710. SENT_SYSLOG=$?
  1711. # -----------------------------------------------------------------------------
  1712. # send the email
  1713. send_email <<EOF
  1714. To: ${to_email}
  1715. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1716. MIME-Version: 1.0
  1717. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1718. ${email_thread_headers}
  1719. This is a MIME-encoded multipart message
  1720. --multipart-boundary
  1721. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1722. Content-Disposition: inline
  1723. Content-Transfer-Encoding: 8bit
  1724. ${host} ${status_message}
  1725. ${alarm} ${info}
  1726. ${raised_for}
  1727. Chart : ${chart}
  1728. Family : ${family}
  1729. Severity: ${severity}
  1730. URL : ${goto_url}
  1731. Source : ${src}
  1732. Date : ${date}
  1733. Notification generated on ${host}
  1734. Evaluated Expression : ${calc_expression}
  1735. Expression Variables : ${calc_param_values}
  1736. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1737. --multipart-boundary
  1738. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1739. Content-Disposition: inline
  1740. Content-Transfer-Encoding: 8bit
  1741. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1742. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1743. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1744. <table>
  1745. <tbody>
  1746. <tr>
  1747. <td style="vertical-align: top;" valign="top"></td>
  1748. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1749. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1750. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1751. <tbody>
  1752. <tr>
  1753. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1754. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1755. </td>
  1756. </tr>
  1757. <tr>
  1758. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1759. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1760. </td>
  1761. </tr>
  1762. <tr>
  1763. <td style="vertical-align: top;" valign="top">
  1764. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1765. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  1766. <tbody>
  1767. <tr>
  1768. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  1769. <span>${chart}</span>
  1770. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  1771. </td>
  1772. </tr>
  1773. <tr style="margin: 0; padding: 0;">
  1774. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1775. <span><b>${alarm}</b>${info_html}</span>
  1776. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  1777. </td>
  1778. </tr>
  1779. <tr>
  1780. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1781. <span>${family}</span>
  1782. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  1783. </td>
  1784. </tr>
  1785. <tr style="margin: 0; padding: 0;">
  1786. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1787. <span>${severity}</span>
  1788. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  1789. </td>
  1790. </tr>
  1791. <tr style="margin: 0; padding: 0;">
  1792. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  1793. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  1794. </td>
  1795. </tr>
  1796. <tr style="margin: 0; padding: 0;">
  1797. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1798. <span>${calc_expression}</span>
  1799. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
  1800. </td>
  1801. </tr>
  1802. <tr style="margin: 0; padding: 0;">
  1803. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1804. <span>${calc_param_values}</span>
  1805. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
  1806. </td>
  1807. </tr>
  1808. <tr style="margin: 0; padding: 0;">
  1809. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1810. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1811. </td>
  1812. </tr>
  1813. <tr style="margin: 0; padding: 0;">
  1814. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  1815. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  1816. </td>
  1817. </tr>
  1818. <tr style="text-align: center; margin: 0; padding: 0;">
  1819. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  1820. </td>
  1821. </tr>
  1822. <tr style="text-align: center; margin: 0; padding: 0;">
  1823. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  1824. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  1825. </td>
  1826. </tr>
  1827. </tbody>
  1828. </table>
  1829. </div>
  1830. </td>
  1831. </tr>
  1832. </tbody>
  1833. </table>
  1834. </div>
  1835. </td>
  1836. </tr>
  1837. </tbody>
  1838. </table>
  1839. </body>
  1840. </html>
  1841. --multipart-boundary--
  1842. EOF
  1843. SENT_EMAIL=$?
  1844. # -----------------------------------------------------------------------------
  1845. # let netdata know
  1846. for state in "${SENT_EMAIL}" \
  1847. "${SENT_PUSHOVER}" \
  1848. "${SENT_TELEGRAM}" \
  1849. "${SENT_SLACK}" \
  1850. "${SENT_ROCKETCHAT}" \
  1851. "${SENT_ALERTA}" \
  1852. "${SENT_FLOCK}" \
  1853. "${SENT_DISCORD}" \
  1854. "${SENT_TWILIO}" \
  1855. "${SENT_HIPCHAT}" \
  1856. "${SENT_MESSAGEBIRD}" \
  1857. "${SENT_KAVENEGAR}" \
  1858. "${SENT_PUSHBULLET}" \
  1859. "${SENT_KAFKA}" \
  1860. "${SENT_PD}" \
  1861. "${SENT_FLEEP}" \
  1862. "${SENT_PROWL}" \
  1863. "${SENT_CUSTOM}" \
  1864. "${SENT_IRC}" \
  1865. "${SENT_AWSSNS}" \
  1866. "${SENT_SYSLOG}" \
  1867. "${SENT_SMS}" \
  1868. "${SENT_MSTEAM}"; do
  1869. if [ "${state}" -eq 0 ]; then
  1870. # we sent something
  1871. exit 0
  1872. fi
  1873. done
  1874. # we did not send anything
  1875. exit 1