alarm-notify.sh.in 79 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285
  1. #!/usr/bin/env bash
  2. # netdata
  3. # real-time performance and health monitoring, done right!
  4. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  5. # SPDX-License-Identifier: GPL-3.0-or-later
  6. #
  7. # Script to send alarm notifications for netdata
  8. #
  9. # Features:
  10. # - multiple notification methods
  11. # - multiple roles per alarm
  12. # - multiple recipients per role
  13. # - severity filtering per recipient
  14. #
  15. # Supported notification methods:
  16. # - emails by @ktsaou
  17. # - slack.com notifications by @ktsaou
  18. # - alerta.io notifications by @kattunga
  19. # - discordapp.com notifications by @lowfive
  20. # - pushover.net notifications by @ktsaou
  21. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  22. # - telegram.org notifications by @hashworks #1002
  23. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  24. # - kafka notifications by @ktsaou #1342
  25. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  26. # - messagebird.com notifications by @tech_no_logical #1453
  27. # - hipchat notifications by @ktsaou #1561
  28. # - fleep notifications by @Ferroin
  29. # - prowlapp.com notifications by @Ferroin
  30. # - custom notifications by @ktsaou
  31. # - syslog messages by @Ferroin
  32. # - Microsoft Team notification by @tioumen
  33. # -----------------------------------------------------------------------------
  34. # testing notifications
  35. if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
  36. then
  37. if [ "${2}" = "test" ]
  38. then
  39. recipient="${1}"
  40. else
  41. recipient="${2}"
  42. fi
  43. [ -z "${recipient}" ] && recipient="sysadmin"
  44. id=1
  45. last="CLEAR"
  46. test_res=0
  47. for x in "WARNING" "CRITICAL" "CLEAR"
  48. do
  49. echo >&2
  50. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  51. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  52. if [ $? -ne 0 ]
  53. then
  54. echo >&2 "# FAILED"
  55. test_res=1
  56. else
  57. echo >&2 "# OK"
  58. fi
  59. last="${x}"
  60. id=$((id + 1))
  61. done
  62. exit $test_res
  63. fi
  64. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  65. export LC_ALL=C
  66. # -----------------------------------------------------------------------------
  67. PROGRAM_NAME="$(basename "${0}")"
  68. logdate() {
  69. date "+%Y-%m-%d %H:%M:%S"
  70. }
  71. log() {
  72. local status="${1}"
  73. shift
  74. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  75. }
  76. warning() {
  77. log WARNING "${@}"
  78. }
  79. error() {
  80. log ERROR "${@}"
  81. }
  82. info() {
  83. log INFO "${@}"
  84. }
  85. fatal() {
  86. log FATAL "${@}"
  87. exit 1
  88. }
  89. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  90. debug() {
  91. [ "${debug}" = "1" ] && log DEBUG "${@}"
  92. }
  93. docurl() {
  94. if [ -z "${curl}" ]
  95. then
  96. error "\${curl} is unset."
  97. return 1
  98. fi
  99. if [ "${debug}" = "1" ]
  100. then
  101. echo >&2 "--- BEGIN curl command ---"
  102. printf >&2 "%q " ${curl} "${@}"
  103. echo >&2
  104. echo >&2 "--- END curl command ---"
  105. local out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  106. local code=$(${curl} ${curl_options} --write-out %{http_code} --output "${out}" --silent --show-error "${@}")
  107. local ret=$?
  108. echo >&2 "--- BEGIN received response ---"
  109. cat >&2 "${out}"
  110. echo >&2
  111. echo >&2 "--- END received response ---"
  112. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  113. rm "${out}"
  114. echo "${code}"
  115. return ${ret}
  116. fi
  117. ${curl} ${curl_options} --write-out %{http_code} --output /dev/null --silent --show-error "${@}"
  118. return $?
  119. }
  120. # -----------------------------------------------------------------------------
  121. # List of all the notification mechanisms we support.
  122. # Used in a couple of places to write more compact code.
  123. method_names="
  124. email
  125. pushover
  126. pushbullet
  127. telegram
  128. slack
  129. alerta
  130. flock
  131. discord
  132. hipchat
  133. twilio
  134. messagebird
  135. pd
  136. fleep
  137. syslog
  138. custom
  139. msteam
  140. kavenegar
  141. prowl
  142. "
  143. # -----------------------------------------------------------------------------
  144. # this is to be overwritten by the config file
  145. custom_sender() {
  146. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  147. }
  148. # -----------------------------------------------------------------------------
  149. # check for BASH v4+ (required for associative arrays)
  150. [ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
  151. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  152. # -----------------------------------------------------------------------------
  153. # defaults to allow running this script by hand
  154. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  155. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  156. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  157. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  158. # -----------------------------------------------------------------------------
  159. # parse command line parameters
  160. if [ ${1} = "unittest" ] ; then
  161. unittest=1 # enable unit testing mode
  162. roles="${2}" # the role that should be used for unit testing
  163. cfgfile="${3}" # the location of the config file to use for unit testing
  164. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  165. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  166. else
  167. roles="${1}" # the roles that should be notified for this event
  168. args_host="${2}" # the host generated this event
  169. unique_id="${3}" # the unique id of this event
  170. alarm_id="${4}" # the unique id of the alarm that generated this event
  171. event_id="${5}" # the incremental id of the event, for this alarm id
  172. when="${6}" # the timestamp this event occurred
  173. name="${7}" # the name of the alarm, as given in netdata health.d entries
  174. chart="${8}" # the name of the chart (type.id)
  175. family="${9}" # the family of the chart
  176. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  177. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  178. value="${12}" # the current value of the alarm
  179. old_value="${13}" # the previous value of the alarm
  180. src="${14}" # the line number and file the alarm has been configured
  181. duration="${15}" # the duration in seconds of the previous alarm state
  182. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  183. units="${17}" # the units of the value
  184. info="${18}" # a short description of the alarm
  185. value_string="${19}" # friendly value (with units)
  186. old_value_string="${20}" # friendly old value (with units)
  187. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  188. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  189. total_warnings="${23}" # Total number of alarms in WARNING state
  190. total_critical="${24}" # Total number of alarms in CRITICAL state
  191. fi
  192. # -----------------------------------------------------------------------------
  193. # find a suitable hostname to use, if netdata did not supply a hostname
  194. if [ -z ${args_host} ]
  195. then
  196. this_host=$(hostname -s 2>/dev/null)
  197. host="${this_host}"
  198. args_host="${this_host}"
  199. else
  200. host="${args_host}"
  201. fi
  202. # -----------------------------------------------------------------------------
  203. # screen statuses we don't need to send a notification
  204. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  205. if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
  206. then
  207. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  208. exit 1
  209. fi
  210. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  211. if [ "${clear_alarm_always}" != "YES" -a "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
  212. then
  213. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  214. exit 1
  215. fi
  216. # -----------------------------------------------------------------------------
  217. # load configuration
  218. # By default fetch images from the global public registry.
  219. # This is required by default, since all notification methods need to download
  220. # images via the Internet, and private registries might not be reachable.
  221. # This can be overwritten at the configuration file.
  222. images_base_url="https://registry.my-netdata.io"
  223. # curl options to use
  224. curl_options=""
  225. # hostname handling
  226. use_fqdn="NO"
  227. # needed commands
  228. # if empty they will be searched in the system path
  229. curl=
  230. sendmail=
  231. # enable / disable features
  232. for method_name in ${method_names^^} ; do
  233. declare SEND_${method_name}="YES"
  234. declare DEFAULT_RECIPIENT_${method_name}
  235. done
  236. for method_name in ${method_names} ; do
  237. declare -A role_recipients_${method_name}
  238. done
  239. # slack configs
  240. SLACK_WEBHOOK_URL=
  241. # Microsoft Team configs
  242. MSTEAM_WEBHOOK_URL=
  243. # rocketchat configs
  244. ROCKETCHAT_WEBHOOK_URL=
  245. # alerta configs
  246. ALERTA_WEBHOOK_URL=
  247. ALERTA_API_KEY=
  248. # flock configs
  249. FLOCK_WEBHOOK_URL=
  250. # discord configs
  251. DISCORD_WEBHOOK_URL=
  252. # pushover configs
  253. PUSHOVER_APP_TOKEN=
  254. # pushbullet configs
  255. PUSHBULLET_ACCESS_TOKEN=
  256. PUSHBULLET_SOURCE_DEVICE=
  257. # twilio configs
  258. TWILIO_ACCOUNT_SID=
  259. TWILIO_ACCOUNT_TOKEN=
  260. TWILIO_NUMBER=
  261. # hipchat configs
  262. HIPCHAT_SERVER=
  263. HIPCHAT_AUTH_TOKEN=
  264. # messagebird configs
  265. MESSAGEBIRD_ACCESS_KEY=
  266. MESSAGEBIRD_NUMBER=
  267. # kavenegar configs
  268. KAVENEGAR_API_KEY=
  269. KAVENEGAR_SENDER=
  270. # telegram configs
  271. TELEGRAM_BOT_TOKEN=
  272. # kafka configs
  273. SEND_KAFKA="YES"
  274. KAFKA_URL=
  275. KAFKA_SENDER_IP=
  276. # pagerduty.com configs
  277. PD_SERVICE_KEY=
  278. # fleep.io configs
  279. FLEEP_SENDER="${host}"
  280. # Amazon SNS configs
  281. DEFAULT_RECIPIENT_AWSSNS=
  282. AWSSNS_MESSAGE_FORMAT=
  283. declare -A role_recipients_awssns=()
  284. # syslog configs
  285. SYSLOG_FACILITY=
  286. # email configs
  287. EMAIL_SENDER=
  288. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  289. EMAIL_THREADING=
  290. DEFAULT_RECIPIENT_EMAIL="root"
  291. # irc configs
  292. IRC_NICKNAME=
  293. IRC_REALNAME=
  294. IRC_NETWORK=
  295. # load the stock and user configuration files
  296. # these will overwrite the variables above
  297. if [ ${unittest} ] ;
  298. then
  299. source "${cfgfile}"
  300. [ $? -ne 0 ] && error "Failed to load requested config file." && exit 1
  301. else
  302. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"
  303. do
  304. if [ -f "${CONFIG}" ]
  305. then
  306. debug "Loading config file '${CONFIG}'..."
  307. source "${CONFIG}"
  308. [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
  309. else
  310. warning "Cannot find file '${CONFIG}'."
  311. fi
  312. done
  313. fi
  314. # If we didn't autodetect the character set for e-mail and it wasn't
  315. # set by the user, we need to set it to a reasonable default. UTF-8
  316. # should be correct for almost all modern UNIX systems.
  317. if [ -z ${EMAIL_CHARSET} ]
  318. then
  319. EMAIL_CHARSET="UTF-8"
  320. fi
  321. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  322. # unless we're sending an alarm for a slave system which we can't get the
  323. # FQDN of easily.
  324. if [ "${use_fqdn}" = "YES" -a "${host}" = "$(hostname -s 2>/dev/null)" ]
  325. then
  326. host="$(hostname -f 2>/dev/null)"
  327. fi
  328. # -----------------------------------------------------------------------------
  329. # filter a recipient based on alarm event severity
  330. filter_recipient_by_criticality() {
  331. local method="${1}" x="${2}" r s
  332. shift
  333. r="${x/|*/}" # the recipient
  334. s="${x/*|/}" # the severity required for notifying this recipient
  335. # no severity filtering for this person
  336. [ "${r}" = "${s}" ] && return 0
  337. # the severity is invalid
  338. s="${s^^}"
  339. if [ "${s}" != "CRITICAL" ]
  340. then
  341. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  342. return 0
  343. fi
  344. # create the status tracking directory for this user
  345. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
  346. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  347. case "${status}" in
  348. CRITICAL)
  349. # make sure he will get future notifications for this alarm too
  350. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  351. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  352. return 0
  353. ;;
  354. WARNING)
  355. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
  356. then
  357. # we do not remove the file, so that he will get future notifications of this alarm
  358. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  359. return 0
  360. fi
  361. ;;
  362. *)
  363. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
  364. then
  365. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  366. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  367. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  368. return 0
  369. fi
  370. ;;
  371. esac
  372. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  373. return 1
  374. }
  375. # -----------------------------------------------------------------------------
  376. # verify the delivery methods supported
  377. # check slack
  378. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  379. # check rocketchat
  380. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  381. # check alerta
  382. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  383. # check flock
  384. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  385. # check discord
  386. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  387. # check pushover
  388. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  389. # check pushbullet
  390. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  391. # check twilio
  392. [ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
  393. # check hipchat
  394. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  395. # check messagebird
  396. [ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
  397. # check kavenegar
  398. [ -z "${KAVENEGAR_API_KEY}" -o -z "${KAVENEGAR_SENDER}" ] && SEND_KAVENEGAR="NO"
  399. # check telegram
  400. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  401. # check kafka
  402. [ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
  403. # check irc
  404. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  405. # check fleep
  406. [ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO"
  407. # if we need curl, check for the curl command
  408. if [ \( \
  409. "${SEND_PUSHOVER}" = "YES" \
  410. -o "${SEND_SLACK}" = "YES" \
  411. -o "${SEND_ROCKETCHAT}" = "YES" \
  412. -o "${SEND_ALERTA}" = "YES" \
  413. -o "${SEND_PD}" = "YES" \
  414. -o "${SEND_FLOCK}" = "YES" \
  415. -o "${SEND_DISCORD}" = "YES" \
  416. -o "${SEND_HIPCHAT}" = "YES" \
  417. -o "${SEND_TWILIO}" = "YES" \
  418. -o "${SEND_MESSAGEBIRD}" = "YES" \
  419. -o "${SEND_KAVENEGAR}" = "YES" \
  420. -o "${SEND_TELEGRAM}" = "YES" \
  421. -o "${SEND_PUSHBULLET}" = "YES" \
  422. -o "${SEND_KAFKA}" = "YES" \
  423. -o "${SEND_FLEEP}" = "YES" \
  424. -o "${SEND_PROWL}" = "YES" \
  425. -o "${SEND_CUSTOM}" = "YES" \
  426. -o "${SEND_MSTEAM}" = "YES" \
  427. \) -a -z "${curl}" ]
  428. then
  429. curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
  430. if [ -z "${curl}" ]
  431. then
  432. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  433. SEND_PUSHOVER="NO"
  434. SEND_PUSHBULLET="NO"
  435. SEND_TELEGRAM="NO"
  436. SEND_SLACK="NO"
  437. SEND_MSTEAM="NO"
  438. SEND_ROCKETCHAT="NO"
  439. SEND_ALERTA="NO"
  440. SEND_PD="NO"
  441. SEND_FLOCK="NO"
  442. SEND_DISCORD="NO"
  443. SEND_TWILIO="NO"
  444. SEND_HIPCHAT="NO"
  445. SEND_MESSAGEBIRD="NO"
  446. SEND_KAVENEGAR="NO"
  447. SEND_KAFKA="NO"
  448. SEND_FLEEP="NO"
  449. SEND_PROWL="NO"
  450. SEND_CUSTOM="NO"
  451. fi
  452. fi
  453. # if we need sendmail, check for the sendmail command
  454. if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
  455. then
  456. sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
  457. if [ -z "${sendmail}" ]
  458. then
  459. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  460. SEND_EMAIL="NO"
  461. fi
  462. fi
  463. # if we need logger, check for the logger command
  464. if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
  465. then
  466. logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)"
  467. if [ -z "${logger}" ]
  468. then
  469. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  470. SEND_SYSLOG="NO"
  471. fi
  472. fi
  473. # if we need aws, check for the aws command
  474. if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ]
  475. then
  476. aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)"
  477. if [ -z "${aws}" ]
  478. then
  479. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  480. SEND_AWSSNS="NO"
  481. fi
  482. fi
  483. # -----------------------------------------------------------------------------
  484. # find the recipients' addresses per method
  485. # netdata may call us with multiple roles, and roles may have multiple but
  486. # overlapping recipients - so, here we find the unique recipients.
  487. for method_name in ${method_names} ; do
  488. send_var="SEND_${method_name^^}"
  489. if [ ${!send_var} = "NO" ] ; then
  490. continue
  491. fi
  492. declare -A arr_var=()
  493. for x in ${roles//,/ } ; do
  494. # the roles 'silent' and 'disabled' mean:
  495. # don't send a notification for this role
  496. [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
  497. role_recipients="role_recipients_${method_name}[$x]"
  498. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  499. a="${!role_recipients}"
  500. [ -z "${a}" ] && a="${!default_recipient_var}"
  501. for r in ${a//,/ } ; do
  502. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  503. done
  504. done
  505. # build the list of recipients
  506. to_var="to_${method_name}"
  507. declare to_${method_name}="${!arr_var[*]}"
  508. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  509. done
  510. # -----------------------------------------------------------------------------
  511. # handle fixup of the email recipient list.
  512. fix_to_email() {
  513. to_email=
  514. while [ ! -z "${1}" ]
  515. do
  516. [ ! -z "${to_email}" ] && to_email="${to_email}, "
  517. to_email="${to_email}${1}"
  518. shift 1
  519. done
  520. }
  521. # ${to_email} without quotes here
  522. fix_to_email ${to_email}
  523. # -----------------------------------------------------------------------------
  524. # handle output if we're running in unit test mode
  525. if [ ${unittest} ] ; then
  526. for method_name in ${method_names} ; do
  527. to_var="to_${method_name}"
  528. echo "results: ${method_name}: ${!to_var}"
  529. done
  530. exit 0
  531. fi
  532. # -----------------------------------------------------------------------------
  533. # check that we have at least a method enabled
  534. if [ "${SEND_EMAIL}" != "YES" \
  535. -a "${SEND_PUSHOVER}" != "YES" \
  536. -a "${SEND_TELEGRAM}" != "YES" \
  537. -a "${SEND_SLACK}" != "YES" \
  538. -a "${SEND_ROCKETCHAT}" != "YES" \
  539. -a "${SEND_ALERTA}" != "YES" \
  540. -a "${SEND_FLOCK}" != "YES" \
  541. -a "${SEND_DISCORD}" != "YES" \
  542. -a "${SEND_TWILIO}" != "YES" \
  543. -a "${SEND_HIPCHAT}" != "YES" \
  544. -a "${SEND_MESSAGEBIRD}" != "YES" \
  545. -a "${SEND_KAVENEGAR}" != "YES" \
  546. -a "${SEND_PUSHBULLET}" != "YES" \
  547. -a "${SEND_KAFKA}" != "YES" \
  548. -a "${SEND_PD}" != "YES" \
  549. -a "${SEND_FLEEP}" != "YES" \
  550. -a "${SEND_CUSTOM}" != "YES" \
  551. -a "${SEND_IRC}" != "YES" \
  552. -a "${SEND_AWSSNS}" != "YES" \
  553. -a "${SEND_PROWL}" != "YES" \
  554. -a "${SEND_SYSLOG}" != "YES" \
  555. -a "${SEND_MSTEAM}" != "YES" \
  556. ]
  557. then
  558. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  559. fi
  560. # -----------------------------------------------------------------------------
  561. # get the date the alarm happened
  562. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  563. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  564. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  565. [ -z "${date}" ] && date=$(date 2>/dev/null)
  566. # ----------------------------------------------------------------------------
  567. # prepare some extra headers if we've been asked to thread e-mails
  568. if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" != "NO" ] ; then
  569. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
  570. else
  571. email_thread_headers=
  572. fi
  573. # -----------------------------------------------------------------------------
  574. # function to URL encode a string
  575. urlencode() {
  576. local string="${1}" strlen encoded pos c o
  577. strlen=${#string}
  578. for (( pos=0 ; pos<strlen ; pos++ ))
  579. do
  580. c=${string:${pos}:1}
  581. case "${c}" in
  582. [-_.~a-zA-Z0-9])
  583. o="${c}"
  584. ;;
  585. *)
  586. printf -v o '%%%02x' "'${c}"
  587. ;;
  588. esac
  589. encoded+="${o}"
  590. done
  591. REPLY="${encoded}"
  592. echo "${REPLY}"
  593. }
  594. # -----------------------------------------------------------------------------
  595. # function to convert a duration in seconds, to a human readable duration
  596. # using DAYS, MINUTES, SECONDS
  597. duration4human() {
  598. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  599. d=$(( s / 86400 ))
  600. s=$(( s - (d * 86400) ))
  601. h=$(( s / 3600 ))
  602. s=$(( s - (h * 3600) ))
  603. m=$(( s / 60 ))
  604. s=$(( s - (m * 60) ))
  605. if [ ${d} -gt 0 ]
  606. then
  607. [ ${m} -ge 30 ] && h=$(( h + 1 ))
  608. [ ${d} -gt 1 ] && ds="days"
  609. [ ${h} -gt 1 ] && hs="hours"
  610. if [ ${h} -gt 0 ]
  611. then
  612. ret="${d} ${ds} and ${h} ${hs}"
  613. else
  614. ret="${d} ${ds}"
  615. fi
  616. elif [ ${h} -gt 0 ]
  617. then
  618. [ ${s} -ge 30 ] && m=$(( m + 1 ))
  619. [ ${h} -gt 1 ] && hs="hours"
  620. [ ${m} -gt 1 ] && ms="minutes"
  621. if [ ${m} -gt 0 ]
  622. then
  623. ret="${h} ${hs} and ${m} ${ms}"
  624. else
  625. ret="${h} ${hs}"
  626. fi
  627. elif [ ${m} -gt 0 ]
  628. then
  629. [ ${m} -gt 1 ] && ms="minutes"
  630. [ ${s} -gt 1 ] && ss="seconds"
  631. if [ ${s} -gt 0 ]
  632. then
  633. ret="${m} ${ms} and ${s} ${ss}"
  634. else
  635. ret="${m} ${ms}"
  636. fi
  637. else
  638. [ ${s} -gt 1 ] && ss="seconds"
  639. ret="${s} ${ss}"
  640. fi
  641. REPLY="${ret}"
  642. echo "${REPLY}"
  643. }
  644. # -----------------------------------------------------------------------------
  645. # email sender
  646. send_email() {
  647. local ret= opts=() sender_email="${EMAIL_SENDER}" sender_name=
  648. if [ "${SEND_EMAIL}" = "YES" ]
  649. then
  650. if [ ! -z "${EMAIL_SENDER}" ]
  651. then
  652. if [[ "${EMAIL_SENDER}" =~ ^\".*\"\ \<.*\>$ ]]
  653. then
  654. # the name includes double quotes
  655. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  656. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  657. elif [[ "${EMAIL_SENDER}" =~ ^\'.*\'\ \<.*\>$ ]]
  658. then
  659. # the name includes single quotes
  660. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  661. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  662. elif [[ "${EMAIL_SENDER}" =~ ^.*\ \<.*\>$ ]]
  663. then
  664. # the name does not have any quotes
  665. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  666. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  667. fi
  668. fi
  669. [ ! -z "${sender_email}" ] && opts+=(-f "${sender_email}")
  670. [ ! -z "${sender_name}" ] && opts+=(-F "${sender_name}")
  671. if [ "${debug}" = "1" ]
  672. then
  673. echo >&2 "--- BEGIN sendmail command ---"
  674. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  675. echo >&2
  676. echo >&2 "--- END sendmail command ---"
  677. fi
  678. "${sendmail}" -t "${opts[@]}"
  679. ret=$?
  680. if [ ${ret} -eq 0 ]
  681. then
  682. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  683. return 0
  684. else
  685. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  686. return 1
  687. fi
  688. fi
  689. return 1
  690. }
  691. # -----------------------------------------------------------------------------
  692. # pushover sender
  693. send_pushover() {
  694. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  695. if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
  696. then
  697. # https://pushover.net/api
  698. priority=-2
  699. case "${status}" in
  700. CLEAR) priority=-1;; # low priority: no sound or vibration
  701. WARNING) priority=0;; # normal priority: respect quiet hours
  702. CRITICAL) priority=1;; # high priority: bypass quiet hours
  703. *) priority=-2;; # lowest priority: no notification at all
  704. esac
  705. for user in ${usertokens}
  706. do
  707. httpcode=$(docurl \
  708. --form-string "token=${apptoken}" \
  709. --form-string "user=${user}" \
  710. --form-string "html=1" \
  711. --form-string "title=${title}" \
  712. --form-string "message=${message}" \
  713. --form-string "timestamp=${when}" \
  714. --form-string "url=${url}" \
  715. --form-string "url_title=Open netdata dashboard to view the alarm" \
  716. --form-string "priority=${priority}" \
  717. https://api.pushover.net/1/messages.json)
  718. if [ "${httpcode}" = "200" ]
  719. then
  720. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  721. sent=$((sent + 1))
  722. else
  723. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  724. fi
  725. done
  726. [ ${sent} -gt 0 ] && return 0
  727. fi
  728. return 1
  729. }
  730. # -----------------------------------------------------------------------------
  731. # pushbullet sender
  732. send_pushbullet() {
  733. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  734. if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  735. then
  736. #https://docs.pushbullet.com/#create-push
  737. for user in ${recipients}
  738. do
  739. httpcode=$(docurl \
  740. --header 'Access-Token: '${userapikey}'' \
  741. --header 'Content-Type: application/json' \
  742. --data-binary @<(cat <<EOF
  743. {"title": "${title}",
  744. "type": "link",
  745. "email": "${user}",
  746. "body": "$( echo -n ${message})",
  747. "url": "${url}",
  748. "source_device_iden": "${source_device}"}
  749. EOF
  750. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  751. if [ "${httpcode}" = "200" ]
  752. then
  753. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  754. sent=$((sent + 1))
  755. else
  756. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  757. fi
  758. done
  759. [ ${sent} -gt 0 ] && return 0
  760. fi
  761. return 1
  762. }
  763. # -----------------------------------------------------------------------------
  764. # kafka sender
  765. send_kafka() {
  766. local httpcode sent=0
  767. if [ "${SEND_KAFKA}" = "YES" ]
  768. then
  769. httpcode=$(docurl -X POST \
  770. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  771. "${KAFKA_URL}")
  772. if [ "${httpcode}" = "204" ]
  773. then
  774. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  775. sent=$((sent + 1))
  776. else
  777. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  778. fi
  779. [ ${sent} -gt 0 ] && return 0
  780. fi
  781. return 1
  782. }
  783. # -----------------------------------------------------------------------------
  784. # pagerduty.com sender
  785. send_pd() {
  786. local recipients="${1}" sent=0
  787. unset t
  788. case ${status} in
  789. CLEAR) t='resolve';;
  790. WARNING) t='trigger';;
  791. CRITICAL) t='trigger';;
  792. esac
  793. if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
  794. then
  795. for PD_SERVICE_KEY in ${recipients}
  796. do
  797. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  798. payload="$(cat << EOF
  799. {
  800. "service_key": "${PD_SERVICE_KEY}",
  801. "event_type": "${t}",
  802. "incident_key" : "${alarm_id}",
  803. "description": "${d}",
  804. "details": {
  805. "value_w_units": "${value_string}",
  806. "when": "${when}",
  807. "duration" : "${duration}",
  808. "roles": "${roles}",
  809. "alarm_id" : "${alarm_id}",
  810. "name" : "${name}",
  811. "chart" : "${chart}",
  812. "family" : "${family}",
  813. "status" : "${status}",
  814. "old_status" : "${old_status}",
  815. "value" : "${value}",
  816. "old_value" : "${old_value}",
  817. "src" : "${src}",
  818. "non_clear_duration" : "${non_clear_duration}",
  819. "units" : "${units}",
  820. "info" : "${info}"
  821. }
  822. }
  823. EOF
  824. )"
  825. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  826. if [ "${httpcode}" = "200" ]
  827. then
  828. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  829. sent=$((sent + 1))
  830. else
  831. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  832. fi
  833. done
  834. [ ${sent} -gt 0 ] && return 0
  835. fi
  836. return 1
  837. }
  838. # -----------------------------------------------------------------------------
  839. # twilio sender
  840. send_twilio() {
  841. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  842. if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  843. then
  844. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  845. for user in ${recipients}
  846. do
  847. httpcode=$(docurl -X POST \
  848. --data-urlencode "From=${twilionumber}" \
  849. --data-urlencode "To=${user}" \
  850. --data-urlencode "Body=${title} ${message}" \
  851. -u "${accountsid}:${accounttoken}" \
  852. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  853. if [ "${httpcode}" = "201" ]
  854. then
  855. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  856. sent=$((sent + 1))
  857. else
  858. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  859. fi
  860. done
  861. [ ${sent} -gt 0 ] && return 0
  862. fi
  863. return 1
  864. }
  865. # -----------------------------------------------------------------------------
  866. # hipchat sender
  867. send_hipchat() {
  868. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
  869. # remove <small></small> from the message
  870. message="${message//<small>/}"
  871. message="${message//<\/small>/}"
  872. if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
  873. then
  874. # A label to be shown in addition to the sender's name
  875. # Valid length range: 0 - 64.
  876. sender="netdata"
  877. # Valid values: html, text.
  878. # Defaults to 'html'.
  879. msg_format="html"
  880. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  881. case "${status}" in
  882. WARNING) color="yellow" ;;
  883. CRITICAL) color="red" ;;
  884. CLEAR) color="green" ;;
  885. *) color="gray" ;;
  886. esac
  887. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  888. # Each recipient's notification preferences are taken into account.
  889. # Defaults to false.
  890. notify="true"
  891. for room in ${recipients}
  892. do
  893. httpcode=$(docurl -X POST \
  894. -H "Content-type: application/json" \
  895. -H "Authorization: Bearer ${authtoken}" \
  896. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  897. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  898. if [ "${httpcode}" = "204" ]
  899. then
  900. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  901. sent=$((sent + 1))
  902. else
  903. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  904. fi
  905. done
  906. [ ${sent} -gt 0 ] && return 0
  907. fi
  908. return 1
  909. }
  910. # -----------------------------------------------------------------------------
  911. # messagebird sender
  912. send_messagebird() {
  913. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  914. if [ "${SEND_MESSAGEBIRD}" = "YES" -a ! -z "${accesskey}" -a ! -z "${messagebirdnumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  915. then
  916. #https://developers.messagebird.com/docs/messaging
  917. for user in ${recipients}
  918. do
  919. httpcode=$(docurl -X POST \
  920. --data-urlencode "originator=${messagebirdnumber}" \
  921. --data-urlencode "recipients=${user}" \
  922. --data-urlencode "body=${title} ${message}" \
  923. --data-urlencode "datacoding=auto" \
  924. -H "Authorization: AccessKey ${accesskey}" \
  925. "https://rest.messagebird.com/messages")
  926. if [ "${httpcode}" = "201" ]
  927. then
  928. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  929. sent=$((sent + 1))
  930. else
  931. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  932. fi
  933. done
  934. [ ${sent} -gt 0 ] && return 0
  935. fi
  936. return 1
  937. }
  938. # -----------------------------------------------------------------------------
  939. # kavenegar sender
  940. send_kavenegar() {
  941. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  942. if [ "${SEND_KAVENEGAR}" = "YES" -a ! -z "${API_KEY}" -a ! -z "${kavenegarsender}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
  943. then
  944. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  945. for user in ${recipients}
  946. do
  947. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  948. --data-urlencode "sender=${kavenegarsender}" \
  949. --data-urlencode "receptor=${user}" \
  950. --data-urlencode "message=${title} ${message}")
  951. if [ "${httpcode}" = "200" ]
  952. then
  953. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  954. sent=$((sent + 1))
  955. else
  956. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  957. fi
  958. done
  959. [ ${sent} -gt 0 ] && return 0
  960. fi
  961. return 1
  962. }
  963. # -----------------------------------------------------------------------------
  964. # telegram sender
  965. send_telegram() {
  966. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  967. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  968. case "${status}" in
  969. WARNING) emoji="⚠️" ;;
  970. CRITICAL) emoji="🔴" ;;
  971. CLEAR) emoji="✅" ;;
  972. *) emoji="⚪️" ;;
  973. esac
  974. if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
  975. then
  976. for chatid in ${chatids}
  977. do
  978. # https://core.telegram.org/bots/api#sendmessage
  979. httpcode=$(docurl ${disableNotification} \
  980. --data-urlencode "parse_mode=HTML" \
  981. --data-urlencode "disable_web_page_preview=true" \
  982. --data-urlencode "text=${emoji} ${message}" \
  983. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  984. if [ "${httpcode}" = "200" ]
  985. then
  986. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  987. sent=$((sent + 1))
  988. elif [ "${httpcode}" = "401" ]
  989. then
  990. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  991. else
  992. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  993. fi
  994. done
  995. [ ${sent} -gt 0 ] && return 0
  996. fi
  997. return 1
  998. }
  999. # -----------------------------------------------------------------------------
  1000. # Microsoft Team sender
  1001. send_msteam() {
  1002. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1003. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  1004. case "${status}" in
  1005. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}";;
  1006. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}";;
  1007. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}";;
  1008. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}";;
  1009. esac
  1010. for channel in ${channels}
  1011. do
  1012. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  1013. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  1014. payload="$(cat <<EOF
  1015. {
  1016. "@context": "http://schema.org/extensions",
  1017. "@type": "MessageCard",
  1018. "themeColor": "${color}",
  1019. "title": "$icon Alert ${status} from netdata for ${host}",
  1020. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  1021. "potentialAction": [
  1022. {
  1023. "@type": "OpenUri",
  1024. "name": "Netdata",
  1025. "targets": [
  1026. { "os": "default", "uri": "${goto_url}" }
  1027. ]
  1028. }
  1029. ]
  1030. }
  1031. EOF
  1032. )"
  1033. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1034. webhook="${webhook//CHANNEL/${channel}}"
  1035. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  1036. if [ "${httpcode}" = "200" ]
  1037. then
  1038. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  1039. sent=$((sent + 1))
  1040. else
  1041. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  1042. fi
  1043. done
  1044. [ ${sent} -gt 0 ] && return 0
  1045. return 1
  1046. }
  1047. # slack sender
  1048. send_slack() {
  1049. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1050. [ "${SEND_SLACK}" != "YES" ] && return 1
  1051. case "${status}" in
  1052. WARNING) color="warning" ;;
  1053. CRITICAL) color="danger" ;;
  1054. CLEAR) color="good" ;;
  1055. *) color="#777777" ;;
  1056. esac
  1057. for channel in ${channels}
  1058. do
  1059. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1060. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ] ; then channel="#$channel"; fi
  1061. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1062. if [ "${channel}" = "#" ] ; then
  1063. ch=""
  1064. chstr="without specifying a channel"
  1065. else
  1066. ch="\"channel\": \"${channel}\","
  1067. chstr="to '${channel}'"
  1068. fi
  1069. payload="$(cat <<EOF
  1070. {
  1071. $ch
  1072. "username": "netdata on ${host}",
  1073. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1074. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1075. "attachments": [
  1076. {
  1077. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1078. "color": "${color}",
  1079. "title": "${alarm}",
  1080. "title_link": "${goto_url}",
  1081. "text": "${info}",
  1082. "fields": [
  1083. {
  1084. "title": "${chart}",
  1085. "short": true
  1086. },
  1087. {
  1088. "title": "${family}",
  1089. "short": true
  1090. }
  1091. ],
  1092. "thumb_url": "${image}",
  1093. "footer": "by <${goto_url}|${host}>",
  1094. "ts": ${when}
  1095. }
  1096. ]
  1097. }
  1098. EOF
  1099. )"
  1100. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1101. if [ "${httpcode}" = "200" ]
  1102. then
  1103. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1104. sent=$((sent + 1))
  1105. else
  1106. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1107. fi
  1108. done
  1109. [ ${sent} -gt 0 ] && return 0
  1110. return 1
  1111. }
  1112. # -----------------------------------------------------------------------------
  1113. # rocketchat sender
  1114. send_rocketchat() {
  1115. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1116. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1117. case "${status}" in
  1118. WARNING) color="warning" ;;
  1119. CRITICAL) color="danger" ;;
  1120. CLEAR) color="good" ;;
  1121. *) color="#777777" ;;
  1122. esac
  1123. for channel in ${channels}
  1124. do
  1125. payload="$(cat <<EOF
  1126. {
  1127. "channel": "#${channel}",
  1128. "alias": "netdata on ${host}",
  1129. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1130. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1131. "attachments": [
  1132. {
  1133. "color": "${color}",
  1134. "title": "${alarm}",
  1135. "title_link": "${goto_url}",
  1136. "text": "${info}",
  1137. "fields": [
  1138. {
  1139. "title": "${chart}",
  1140. "short": true,
  1141. "value": "chart"
  1142. },
  1143. {
  1144. "title": "${family}",
  1145. "short": true,
  1146. "value": "family"
  1147. }
  1148. ],
  1149. "thumb_url": "${image}",
  1150. "ts": "${when}"
  1151. }
  1152. ]
  1153. }
  1154. EOF
  1155. )"
  1156. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1157. if [ "${httpcode}" = "200" ]
  1158. then
  1159. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1160. sent=$((sent + 1))
  1161. else
  1162. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1163. fi
  1164. done
  1165. [ ${sent} -gt 0 ] && return 0
  1166. return 1
  1167. }
  1168. # -----------------------------------------------------------------------------
  1169. # alerta sender
  1170. send_alerta() {
  1171. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1172. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1173. case "${status}" in
  1174. CRITICAL) severity="critical" ;;
  1175. WARNING) severity="warning" ;;
  1176. CLEAR) severity="cleared" ;;
  1177. *) severity="indeterminate" ;;
  1178. esac
  1179. if [[ "${chart}" == httpcheck* ]]
  1180. then
  1181. resource=$chart
  1182. event=$name
  1183. else
  1184. resource="${host}:${family}"
  1185. event="${chart}.${name}"
  1186. fi
  1187. for channel in ${channels}
  1188. do
  1189. payload="$(cat <<EOF
  1190. {
  1191. "resource": "${resource}",
  1192. "event": "${event}",
  1193. "environment": "${channel}",
  1194. "severity": "${severity}",
  1195. "service": ["Netdata"],
  1196. "group": "Performance",
  1197. "value": "${value_string}",
  1198. "text": "${info}",
  1199. "tags": ["alarm_id:${alarm_id}"],
  1200. "attributes": {
  1201. "roles": "${roles}",
  1202. "name": "${name}",
  1203. "chart": "${chart}",
  1204. "family": "${family}",
  1205. "source": "${src}",
  1206. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1207. },
  1208. "origin": "netdata/${host}",
  1209. "type": "netdataAlarm",
  1210. "rawData": "${BASH_ARGV[@]}"
  1211. }
  1212. EOF
  1213. )"
  1214. if [[ -n "${ALERTA_API_KEY}" ]]
  1215. then
  1216. auth="Key ${ALERTA_API_KEY}"
  1217. fi
  1218. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1219. if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
  1220. then
  1221. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1222. sent=$((sent + 1))
  1223. elif [[ "${httpcode}" = "202" ]]
  1224. then
  1225. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1226. else
  1227. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1228. fi
  1229. done
  1230. [ ${sent} -gt 0 ] && return 0
  1231. return 1
  1232. }
  1233. # -----------------------------------------------------------------------------
  1234. # flock sender
  1235. send_flock() {
  1236. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1237. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1238. case "${status}" in
  1239. WARNING) color="warning" ;;
  1240. CRITICAL) color="danger" ;;
  1241. CLEAR) color="good" ;;
  1242. *) color="#777777" ;;
  1243. esac
  1244. for channel in ${channels}
  1245. do
  1246. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1247. \"sendAs\": {
  1248. \"name\" : \"netdata on ${host}\",
  1249. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1250. },
  1251. \"text\": \"${host} *${status_message}*\",
  1252. \"timestamp\": \"${when}\",
  1253. \"attachments\": [
  1254. {
  1255. \"description\": \"${chart} (${family}) - ${info}\",
  1256. \"color\": \"${color}\",
  1257. \"title\": \"${alarm}\",
  1258. \"url\": \"${goto_url}\",
  1259. \"text\": \"${info}\",
  1260. \"views\": {
  1261. \"image\": {
  1262. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1263. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1264. \"filename\": \"${image}\"
  1265. }
  1266. }
  1267. }
  1268. ]
  1269. }" )
  1270. if [ "${httpcode}" = "200" ]
  1271. then
  1272. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1273. sent=$((sent + 1))
  1274. else
  1275. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1276. fi
  1277. done
  1278. [ ${sent} -gt 0 ] && return 0
  1279. return 1
  1280. }
  1281. # -----------------------------------------------------------------------------
  1282. # discord sender
  1283. send_discord() {
  1284. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1285. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1286. case "${status}" in
  1287. WARNING) color="warning" ;;
  1288. CRITICAL) color="danger" ;;
  1289. CLEAR) color="good" ;;
  1290. *) color="#777777" ;;
  1291. esac
  1292. for channel in ${channels}
  1293. do
  1294. username="netdata on ${host}"
  1295. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1296. payload="$(cat <<EOF
  1297. {
  1298. "channel": "#${channel}",
  1299. "username": "${username}",
  1300. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1301. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1302. "attachments": [
  1303. {
  1304. "color": "${color}",
  1305. "title": "${alarm}",
  1306. "title_link": "${goto_url}",
  1307. "text": "${info}",
  1308. "fields": [
  1309. {
  1310. "title": "${chart}",
  1311. "value": "${family}"
  1312. }
  1313. ],
  1314. "thumb_url": "${image}",
  1315. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1316. "footer": "${host}",
  1317. "ts": ${when}
  1318. }
  1319. ]
  1320. }
  1321. EOF
  1322. )"
  1323. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1324. if [ "${httpcode}" = "200" ]
  1325. then
  1326. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1327. sent=$((sent + 1))
  1328. else
  1329. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1330. fi
  1331. done
  1332. [ ${sent} -gt 0 ] && return 0
  1333. return 1
  1334. }
  1335. # -----------------------------------------------------------------------------
  1336. # fleep sender
  1337. send_fleep() {
  1338. local httpcode sent=0 webhooks="${1}" data message
  1339. if [ "${SEND_FLEEP}" = "YES" ] ; then
  1340. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1341. for hook in "${webhooks}" ; do
  1342. data="{ "
  1343. data="${data} 'message': '${message}', "
  1344. data="${data} 'user': '${FLEEP_SENDER}' "
  1345. data="${data} }"
  1346. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1347. if [ "${httpcode}" = "200" ] ; then
  1348. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1349. sent=$((sent + 1))
  1350. else
  1351. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1352. fi
  1353. done
  1354. [ ${sent} -gt 0 ] && return 0
  1355. fi
  1356. return 1
  1357. }
  1358. # -----------------------------------------------------------------------------
  1359. # Prowl sender
  1360. send_prowl() {
  1361. local httpcode sent=0 data message keys prio=0 alarm_url event
  1362. if [ "${SEND_PROWL}" = "YES" ] ; then
  1363. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1364. message="description=${message}"
  1365. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1366. keys="apikey=${keys}"
  1367. app="application=Netdata"
  1368. case "${status}" in
  1369. CRITICAL)
  1370. prio=2
  1371. ;;
  1372. WARNING)
  1373. prio=1
  1374. ;;
  1375. esac
  1376. pri="priority=${pri}"
  1377. alarm_url="$(urlencode ${goto_url})"
  1378. alarm_url="url=${alarm_url}"
  1379. event="$(urlencode "${host} ${status_message}")"
  1380. event="event=${event}"
  1381. data="${keys}&${pri}&${alarm_url}&${app}&${event}&${message}"
  1382. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1383. if [ "${httpcode}" = "200" ] ; then
  1384. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1385. sent=1
  1386. else
  1387. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1388. fi
  1389. [ ${sent} -gt 0 ] && return 0
  1390. fi
  1391. return 1
  1392. }
  1393. # -----------------------------------------------------------------------------
  1394. # irc sender
  1395. send_irc() {
  1396. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1397. if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ]
  1398. then
  1399. case "${status}" in
  1400. WARNING) color="warning" ;;
  1401. CRITICAL) color="danger" ;;
  1402. CLEAR) color="good" ;;
  1403. *) color="#777777" ;;
  1404. esac
  1405. for CHANNEL in ${CHANNELS}
  1406. do
  1407. error=0
  1408. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\nNICK ${NICKNAME}\nJOIN ${CHANNEL}\nPRIVMSG ${CHANNEL} :${MESSAGE}\nQUIT\n" \ | nc ${NETWORK} 6667)
  1409. reply_codes=$(echo ${send_alarm} | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1410. for code in ${reply_codes}
  1411. do
  1412. [ "${code}" -ge 400 -a "${code}" -le 599 ] && error=1 && break
  1413. done
  1414. if [ "${error}" -eq 0 ]
  1415. then
  1416. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1417. sent=$((sent + 1))
  1418. else
  1419. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1420. fi
  1421. done
  1422. fi
  1423. [ ${sent} -gt 0 ] && return 0
  1424. return 1
  1425. }
  1426. # -----------------------------------------------------------------------------
  1427. # Amazon SNS sender
  1428. send_awssns() {
  1429. local targets="${1}" message='' sent=0 region=''
  1430. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1431. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1432. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1433. for target in ${targets} ; do
  1434. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1435. region="$(echo ${target} | cut -f 4 -d ':')"
  1436. ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null
  1437. if [ $? = 0 ]; then
  1438. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1439. sent=$((sent + 1))
  1440. else
  1441. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1442. fi
  1443. done
  1444. [ ${sent} -gt 0 ] && return 0
  1445. return 1
  1446. }
  1447. # -----------------------------------------------------------------------------
  1448. # syslog sender
  1449. send_syslog() {
  1450. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1451. local priority='' message='' host='' port='' prefix=''
  1452. local temp1='' temp2=''
  1453. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1454. if [ "${status}" = "CRITICAL" ] ; then
  1455. level='crit'
  1456. elif [ "${status}" = "WARNING" ] ; then
  1457. level='warning'
  1458. fi
  1459. for target in ${targets} ; do
  1460. priority="${facility}.${level}"
  1461. message=''
  1462. host=''
  1463. port=''
  1464. prefix=''
  1465. temp1=''
  1466. temp2=''
  1467. prefix=$(echo ${target} | cut -d '/' -f 2)
  1468. temp1=$(echo ${target} | cut -d '/' -f 1)
  1469. if [ ${prefix} != ${temp1} ] ; then
  1470. if (echo ${temp1} | grep -q '@' ) ; then
  1471. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1472. host=$(echo ${temp1} | cut -d '@' -f 2)
  1473. if [ ${temp2} != ${host} ] ; then
  1474. priority=${temp2}
  1475. fi
  1476. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1477. if ( echo ${host} | grep -E -q '\[.*\]' ) ; then
  1478. if ( echo ${port} | grep -q ']' ) ; then
  1479. port=''
  1480. else
  1481. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1482. fi
  1483. else
  1484. if [ ${port} = ${host} ] ; then
  1485. port=''
  1486. else
  1487. host=$(echo ${host} | cut -d ':' -f 1)
  1488. fi
  1489. fi
  1490. else
  1491. priority=${temp1}
  1492. fi
  1493. fi
  1494. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1495. if [ ${host} ] ; then
  1496. logger_options="${logger_options} -n ${host}"
  1497. if [ ${port} ] ; then
  1498. logger_options="${logger_options} -P ${port}"
  1499. fi
  1500. fi
  1501. ${logger} -p ${priority} ${logger_options} "${message}"
  1502. done
  1503. return $?
  1504. }
  1505. # -----------------------------------------------------------------------------
  1506. # prepare the content of the notification
  1507. # the url to send the user on click
  1508. urlencode "${args_host}" >/dev/null; url_host="${REPLY}"
  1509. urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
  1510. urlencode "${family}" >/dev/null; url_family="${REPLY}"
  1511. urlencode "${name}" >/dev/null; url_name="${REPLY}"
  1512. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
  1513. # the severity of the alarm
  1514. severity="${status}"
  1515. # the time the alarm was raised
  1516. duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
  1517. duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
  1518. raised_for="(was ${old_status,,} for ${duration_txt})"
  1519. # the key status message
  1520. status_message="status unknown"
  1521. # the color of the alarm
  1522. color="grey"
  1523. # the alarm value
  1524. alarm="${name//_/ } = ${value_string}"
  1525. # the image of the alarm
  1526. image="${images_base_url}/images/banner-icon-144x144.png"
  1527. # prepare the title based on status
  1528. case "${status}" in
  1529. CRITICAL)
  1530. image="${images_base_url}/images/alert-128-red.png"
  1531. status_message="is critical"
  1532. color="#ca414b"
  1533. ;;
  1534. WARNING)
  1535. image="${images_base_url}/images/alert-128-orange.png"
  1536. status_message="needs attention"
  1537. color="#ffc107"
  1538. ;;
  1539. CLEAR)
  1540. image="${images_base_url}/images/check-mark-2-128-green.png"
  1541. status_message="recovered"
  1542. color="#77ca6d"
  1543. ;;
  1544. esac
  1545. if [ "${status}" = "CLEAR" ]
  1546. then
  1547. severity="Recovered from ${old_status}"
  1548. if [ ${non_clear_duration} -gt ${duration} ]
  1549. then
  1550. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1551. fi
  1552. # don't show the value when the status is CLEAR
  1553. # for certain alarms, this value might not have any meaning
  1554. alarm="${name//_/ } ${raised_for}"
  1555. elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
  1556. then
  1557. severity="Escalated to ${status}"
  1558. if [ ${non_clear_duration} -gt ${duration} ]
  1559. then
  1560. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1561. fi
  1562. elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
  1563. then
  1564. severity="Demoted to ${status}"
  1565. if [ ${non_clear_duration} -gt ${duration} ]
  1566. then
  1567. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1568. fi
  1569. else
  1570. raised_for=
  1571. fi
  1572. # prepare HTML versions of elements
  1573. info_html=
  1574. [ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
  1575. raised_for_html=
  1576. [ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1577. # -----------------------------------------------------------------------------
  1578. # send the slack notification
  1579. # slack aggregates posts from the same username
  1580. # so we use "${host} ${status}" as the bot username, to make them diff
  1581. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1582. SENT_SLACK=$?
  1583. # -----------------------------------------------------------------------------
  1584. # send the Microsoft notification
  1585. # Microsoft team aggregates posts from the same username
  1586. # so we use "${host} ${status}" as the bot username, to make them diff
  1587. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1588. SENT_MSTEAM=$?
  1589. # -----------------------------------------------------------------------------
  1590. # send the rocketchat notification
  1591. # rocketchat aggregates posts from the same username
  1592. # so we use "${host} ${status}" as the bot username, to make them diff
  1593. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1594. SENT_ROCKETCHAT=$?
  1595. # -----------------------------------------------------------------------------
  1596. # send the alerta notification
  1597. # alerta aggregates posts from the same username
  1598. # so we use "${host} ${status}" as the bot username, to make them diff
  1599. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1600. SENT_ALERTA=$?
  1601. # -----------------------------------------------------------------------------
  1602. # send the flock notification
  1603. # flock aggregates posts from the same username
  1604. # so we use "${host} ${status}" as the bot username, to make them diff
  1605. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1606. SENT_FLOCK=$?
  1607. # -----------------------------------------------------------------------------
  1608. # send the discord notification
  1609. # discord aggregates posts from the same username
  1610. # so we use "${host} ${status}" as the bot username, to make them diff
  1611. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1612. SENT_DISCORD=$?
  1613. # -----------------------------------------------------------------------------
  1614. # send the pushover notification
  1615. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1616. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1617. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1618. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1619. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1620. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1621. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1622. <small><small>The source of this alarm is line ${src}</small></small>
  1623. "
  1624. SENT_PUSHOVER=$?
  1625. # -----------------------------------------------------------------------------
  1626. # send the pushbullet notification
  1627. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
  1628. Severity: ${severity}\n
  1629. Chart: ${chart}\n
  1630. Family: ${family}\n
  1631. $(date -d @${when})\n
  1632. The source of this alarm is line ${src}"
  1633. SENT_PUSHBULLET=$?
  1634. # -----------------------------------------------------------------------------
  1635. # send the twilio SMS
  1636. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1637. Severity: ${severity}
  1638. Chart: ${chart}
  1639. Family: ${family}
  1640. ${info}"
  1641. SENT_TWILIO=$?
  1642. # -----------------------------------------------------------------------------
  1643. # send the messagebird SMS
  1644. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1645. Severity: ${severity}
  1646. Chart: ${chart}
  1647. Family: ${family}
  1648. ${info}"
  1649. SENT_MESSAGEBIRD=$?
  1650. # -----------------------------------------------------------------------------
  1651. # send the kavenegar SMS
  1652. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1653. Severity: ${severity}
  1654. Chart: ${chart}
  1655. Family: ${family}
  1656. ${info}"
  1657. SENT_KAVENEGAR=$?
  1658. # -----------------------------------------------------------------------------
  1659. # send the telegram.org message
  1660. # https://core.telegram.org/bots/api#formatting-options
  1661. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1662. ${chart} (${family})
  1663. <a href=\"${goto_url}\">${alarm}</a>
  1664. <i>${info}</i>"
  1665. SENT_TELEGRAM=$?
  1666. # -----------------------------------------------------------------------------
  1667. # send the kafka message
  1668. send_kafka
  1669. SENT_KAFKA=$?
  1670. # -----------------------------------------------------------------------------
  1671. # send the pagerduty.com message
  1672. send_pd "${to_pd}"
  1673. SENT_PD=$?
  1674. # -----------------------------------------------------------------------------
  1675. # send the fleep message
  1676. send_fleep "${to_fleep}"
  1677. SENT_FLEEP=$?
  1678. # -----------------------------------------------------------------------------
  1679. # send the Prowl message
  1680. send_prowl "${to_prowl}"
  1681. SENT_PROWL=$?
  1682. # -----------------------------------------------------------------------------
  1683. # send the irc message
  1684. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1685. Severity: ${severity}
  1686. Chart: ${chart}
  1687. Family: ${family}
  1688. ${info}"
  1689. SENT_IRC=$?
  1690. # -----------------------------------------------------------------------------
  1691. # send the custom message
  1692. send_custom() {
  1693. # is it enabled?
  1694. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1695. # do we have any sender?
  1696. [ -z "${1}" ] && return 1
  1697. # call the custom_sender function
  1698. custom_sender "${@}"
  1699. }
  1700. send_custom "${to_custom}"
  1701. SENT_CUSTOM=$?
  1702. # -----------------------------------------------------------------------------
  1703. # send hipchat message
  1704. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1705. ${host} ${status_message}<br/> \
  1706. <b>${alarm}</b> ${info_html}<br/> \
  1707. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1708. <b>${date}${raised_for_html}</b><br/> \
  1709. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1710. (source of alarm ${src}) \
  1711. "
  1712. SENT_HIPCHAT=$?
  1713. # -----------------------------------------------------------------------------
  1714. # send the Amazon SNS message
  1715. send_awssns ${to_awssns}
  1716. SENT_AWSSNS=$?
  1717. # -----------------------------------------------------------------------------
  1718. # send the syslog message
  1719. send_syslog ${to_syslog}
  1720. SENT_SYSLOG=$?
  1721. # -----------------------------------------------------------------------------
  1722. # send the email
  1723. send_email <<EOF
  1724. To: ${to_email}
  1725. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1726. MIME-Version: 1.0
  1727. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1728. ${email_thread_headers}
  1729. This is a MIME-encoded multipart message
  1730. --multipart-boundary
  1731. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1732. Content-Disposition: inline
  1733. Content-Transfer-Encoding: 8bit
  1734. ${host} ${status_message}
  1735. ${alarm} ${info}
  1736. ${raised_for}
  1737. Chart : ${chart}
  1738. Family : ${family}
  1739. Severity: ${severity}
  1740. URL : ${goto_url}
  1741. Source : ${src}
  1742. Date : ${date}
  1743. Notification generated on ${host}
  1744. Evaluated Expression : ${calc_expression}
  1745. Expression Variables : ${calc_param_values}
  1746. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1747. --multipart-boundary
  1748. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1749. Content-Disposition: inline
  1750. Content-Transfer-Encoding: 8bit
  1751. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1752. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1753. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1754. <table>
  1755. <tbody>
  1756. <tr>
  1757. <td style="vertical-align: top;" valign="top"></td>
  1758. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1759. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1760. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1761. <tbody>
  1762. <tr>
  1763. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1764. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1765. </td>
  1766. </tr>
  1767. <tr>
  1768. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1769. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1770. </td>
  1771. </tr>
  1772. <tr>
  1773. <td style="vertical-align: top;" valign="top">
  1774. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1775. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  1776. <tbody>
  1777. <tr>
  1778. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  1779. <span>${chart}</span>
  1780. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  1781. </td>
  1782. </tr>
  1783. <tr style="margin: 0; padding: 0;">
  1784. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1785. <span><b>${alarm}</b>${info_html}</span>
  1786. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  1787. </td>
  1788. </tr>
  1789. <tr>
  1790. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1791. <span>${family}</span>
  1792. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  1793. </td>
  1794. </tr>
  1795. <tr style="margin: 0; padding: 0;">
  1796. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1797. <span>${severity}</span>
  1798. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  1799. </td>
  1800. </tr>
  1801. <tr style="margin: 0; padding: 0;">
  1802. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  1803. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  1804. </td>
  1805. </tr>
  1806. <tr style="margin: 0; padding: 0;">
  1807. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1808. <span>${calc_expression}</span>
  1809. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
  1810. </td>
  1811. </tr>
  1812. <tr style="margin: 0; padding: 0;">
  1813. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1814. <span>${calc_param_values}</span>
  1815. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
  1816. </td>
  1817. </tr>
  1818. <tr style="margin: 0; padding: 0;">
  1819. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  1820. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1821. </td>
  1822. </tr>
  1823. <tr style="margin: 0; padding: 0;">
  1824. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  1825. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  1826. </td>
  1827. </tr>
  1828. <tr style="text-align: center; margin: 0; padding: 0;">
  1829. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  1830. </td>
  1831. </tr>
  1832. <tr style="text-align: center; margin: 0; padding: 0;">
  1833. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  1834. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  1835. </td>
  1836. </tr>
  1837. </tbody>
  1838. </table>
  1839. </div>
  1840. </td>
  1841. </tr>
  1842. </tbody>
  1843. </table>
  1844. </div>
  1845. </td>
  1846. </tr>
  1847. </tbody>
  1848. </table>
  1849. </body>
  1850. </html>
  1851. --multipart-boundary--
  1852. EOF
  1853. SENT_EMAIL=$?
  1854. # -----------------------------------------------------------------------------
  1855. # let netdata know
  1856. if [ ${SENT_EMAIL} -eq 0 \
  1857. -o ${SENT_PUSHOVER} -eq 0 \
  1858. -o ${SENT_TELEGRAM} -eq 0 \
  1859. -o ${SENT_SLACK} -eq 0 \
  1860. -o ${SENT_MSTEAM} -eq 0 \
  1861. -o ${SENT_ROCKETCHAT} -eq 0 \
  1862. -o ${SENT_ALERTA} -eq 0 \
  1863. -o ${SENT_FLOCK} -eq 0 \
  1864. -o ${SENT_DISCORD} -eq 0 \
  1865. -o ${SENT_TWILIO} -eq 0 \
  1866. -o ${SENT_HIPCHAT} -eq 0 \
  1867. -o ${SENT_MESSAGEBIRD} -eq 0 \
  1868. -o ${SENT_KAVENEGAR} -eq 0 \
  1869. -o ${SENT_PUSHBULLET} -eq 0 \
  1870. -o ${SENT_KAFKA} -eq 0 \
  1871. -o ${SENT_PD} -eq 0 \
  1872. -o ${SENT_FLEEP} -eq 0 \
  1873. -o ${SENT_PROWL} -eq 0 \
  1874. -o ${SENT_IRC} -eq 0 \
  1875. -o ${SENT_AWSSNS} -eq 0 \
  1876. -o ${SENT_CUSTOM} -eq 0 \
  1877. -o ${SENT_SYSLOG} -eq 0 \
  1878. ]
  1879. then
  1880. # we did send something
  1881. exit 0
  1882. fi
  1883. # we did not send anything
  1884. exit 1