alarm-notify.sh.in 82 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - irc notifications by @manosf
  32. # - custom notifications by @ktsaou
  33. # - syslog messages by @Ferroin
  34. # - Microsoft Team notification by @tioumen
  35. # - RocketChat notifications by @Hermsi1337 #3777
  36. # - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
  37. # - Dynatrace Event by @illumine
  38. # -----------------------------------------------------------------------------
  39. # testing notifications
  40. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  41. if [ "${2}" = "test" ]; then
  42. recipient="${1}"
  43. else
  44. recipient="${2}"
  45. fi
  46. [ -z "${recipient}" ] && recipient="sysadmin"
  47. id=1
  48. last="CLEAR"
  49. test_res=0
  50. for x in "WARNING" "CRITICAL" "CLEAR"; do
  51. echo >&2
  52. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  53. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  54. #shellcheck disable=SC2181
  55. if [ $? -ne 0 ]; then
  56. echo >&2 "# FAILED"
  57. test_res=1
  58. else
  59. echo >&2 "# OK"
  60. fi
  61. last="${x}"
  62. id=$((id + 1))
  63. done
  64. exit $test_res
  65. fi
  66. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  67. export LC_ALL=C
  68. # -----------------------------------------------------------------------------
  69. PROGRAM_NAME="$(basename "${0}")"
  70. logdate() {
  71. date "+%Y-%m-%d %H:%M:%S"
  72. }
  73. log() {
  74. local status="${1}"
  75. shift
  76. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  77. }
  78. warning() {
  79. log WARNING "${@}"
  80. }
  81. error() {
  82. log ERROR "${@}"
  83. }
  84. info() {
  85. log INFO "${@}"
  86. }
  87. fatal() {
  88. log FATAL "${@}"
  89. exit 1
  90. }
  91. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  92. debug() {
  93. [ "${debug}" = "1" ] && log DEBUG "${@}"
  94. }
  95. docurl() {
  96. if [ -z "${curl}" ]; then
  97. error "${curl} is unset."
  98. return 1
  99. fi
  100. if [ "${debug}" = "1" ]; then
  101. echo >&2 "--- BEGIN curl command ---"
  102. printf >&2 "%q " ${curl} "${@}"
  103. echo >&2
  104. echo >&2 "--- END curl command ---"
  105. local out code ret
  106. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  107. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  108. ret=$?
  109. echo >&2 "--- BEGIN received response ---"
  110. cat >&2 "${out}"
  111. echo >&2
  112. echo >&2 "--- END received response ---"
  113. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  114. rm "${out}"
  115. echo "${code}"
  116. return ${ret}
  117. fi
  118. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  119. return $?
  120. }
  121. # -----------------------------------------------------------------------------
  122. # List of all the notification mechanisms we support.
  123. # Used in a couple of places to write more compact code.
  124. method_names="
  125. email
  126. pushover
  127. pushbullet
  128. telegram
  129. slack
  130. alerta
  131. flock
  132. discord
  133. hipchat
  134. twilio
  135. messagebird
  136. pd
  137. fleep
  138. syslog
  139. custom
  140. msteam
  141. kavenegar
  142. prowl
  143. irc
  144. awssns
  145. rocketchat
  146. sms
  147. hangouts
  148. dynatrace
  149. matrix
  150. "
  151. # -----------------------------------------------------------------------------
  152. # this is to be overwritten by the config file
  153. custom_sender() {
  154. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  155. }
  156. # -----------------------------------------------------------------------------
  157. # check for BASH v4+ (required for associative arrays)
  158. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  159. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  160. fi
  161. # -----------------------------------------------------------------------------
  162. # defaults to allow running this script by hand
  163. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  164. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  165. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  166. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  167. [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
  168. # -----------------------------------------------------------------------------
  169. # parse command line parameters
  170. if [[ ${1} = "unittest" ]]; then
  171. unittest=1 # enable unit testing mode
  172. roles="${2}" # the role that should be used for unit testing
  173. cfgfile="${3}" # the location of the config file to use for unit testing
  174. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  175. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  176. else
  177. roles="${1}" # the roles that should be notified for this event
  178. args_host="${2}" # the host generated this event
  179. unique_id="${3}" # the unique id of this event
  180. alarm_id="${4}" # the unique id of the alarm that generated this event
  181. event_id="${5}" # the incremental id of the event, for this alarm id
  182. when="${6}" # the timestamp this event occurred
  183. name="${7}" # the name of the alarm, as given in netdata health.d entries
  184. chart="${8}" # the name of the chart (type.id)
  185. family="${9}" # the family of the chart
  186. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  187. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  188. value="${12}" # the current value of the alarm
  189. old_value="${13}" # the previous value of the alarm
  190. src="${14}" # the line number and file the alarm has been configured
  191. duration="${15}" # the duration in seconds of the previous alarm state
  192. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  193. units="${17}" # the units of the value
  194. info="${18}" # a short description of the alarm
  195. value_string="${19}" # friendly value (with units)
  196. # shellcheck disable=SC2034
  197. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  198. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  199. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  200. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  201. total_warnings="${23}" # Total number of alarms in WARNING state
  202. total_critical="${24}" # Total number of alarms in CRITICAL state
  203. fi
  204. # -----------------------------------------------------------------------------
  205. # find a suitable hostname to use, if netdata did not supply a hostname
  206. if [ -z ${args_host} ]; then
  207. this_host=$(hostname -s 2>/dev/null)
  208. host="${this_host}"
  209. args_host="${this_host}"
  210. else
  211. host="${args_host}"
  212. fi
  213. # -----------------------------------------------------------------------------
  214. # screen statuses we don't need to send a notification
  215. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  216. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  217. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  218. exit 1
  219. fi
  220. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  221. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  222. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  223. exit 1
  224. fi
  225. # -----------------------------------------------------------------------------
  226. # load configuration
  227. # By default fetch images from the global public registry.
  228. # This is required by default, since all notification methods need to download
  229. # images via the Internet, and private registries might not be reachable.
  230. # This can be overwritten at the configuration file.
  231. images_base_url="https://registry.my-netdata.io"
  232. # curl options to use
  233. curl_options=""
  234. # hostname handling
  235. use_fqdn="NO"
  236. # needed commands
  237. # if empty they will be searched in the system path
  238. curl=
  239. sendmail=
  240. # enable / disable features
  241. for method_name in ${method_names^^}; do
  242. declare SEND_${method_name}="YES"
  243. declare DEFAULT_RECIPIENT_${method_name}
  244. done
  245. for method_name in ${method_names}; do
  246. declare -A role_recipients_${method_name}
  247. done
  248. # slack configs
  249. SLACK_WEBHOOK_URL=
  250. # Microsoft Team configs
  251. MSTEAM_WEBHOOK_URL=
  252. # rocketchat configs
  253. ROCKETCHAT_WEBHOOK_URL=
  254. # alerta configs
  255. ALERTA_WEBHOOK_URL=
  256. ALERTA_API_KEY=
  257. # flock configs
  258. FLOCK_WEBHOOK_URL=
  259. # discord configs
  260. DISCORD_WEBHOOK_URL=
  261. # pushover configs
  262. PUSHOVER_APP_TOKEN=
  263. # pushbullet configs
  264. PUSHBULLET_ACCESS_TOKEN=
  265. PUSHBULLET_SOURCE_DEVICE=
  266. # twilio configs
  267. TWILIO_ACCOUNT_SID=
  268. TWILIO_ACCOUNT_TOKEN=
  269. TWILIO_NUMBER=
  270. # hipchat configs
  271. HIPCHAT_SERVER=
  272. HIPCHAT_AUTH_TOKEN=
  273. # messagebird configs
  274. MESSAGEBIRD_ACCESS_KEY=
  275. MESSAGEBIRD_NUMBER=
  276. # kavenegar configs
  277. KAVENEGAR_API_KEY=
  278. KAVENEGAR_SENDER=
  279. # telegram configs
  280. TELEGRAM_BOT_TOKEN=
  281. # kafka configs
  282. SEND_KAFKA="YES"
  283. KAFKA_URL=
  284. KAFKA_SENDER_IP=
  285. # pagerduty.com configs
  286. PD_SERVICE_KEY=
  287. # fleep.io configs
  288. FLEEP_SENDER="${host}"
  289. # Amazon SNS configs
  290. AWSSNS_MESSAGE_FORMAT=
  291. # Matrix configs
  292. MATRIX_HOMESERVER=
  293. MATRIX_ACCESSTOKEN=
  294. # syslog configs
  295. SYSLOG_FACILITY=
  296. # email configs
  297. EMAIL_SENDER=
  298. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  299. EMAIL_THREADING=
  300. EMAIL_PLAINTEXT_ONLY=
  301. # irc configs
  302. IRC_NICKNAME=
  303. IRC_REALNAME=
  304. IRC_NETWORK=
  305. # hangouts configs
  306. declare -A HANGOUTS_WEBHOOK_URI
  307. # dynatrace configs
  308. DYNATRACE_SPACE=
  309. DYNATRACE_SERVER=
  310. DYNATRACE_TOKEN=
  311. DYNATRACE_TAG_VALUE=
  312. DYNATRACE_ANNOTATION_TYPE=
  313. DYNATRACE_EVENT=
  314. SEND_DYNATRACE=
  315. # load the stock and user configuration files
  316. # these will overwrite the variables above
  317. if [ ${unittest} ]; then
  318. if source "${cfgfile}"; then
  319. error "Failed to load requested config file."
  320. exit 1
  321. fi
  322. else
  323. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  324. if [ -f "${CONFIG}" ]; then
  325. debug "Loading config file '${CONFIG}'..."
  326. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  327. else
  328. warning "Cannot find file '${CONFIG}'."
  329. fi
  330. done
  331. fi
  332. # If we didn't autodetect the character set for e-mail and it wasn't
  333. # set by the user, we need to set it to a reasonable default. UTF-8
  334. # should be correct for almost all modern UNIX systems.
  335. if [ -z ${EMAIL_CHARSET} ]; then
  336. EMAIL_CHARSET="UTF-8"
  337. fi
  338. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  339. # unless we're sending an alarm for a child system which we can't get the
  340. # FQDN of easily.
  341. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  342. host="$(hostname -f 2>/dev/null)"
  343. fi
  344. # -----------------------------------------------------------------------------
  345. # filter a recipient based on alarm event severity
  346. filter_recipient_by_criticality() {
  347. local method="${1}" x="${2}" r s
  348. shift
  349. r="${x/|*/}" # the recipient
  350. s="${x/*|/}" # the severity required for notifying this recipient
  351. # no severity filtering for this person
  352. [ "${r}" = "${s}" ] && return 0
  353. # the severity is invalid
  354. s="${s^^}"
  355. if [ "${s}" != "CRITICAL" ]; then
  356. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  357. return 0
  358. fi
  359. # create the status tracking directory for this user
  360. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  361. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  362. case "${status}" in
  363. CRITICAL)
  364. # make sure he will get future notifications for this alarm too
  365. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  366. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  367. return 0
  368. ;;
  369. WARNING)
  370. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  371. # we do not remove the file, so that he will get future notifications of this alarm
  372. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  373. return 0
  374. fi
  375. ;;
  376. *)
  377. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  378. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  379. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  380. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  381. return 0
  382. fi
  383. ;;
  384. esac
  385. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  386. return 1
  387. }
  388. # -----------------------------------------------------------------------------
  389. # verify the delivery methods supported
  390. # check slack
  391. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  392. # check rocketchat
  393. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  394. # check alerta
  395. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  396. # check flock
  397. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  398. # check discord
  399. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  400. # check pushover
  401. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  402. # check pushbullet
  403. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  404. # check twilio
  405. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  406. # check hipchat
  407. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  408. # check messagebird
  409. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  410. # check kavenegar
  411. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  412. # check telegram
  413. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  414. # check kafka
  415. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  416. # check irc
  417. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  418. # check hangouts
  419. [ ${#HANGOUTS_WEBHOOK_URI[@]} -eq 0 ] && SEND_HANGOUTS="NO"
  420. # check fleep
  421. #shellcheck disable=SC2153
  422. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  423. # check dynatrace
  424. { [ -z "${DYNATRACE_SPACE}" ] ||
  425. [ -z "${DYNATRACE_SERVER}" ] ||
  426. [ -z "${DYNATRACE_TOKEN}" ] ||
  427. [ -z "${DYNATRACE_TAG_VALUE}" ] ||
  428. [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
  429. # check matrix
  430. { [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
  431. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  432. [ "${SEND_SLACK}" = "YES" ] ||
  433. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  434. [ "${SEND_ALERTA}" = "YES" ] ||
  435. [ "${SEND_PD}" = "YES" ] ||
  436. [ "${SEND_FLOCK}" = "YES" ] ||
  437. [ "${SEND_DISCORD}" = "YES" ] ||
  438. [ "${SEND_HIPCHAT}" = "YES" ] ||
  439. [ "${SEND_TWILIO}" = "YES" ] ||
  440. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  441. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  442. [ "${SEND_TELEGRAM}" = "YES" ] ||
  443. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  444. [ "${SEND_KAFKA}" = "YES" ] ||
  445. [ "${SEND_FLEEP}" = "YES" ] ||
  446. [ "${SEND_PROWL}" = "YES" ] ||
  447. [ "${SEND_HANGOUTS}" = "YES" ] ||
  448. [ "${SEND_MATRIX}" = "YES" ] ||
  449. [ "${SEND_CUSTOM}" = "YES" ] ||
  450. [ "${SEND_MSTEAM}" = "YES" ] ||
  451. [ "${SEND_DYNATRACE}" = "YES" ]; then
  452. # if we need curl, check for the curl command
  453. if [ -z "${curl}" ]; then
  454. curl="$(command -v curl 2>/dev/null)"
  455. fi
  456. if [ -z "${curl}" ]; then
  457. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  458. SEND_PUSHOVER="NO"
  459. SEND_PUSHBULLET="NO"
  460. SEND_TELEGRAM="NO"
  461. SEND_SLACK="NO"
  462. SEND_MSTEAM="NO"
  463. SEND_ROCKETCHAT="NO"
  464. SEND_ALERTA="NO"
  465. SEND_PD="NO"
  466. SEND_FLOCK="NO"
  467. SEND_DISCORD="NO"
  468. SEND_TWILIO="NO"
  469. SEND_HIPCHAT="NO"
  470. SEND_MESSAGEBIRD="NO"
  471. SEND_KAVENEGAR="NO"
  472. SEND_KAFKA="NO"
  473. SEND_FLEEP="NO"
  474. SEND_PROWL="NO"
  475. SEND_HANGOUTS="NO"
  476. SEND_MATRIX="NO"
  477. SEND_CUSTOM="NO"
  478. SEND_DYNATRACE="NO"
  479. fi
  480. fi
  481. if [ "${SEND_SMS}" = "YES" ]; then
  482. if [ -z "${sendsms}" ]; then
  483. sendsms="$(command -v sendsms 2>/dev/null)"
  484. fi
  485. if [ -z "${sendsms}" ]; then
  486. SEND_SMS="NO"
  487. fi
  488. fi
  489. # if we need sendmail, check for the sendmail command
  490. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  491. sendmail="$(command -v sendmail 2>/dev/null)"
  492. if [ -z "${sendmail}" ]; then
  493. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  494. SEND_EMAIL="NO"
  495. fi
  496. fi
  497. # if we need logger, check for the logger command
  498. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  499. logger="$(command -v logger 2>/dev/null)"
  500. if [ -z "${logger}" ]; then
  501. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  502. SEND_SYSLOG="NO"
  503. fi
  504. fi
  505. # if we need aws, check for the aws command
  506. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  507. aws="$(command -v aws 2>/dev/null)"
  508. if [ -z "${aws}" ]; then
  509. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  510. SEND_AWSSNS="NO"
  511. fi
  512. fi
  513. # -----------------------------------------------------------------------------
  514. # find the recipients' addresses per method
  515. # netdata may call us with multiple roles, and roles may have multiple but
  516. # overlapping recipients - so, here we find the unique recipients.
  517. for method_name in ${method_names}; do
  518. send_var="SEND_${method_name^^}"
  519. if [ "${!send_var}" = "NO" ]; then
  520. continue
  521. fi
  522. declare -A arr_var=()
  523. for x in ${roles//,/ }; do
  524. # the roles 'silent' and 'disabled' mean:
  525. # don't send a notification for this role
  526. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  527. continue
  528. fi
  529. role_recipients="role_recipients_${method_name}[$x]"
  530. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  531. a="${!role_recipients}"
  532. [ -z "${a}" ] && a="${!default_recipient_var}"
  533. for r in ${a//,/ }; do
  534. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  535. done
  536. done
  537. # build the list of recipients
  538. to_var="to_${method_name}"
  539. declare to_${method_name}="${!arr_var[*]}"
  540. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  541. done
  542. # -----------------------------------------------------------------------------
  543. # handle fixup of the email recipient list.
  544. fix_to_email() {
  545. to_email=
  546. while [ -n "${1}" ]; do
  547. [ -n "${to_email}" ] && to_email="${to_email}, "
  548. to_email="${to_email}${1}"
  549. shift 1
  550. done
  551. }
  552. # ${to_email} without quotes here
  553. fix_to_email ${to_email}
  554. # -----------------------------------------------------------------------------
  555. # handle output if we're running in unit test mode
  556. if [ ${unittest} ]; then
  557. for method_name in ${method_names}; do
  558. to_var="to_${method_name}"
  559. echo "results: ${method_name}: ${!to_var}"
  560. done
  561. exit 0
  562. fi
  563. # -----------------------------------------------------------------------------
  564. # check that we have at least a method enabled
  565. proceed=0
  566. for method in "${SEND_EMAIL}" \
  567. "${SEND_PUSHOVER}" \
  568. "${SEND_TELEGRAM}" \
  569. "${SEND_SLACK}" \
  570. "${SEND_ROCKETCHAT}" \
  571. "${SEND_ALERTA}" \
  572. "${SEND_FLOCK}" \
  573. "${SEND_DISCORD}" \
  574. "${SEND_TWILIO}" \
  575. "${SEND_HIPCHAT}" \
  576. "${SEND_MESSAGEBIRD}" \
  577. "${SEND_KAVENEGAR}" \
  578. "${SEND_PUSHBULLET}" \
  579. "${SEND_KAFKA}" \
  580. "${SEND_PD}" \
  581. "${SEND_FLEEP}" \
  582. "${SEND_PROWL}" \
  583. "${SEND_MATRIX}" \
  584. "${SEND_CUSTOM}" \
  585. "${SEND_IRC}" \
  586. "${SEND_HANGOUTS}" \
  587. "${SEND_AWSSNS}" \
  588. "${SEND_SYSLOG}" \
  589. "${SEND_SMS}" \
  590. "${SEND_MSTEAM}" \
  591. "${SEND_DYNATRACE}"; do
  592. if [ "${method}" == "YES" ]; then
  593. proceed=1
  594. break
  595. fi
  596. done
  597. if [ "$proceed" -eq 0 ]; then
  598. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  599. fi
  600. # -----------------------------------------------------------------------------
  601. # get the date the alarm happened
  602. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  603. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  604. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  605. [ -z "${date}" ] && date=$(date 2>/dev/null)
  606. # ----------------------------------------------------------------------------
  607. # prepare some extra headers if we've been asked to thread e-mails
  608. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  609. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
  610. else
  611. email_thread_headers=
  612. fi
  613. # -----------------------------------------------------------------------------
  614. # function to URL encode a string
  615. urlencode() {
  616. local string="${1}" strlen encoded pos c o
  617. strlen=${#string}
  618. for ((pos = 0; pos < strlen; pos++)); do
  619. c=${string:pos:1}
  620. case "${c}" in
  621. [-_.~a-zA-Z0-9])
  622. o="${c}"
  623. ;;
  624. *)
  625. printf -v o '%%%02x' "'${c}"
  626. ;;
  627. esac
  628. encoded+="${o}"
  629. done
  630. REPLY="${encoded}"
  631. echo "${REPLY}"
  632. }
  633. # -----------------------------------------------------------------------------
  634. # function to convert a duration in seconds, to a human readable duration
  635. # using DAYS, MINUTES, SECONDS
  636. duration4human() {
  637. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  638. d=$((s / 86400))
  639. s=$((s - (d * 86400)))
  640. h=$((s / 3600))
  641. s=$((s - (h * 3600)))
  642. m=$((s / 60))
  643. s=$((s - (m * 60)))
  644. if [ ${d} -gt 0 ]; then
  645. [ ${m} -ge 30 ] && h=$((h + 1))
  646. [ ${d} -gt 1 ] && ds="days"
  647. [ ${h} -gt 1 ] && hs="hours"
  648. if [ ${h} -gt 0 ]; then
  649. ret="${d} ${ds} and ${h} ${hs}"
  650. else
  651. ret="${d} ${ds}"
  652. fi
  653. elif [ ${h} -gt 0 ]; then
  654. [ ${s} -ge 30 ] && m=$((m + 1))
  655. [ ${h} -gt 1 ] && hs="hours"
  656. [ ${m} -gt 1 ] && ms="minutes"
  657. if [ ${m} -gt 0 ]; then
  658. ret="${h} ${hs} and ${m} ${ms}"
  659. else
  660. ret="${h} ${hs}"
  661. fi
  662. elif [ ${m} -gt 0 ]; then
  663. [ ${m} -gt 1 ] && ms="minutes"
  664. [ ${s} -gt 1 ] && ss="seconds"
  665. if [ ${s} -gt 0 ]; then
  666. ret="${m} ${ms} and ${s} ${ss}"
  667. else
  668. ret="${m} ${ms}"
  669. fi
  670. else
  671. [ ${s} -gt 1 ] && ss="seconds"
  672. ret="${s} ${ss}"
  673. fi
  674. REPLY="${ret}"
  675. echo "${REPLY}"
  676. }
  677. # -----------------------------------------------------------------------------
  678. # email sender
  679. send_email() {
  680. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  681. if [ "${SEND_EMAIL}" = "YES" ]; then
  682. if [ -n "${EMAIL_SENDER}" ]; then
  683. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  684. # the name includes double quotes
  685. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  686. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  687. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  688. # the name includes single quotes
  689. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  690. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  691. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  692. # the name does not have any quotes
  693. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  694. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  695. fi
  696. fi
  697. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  698. [ -n "${sender_name}" ] && opts+=(-F "${sender_name}")
  699. if [ "${debug}" = "1" ]; then
  700. echo >&2 "--- BEGIN sendmail command ---"
  701. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  702. echo >&2
  703. echo >&2 "--- END sendmail command ---"
  704. fi
  705. "${sendmail}" -t "${opts[@]}"
  706. ret=$?
  707. if [ ${ret} -eq 0 ]; then
  708. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  709. return 0
  710. else
  711. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
  712. return 1
  713. fi
  714. fi
  715. return 1
  716. }
  717. # -----------------------------------------------------------------------------
  718. # pushover sender
  719. send_pushover() {
  720. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  721. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  722. # https://pushover.net/api
  723. priority=-2
  724. case "${status}" in
  725. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  726. WARNING) priority=0 ;; # normal priority: respect quiet hours
  727. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  728. *) priority=-2 ;; # lowest priority: no notification at all
  729. esac
  730. for user in ${usertokens}; do
  731. httpcode=$(docurl \
  732. --form-string "token=${apptoken}" \
  733. --form-string "user=${user}" \
  734. --form-string "html=1" \
  735. --form-string "title=${title}" \
  736. --form-string "message=${message}" \
  737. --form-string "timestamp=${when}" \
  738. --form-string "url=${url}" \
  739. --form-string "url_title=Open netdata dashboard to view the alarm" \
  740. --form-string "priority=${priority}" \
  741. https://api.pushover.net/1/messages.json)
  742. if [ "${httpcode}" = "200" ]; then
  743. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  744. sent=$((sent + 1))
  745. else
  746. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  747. fi
  748. done
  749. [ ${sent} -gt 0 ] && return 0
  750. fi
  751. return 1
  752. }
  753. # -----------------------------------------------------------------------------
  754. # pushbullet sender
  755. send_pushbullet() {
  756. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
  757. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  758. #https://docs.pushbullet.com/#create-push
  759. for user in ${recipients}; do
  760. httpcode=$(docurl \
  761. --header 'Access-Token: '${userapikey}'' \
  762. --header 'Content-Type: application/json' \
  763. --data-binary @<(
  764. cat <<EOF
  765. {"title": "${title}",
  766. "type": "link",
  767. "email": "${user}",
  768. "body": "$(echo -n ${message})",
  769. "url": "${url}",
  770. "source_device_iden": "${source_device}"}
  771. EOF
  772. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  773. if [ "${httpcode}" = "200" ]; then
  774. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  775. sent=$((sent + 1))
  776. else
  777. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  778. fi
  779. done
  780. [ ${sent} -gt 0 ] && return 0
  781. fi
  782. return 1
  783. }
  784. # -----------------------------------------------------------------------------
  785. # kafka sender
  786. send_kafka() {
  787. local httpcode sent=0
  788. if [ "${SEND_KAFKA}" = "YES" ]; then
  789. httpcode=$(docurl -X POST \
  790. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  791. "${KAFKA_URL}")
  792. if [ "${httpcode}" = "204" ]; then
  793. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  794. sent=$((sent + 1))
  795. else
  796. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
  797. fi
  798. [ ${sent} -gt 0 ] && return 0
  799. fi
  800. return 1
  801. }
  802. # -----------------------------------------------------------------------------
  803. # pagerduty.com sender
  804. send_pd() {
  805. local recipients="${1}" sent=0
  806. unset t
  807. case ${status} in
  808. CLEAR) t='resolve' ;;
  809. WARNING) t='trigger' ;;
  810. CRITICAL) t='trigger' ;;
  811. esac
  812. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  813. for PD_SERVICE_KEY in ${recipients}; do
  814. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  815. payload="$(
  816. cat <<EOF
  817. {
  818. "service_key": "${PD_SERVICE_KEY}",
  819. "event_type": "${t}",
  820. "incident_key" : "${alarm_id}",
  821. "description": "${d}",
  822. "details": {
  823. "value_w_units": "${value_string}",
  824. "when": "${when}",
  825. "duration" : "${duration}",
  826. "roles": "${roles}",
  827. "alarm_id" : "${alarm_id}",
  828. "name" : "${name}",
  829. "chart" : "${chart}",
  830. "family" : "${family}",
  831. "status" : "${status}",
  832. "old_status" : "${old_status}",
  833. "value" : "${value}",
  834. "old_value" : "${old_value}",
  835. "src" : "${src}",
  836. "non_clear_duration" : "${non_clear_duration}",
  837. "units" : "${units}",
  838. "info" : "${info}"
  839. }
  840. }
  841. EOF
  842. )"
  843. httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
  844. if [ "${httpcode}" = "200" ]; then
  845. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  846. sent=$((sent + 1))
  847. else
  848. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  849. fi
  850. done
  851. [ ${sent} -gt 0 ] && return 0
  852. fi
  853. return 1
  854. }
  855. # -----------------------------------------------------------------------------
  856. # twilio sender
  857. send_twilio() {
  858. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  859. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  860. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  861. for user in ${recipients}; do
  862. httpcode=$(docurl -X POST \
  863. --data-urlencode "From=${twilionumber}" \
  864. --data-urlencode "To=${user}" \
  865. --data-urlencode "Body=${title} ${message}" \
  866. -u "${accountsid}:${accounttoken}" \
  867. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  868. if [ "${httpcode}" = "201" ]; then
  869. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  870. sent=$((sent + 1))
  871. else
  872. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  873. fi
  874. done
  875. [ ${sent} -gt 0 ] && return 0
  876. fi
  877. return 1
  878. }
  879. # -----------------------------------------------------------------------------
  880. # hipchat sender
  881. send_hipchat() {
  882. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  883. # remove <small></small> from the message
  884. message="${message//<small>/}"
  885. message="${message//<\/small>/}"
  886. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  887. # Valid values: html, text.
  888. # Defaults to 'html'.
  889. msg_format="html"
  890. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  891. case "${status}" in
  892. WARNING) color="yellow" ;;
  893. CRITICAL) color="red" ;;
  894. CLEAR) color="green" ;;
  895. *) color="gray" ;;
  896. esac
  897. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  898. # Each recipient's notification preferences are taken into account.
  899. # Defaults to false.
  900. notify="true"
  901. for room in ${recipients}; do
  902. httpcode=$(docurl -X POST \
  903. -H "Content-type: application/json" \
  904. -H "Authorization: Bearer ${authtoken}" \
  905. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  906. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  907. if [ "${httpcode}" = "204" ]; then
  908. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  909. sent=$((sent + 1))
  910. else
  911. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
  912. fi
  913. done
  914. [ ${sent} -gt 0 ] && return 0
  915. fi
  916. return 1
  917. }
  918. # -----------------------------------------------------------------------------
  919. # messagebird sender
  920. send_messagebird() {
  921. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  922. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  923. #https://developers.messagebird.com/docs/messaging
  924. for user in ${recipients}; do
  925. httpcode=$(docurl -X POST \
  926. --data-urlencode "originator=${messagebirdnumber}" \
  927. --data-urlencode "recipients=${user}" \
  928. --data-urlencode "body=${title} ${message}" \
  929. --data-urlencode "datacoding=auto" \
  930. -H "Authorization: AccessKey ${accesskey}" \
  931. "https://rest.messagebird.com/messages")
  932. if [ "${httpcode}" = "201" ]; then
  933. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  934. sent=$((sent + 1))
  935. else
  936. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  937. fi
  938. done
  939. [ ${sent} -gt 0 ] && return 0
  940. fi
  941. return 1
  942. }
  943. # -----------------------------------------------------------------------------
  944. # kavenegar sender
  945. send_kavenegar() {
  946. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  947. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  948. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  949. for user in ${recipients}; do
  950. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  951. --data-urlencode "sender=${kavenegarsender}" \
  952. --data-urlencode "receptor=${user}" \
  953. --data-urlencode "message=${title} ${message}")
  954. if [ "${httpcode}" = "200" ]; then
  955. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  956. sent=$((sent + 1))
  957. else
  958. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
  959. fi
  960. done
  961. [ ${sent} -gt 0 ] && return 0
  962. fi
  963. return 1
  964. }
  965. # -----------------------------------------------------------------------------
  966. # telegram sender
  967. send_telegram() {
  968. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  969. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  970. case "${status}" in
  971. WARNING) emoji="⚠️" ;;
  972. CRITICAL) emoji="🔴" ;;
  973. CLEAR) emoji="✅" ;;
  974. *) emoji="⚪️" ;;
  975. esac
  976. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  977. for chatid in ${chatids}; do
  978. # https://core.telegram.org/bots/api#sendmessage
  979. httpcode=$(docurl ${disableNotification} \
  980. --data-urlencode "parse_mode=HTML" \
  981. --data-urlencode "disable_web_page_preview=true" \
  982. --data-urlencode "text=${emoji} ${message}" \
  983. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  984. if [ "${httpcode}" = "200" ]; then
  985. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  986. sent=$((sent + 1))
  987. elif [ "${httpcode}" = "401" ]; then
  988. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  989. else
  990. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
  991. fi
  992. done
  993. [ ${sent} -gt 0 ] && return 0
  994. fi
  995. return 1
  996. }
  997. # -----------------------------------------------------------------------------
  998. # Microsoft Team sender
  999. send_msteam() {
  1000. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1001. [ "${SEND_MSTEAM}" != "YES" ] && return 1
  1002. case "${status}" in
  1003. WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}" ;;
  1004. CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}" ;;
  1005. CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}" ;;
  1006. *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}" ;;
  1007. esac
  1008. for channel in ${channels}; do
  1009. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  1010. ## Online designer : https://acdesignerbeta.azurewebsites.net/
  1011. payload="$(
  1012. cat <<EOF
  1013. {
  1014. "@context": "http://schema.org/extensions",
  1015. "@type": "MessageCard",
  1016. "themeColor": "${color}",
  1017. "title": "$icon Alert ${status} from netdata for ${host}",
  1018. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  1019. "potentialAction": [
  1020. {
  1021. "@type": "OpenUri",
  1022. "name": "Netdata",
  1023. "targets": [
  1024. { "os": "default", "uri": "${goto_url}" }
  1025. ]
  1026. }
  1027. ]
  1028. }
  1029. EOF
  1030. )"
  1031. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1032. webhook="${webhook//CHANNEL/${channel}}"
  1033. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
  1034. if [ "${httpcode}" = "200" ]; then
  1035. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
  1036. sent=$((sent + 1))
  1037. else
  1038. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
  1039. fi
  1040. done
  1041. [ ${sent} -gt 0 ] && return 0
  1042. return 1
  1043. }
  1044. # slack sender
  1045. send_slack() {
  1046. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1047. [ "${SEND_SLACK}" != "YES" ] && return 1
  1048. case "${status}" in
  1049. WARNING) color="warning" ;;
  1050. CRITICAL) color="danger" ;;
  1051. CLEAR) color="good" ;;
  1052. *) color="#777777" ;;
  1053. esac
  1054. for channel in ${channels}; do
  1055. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1056. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1057. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1058. if [ "${channel}" = "#" ]; then
  1059. ch=""
  1060. chstr="without specifying a channel"
  1061. else
  1062. ch="\"channel\": \"${channel}\","
  1063. chstr="to '${channel}'"
  1064. fi
  1065. payload="$(
  1066. cat <<EOF
  1067. {
  1068. $ch
  1069. "username": "netdata on ${host}",
  1070. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1071. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1072. "attachments": [
  1073. {
  1074. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1075. "color": "${color}",
  1076. "title": "${alarm}",
  1077. "title_link": "${goto_url}",
  1078. "text": "${info}",
  1079. "fields": [
  1080. {
  1081. "title": "${chart}",
  1082. "short": true
  1083. },
  1084. {
  1085. "title": "${family}",
  1086. "short": true
  1087. }
  1088. ],
  1089. "thumb_url": "${image}",
  1090. "footer": "by ${host}",
  1091. "ts": ${when}
  1092. }
  1093. ]
  1094. }
  1095. EOF
  1096. )"
  1097. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1098. if [ "${httpcode}" = "200" ]; then
  1099. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1100. sent=$((sent + 1))
  1101. else
  1102. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}."
  1103. fi
  1104. done
  1105. [ ${sent} -gt 0 ] && return 0
  1106. return 1
  1107. }
  1108. # -----------------------------------------------------------------------------
  1109. # rocketchat sender
  1110. send_rocketchat() {
  1111. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1112. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1113. case "${status}" in
  1114. WARNING) color="warning" ;;
  1115. CRITICAL) color="danger" ;;
  1116. CLEAR) color="good" ;;
  1117. *) color="#777777" ;;
  1118. esac
  1119. for channel in ${channels}; do
  1120. payload="$(
  1121. cat <<EOF
  1122. {
  1123. "channel": "#${channel}",
  1124. "alias": "netdata on ${host}",
  1125. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1126. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1127. "attachments": [
  1128. {
  1129. "color": "${color}",
  1130. "title": "${alarm}",
  1131. "title_link": "${goto_url}",
  1132. "text": "${info}",
  1133. "fields": [
  1134. {
  1135. "title": "${chart}",
  1136. "short": true,
  1137. "value": "chart"
  1138. },
  1139. {
  1140. "title": "${family}",
  1141. "short": true,
  1142. "value": "family"
  1143. }
  1144. ],
  1145. "thumb_url": "${image}",
  1146. "ts": "${when}"
  1147. }
  1148. ]
  1149. }
  1150. EOF
  1151. )"
  1152. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1153. if [ "${httpcode}" = "200" ]; then
  1154. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1155. sent=$((sent + 1))
  1156. else
  1157. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1158. fi
  1159. done
  1160. [ ${sent} -gt 0 ] && return 0
  1161. return 1
  1162. }
  1163. # -----------------------------------------------------------------------------
  1164. # alerta sender
  1165. send_alerta() {
  1166. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1167. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1168. case "${status}" in
  1169. CRITICAL) severity="critical" ;;
  1170. WARNING) severity="warning" ;;
  1171. CLEAR) severity="cleared" ;;
  1172. *) severity="indeterminate" ;;
  1173. esac
  1174. if [[ ${chart} == httpcheck* ]]; then
  1175. resource=$chart
  1176. event=$name
  1177. else
  1178. resource="${host}:${family}"
  1179. event="${chart}.${name}"
  1180. fi
  1181. for channel in ${channels}; do
  1182. payload="$(
  1183. cat <<EOF
  1184. {
  1185. "resource": "${resource}",
  1186. "event": "${event}",
  1187. "environment": "${channel}",
  1188. "severity": "${severity}",
  1189. "service": ["Netdata"],
  1190. "group": "Performance",
  1191. "value": "${value_string}",
  1192. "text": "${info}",
  1193. "tags": ["alarm_id:${alarm_id}"],
  1194. "attributes": {
  1195. "roles": "${roles}",
  1196. "name": "${name}",
  1197. "chart": "${chart}",
  1198. "family": "${family}",
  1199. "source": "${src}",
  1200. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1201. },
  1202. "origin": "netdata/${host}",
  1203. "type": "netdataAlarm",
  1204. "rawData": "${BASH_ARGV[@]}"
  1205. }
  1206. EOF
  1207. )"
  1208. if [ -n "${ALERTA_API_KEY}" ]; then
  1209. auth="Key ${ALERTA_API_KEY}"
  1210. fi
  1211. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1212. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1213. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1214. sent=$((sent + 1))
  1215. elif [ "${httpcode}" = "202" ]; then
  1216. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1217. else
  1218. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1219. fi
  1220. done
  1221. [ ${sent} -gt 0 ] && return 0
  1222. return 1
  1223. }
  1224. # -----------------------------------------------------------------------------
  1225. # flock sender
  1226. send_flock() {
  1227. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1228. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1229. case "${status}" in
  1230. WARNING) color="warning" ;;
  1231. CRITICAL) color="danger" ;;
  1232. CLEAR) color="good" ;;
  1233. *) color="#777777" ;;
  1234. esac
  1235. for channel in ${channels}; do
  1236. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1237. \"sendAs\": {
  1238. \"name\" : \"netdata on ${host}\",
  1239. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1240. },
  1241. \"text\": \"${host} *${status_message}*\",
  1242. \"timestamp\": \"${when}\",
  1243. \"attachments\": [
  1244. {
  1245. \"description\": \"${chart} (${family}) - ${info}\",
  1246. \"color\": \"${color}\",
  1247. \"title\": \"${alarm}\",
  1248. \"url\": \"${goto_url}\",
  1249. \"text\": \"${info}\",
  1250. \"views\": {
  1251. \"image\": {
  1252. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1253. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1254. \"filename\": \"${image}\"
  1255. }
  1256. }
  1257. }
  1258. ]
  1259. }")
  1260. if [ "${httpcode}" = "200" ]; then
  1261. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1262. sent=$((sent + 1))
  1263. else
  1264. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1265. fi
  1266. done
  1267. [ ${sent} -gt 0 ] && return 0
  1268. return 1
  1269. }
  1270. # -----------------------------------------------------------------------------
  1271. # discord sender
  1272. send_discord() {
  1273. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1274. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1275. case "${status}" in
  1276. WARNING) color="warning" ;;
  1277. CRITICAL) color="danger" ;;
  1278. CLEAR) color="good" ;;
  1279. *) color="#777777" ;;
  1280. esac
  1281. for channel in ${channels}; do
  1282. username="netdata on ${host}"
  1283. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1284. payload="$(
  1285. cat <<EOF
  1286. {
  1287. "channel": "#${channel}",
  1288. "username": "${username}",
  1289. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1290. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1291. "attachments": [
  1292. {
  1293. "color": "${color}",
  1294. "title": "${alarm}",
  1295. "title_link": "${goto_url}",
  1296. "text": "${info}",
  1297. "fields": [
  1298. {
  1299. "title": "${chart}",
  1300. "value": "${family}"
  1301. }
  1302. ],
  1303. "thumb_url": "${image}",
  1304. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1305. "footer": "${host}",
  1306. "ts": ${when}
  1307. }
  1308. ]
  1309. }
  1310. EOF
  1311. )"
  1312. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1313. if [ "${httpcode}" = "200" ]; then
  1314. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1315. sent=$((sent + 1))
  1316. else
  1317. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
  1318. fi
  1319. done
  1320. [ ${sent} -gt 0 ] && return 0
  1321. return 1
  1322. }
  1323. # -----------------------------------------------------------------------------
  1324. # fleep sender
  1325. send_fleep() {
  1326. local httpcode sent=0 webhooks="${1}" data message
  1327. if [ "${SEND_FLEEP}" = "YES" ]; then
  1328. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1329. for hook in ${webhooks}; do
  1330. data="{ "
  1331. data="${data} 'message': '${message}', "
  1332. data="${data} 'user': '${FLEEP_SENDER}' "
  1333. data="${data} }"
  1334. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1335. if [ "${httpcode}" = "200" ]; then
  1336. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1337. sent=$((sent + 1))
  1338. else
  1339. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
  1340. fi
  1341. done
  1342. [ ${sent} -gt 0 ] && return 0
  1343. fi
  1344. return 1
  1345. }
  1346. # -----------------------------------------------------------------------------
  1347. # Prowl sender
  1348. send_prowl() {
  1349. local httpcode sent=0 data message keys prio=0 alarm_url event
  1350. if [ "${SEND_PROWL}" = "YES" ]; then
  1351. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1352. message="description=${message}"
  1353. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1354. keys="apikey=${keys}"
  1355. app="application=Netdata"
  1356. case "${status}" in
  1357. CRITICAL)
  1358. prio=2
  1359. ;;
  1360. WARNING)
  1361. prio=1
  1362. ;;
  1363. esac
  1364. prio="priority=${prio}"
  1365. alarm_url="$(urlencode ${goto_url})"
  1366. alarm_url="url=${alarm_url}"
  1367. event="$(urlencode "${host} ${status_message}")"
  1368. event="event=${event}"
  1369. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1370. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1371. if [ "${httpcode}" = "200" ]; then
  1372. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1373. sent=1
  1374. else
  1375. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1376. fi
  1377. [ ${sent} -gt 0 ] && return 0
  1378. fi
  1379. return 1
  1380. }
  1381. # -----------------------------------------------------------------------------
  1382. # irc sender
  1383. send_irc() {
  1384. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
  1385. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ]; then
  1386. case "${status}" in
  1387. WARNING) color="warning" ;;
  1388. CRITICAL) color="danger" ;;
  1389. CLEAR) color="good" ;;
  1390. *) color="#777777" ;;
  1391. esac
  1392. SNDMESSAGE="${MESSAGE//$'\n'/", "}"
  1393. for CHANNEL in ${CHANNELS}; do
  1394. error=0
  1395. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" 6667)
  1396. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1397. for code in ${reply_codes}; do
  1398. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1399. error=1
  1400. break
  1401. fi
  1402. done
  1403. if [ "${error}" -eq 0 ]; then
  1404. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1405. sent=$((sent + 1))
  1406. else
  1407. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1408. fi
  1409. done
  1410. fi
  1411. [ ${sent} -gt 0 ] && return 0
  1412. return 1
  1413. }
  1414. # -----------------------------------------------------------------------------
  1415. # Amazon SNS sender
  1416. send_awssns() {
  1417. local targets="${1}" message='' sent=0 region=''
  1418. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1419. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1420. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1421. for target in ${targets}; do
  1422. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1423. region="$(echo ${target} | cut -f 4 -d ':')"
  1424. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1425. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1426. sent=$((sent + 1))
  1427. else
  1428. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1429. fi
  1430. done
  1431. [ ${sent} -gt 0 ] && return 0
  1432. return 1
  1433. }
  1434. # -----------------------------------------------------------------------------
  1435. # Matrix sender
  1436. send_matrix() {
  1437. local homeserver="${1}" webhook accesstoken rooms="${2}" httpcode sent=0 payload
  1438. [ "${SEND_MATRIX}" != "YES" ] && return 1
  1439. [ -z "${MATRIX_ACCESSTOKEN}" ] && return 1
  1440. accesstoken="${MATRIX_ACCESSTOKEN}"
  1441. case "${status}" in
  1442. WARNING) emoji="⚠️" ;;
  1443. CRITICAL) emoji="🔴" ;;
  1444. CLEAR) emoji="✅" ;;
  1445. *) emoji="⚪️" ;;
  1446. esac
  1447. for room in ${rooms}; do
  1448. webhook="$homeserver/_matrix/client/r0/rooms/$(urlencode $room)/send/m.room.message?access_token=$accesstoken"
  1449. payload="$(
  1450. cat <<EOF
  1451. {
  1452. "msgtype": "m.notice",
  1453. "format": "org.matrix.custom.html",
  1454. "formatted_body": "${emoji} ${host} ${status_message} - <b>${name//_/ }</b><br>${chart} (${family})<br><a href=\"${goto_url}\">${alarm}</a><br><i>${info}</i>",
  1455. "body": "${emoji} ${host} ${status_message} - ${name//_/ } ${chart} (${family}) ${goto_url} ${alarm} ${info}"
  1456. }
  1457. EOF
  1458. )"
  1459. httpcode=$(docurl -X POST --data "${payload}" "${webhook}")
  1460. if [ "${httpcode}" == "200" ]; then
  1461. info "sent Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1462. sent=$((sent + 1))
  1463. else
  1464. error "failed to send Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP error code ${httpcode}."
  1465. fi
  1466. done
  1467. [ ${sent} -gt 0 ] && return 0
  1468. return 1
  1469. }
  1470. # -----------------------------------------------------------------------------
  1471. # syslog sender
  1472. send_syslog() {
  1473. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1474. local priority='' message='' host='' port='' prefix=''
  1475. local temp1='' temp2=''
  1476. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1477. if [ "${status}" = "CRITICAL" ]; then
  1478. level='crit'
  1479. elif [ "${status}" = "WARNING" ]; then
  1480. level='warning'
  1481. fi
  1482. for target in ${targets}; do
  1483. priority="${facility}.${level}"
  1484. message=''
  1485. host=''
  1486. port=''
  1487. prefix=''
  1488. temp1=''
  1489. temp2=''
  1490. prefix=$(echo ${target} | cut -d '/' -f 2)
  1491. temp1=$(echo ${target} | cut -d '/' -f 1)
  1492. if [ ${prefix} != ${temp1} ]; then
  1493. if (echo ${temp1} | grep -q '@'); then
  1494. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1495. host=$(echo ${temp1} | cut -d '@' -f 2)
  1496. if [ ${temp2} != ${host} ]; then
  1497. priority=${temp2}
  1498. fi
  1499. port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
  1500. if (echo ${host} | grep -E -q '\[.*\]'); then
  1501. if (echo ${port} | grep -q ']'); then
  1502. port=''
  1503. else
  1504. host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
  1505. fi
  1506. else
  1507. if [ ${port} = ${host} ]; then
  1508. port=''
  1509. else
  1510. host=$(echo ${host} | cut -d ':' -f 1)
  1511. fi
  1512. fi
  1513. else
  1514. priority=${temp1}
  1515. fi
  1516. fi
  1517. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1518. if [ ${host} ]; then
  1519. logger_options="${logger_options} -n ${host}"
  1520. if [ ${port} ]; then
  1521. logger_options="${logger_options} -P ${port}"
  1522. fi
  1523. fi
  1524. ${logger} -p ${priority} ${logger_options} "${message}"
  1525. done
  1526. return $?
  1527. }
  1528. # -----------------------------------------------------------------------------
  1529. # SMS sender
  1530. send_sms() {
  1531. local recipients="${1}" errcode errmessage sent=0
  1532. # Human readable SMS
  1533. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1534. # limit it to 160 characters
  1535. msg="${msg:0:160}"
  1536. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1537. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1538. for phone in ${recipients}; do
  1539. errmessage=$($sendsms $phone "$msg" 2>&1)
  1540. errcode=$?
  1541. if [ ${errcode} -eq 0 ]; then
  1542. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1543. sent=$((sent + 1))
  1544. else
  1545. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1546. fi
  1547. done
  1548. [ ${sent} -gt 0 ] && return 0
  1549. fi
  1550. return 1
  1551. }
  1552. # -----------------------------------------------------------------------------
  1553. # hangouts sender
  1554. send_hangouts() {
  1555. local rooms="${1}" httpcode sent=0 room color payload webhook
  1556. [ "${SEND_HANGOUTS}" != "YES" ] && return 1
  1557. case "${status}" in
  1558. WARNING) color="#ffa700" ;;
  1559. CRITICAL) color="#d62d20" ;;
  1560. CLEAR) color="#008744" ;;
  1561. *) color="#777777" ;;
  1562. esac
  1563. for room in ${rooms}; do
  1564. if [ -z "${HANGOUTS_WEBHOOK_URI[$room]}" ] ; then
  1565. info "Can't send Hangouts notification for: ${host} ${chart}.${name} to room ${room}. HANGOUTS_WEBHOOK_URI[$room] not defined"
  1566. else
  1567. webhook="${HANGOUTS_WEBHOOK_URI[$room]}"
  1568. payload="$(
  1569. cat <<EOF
  1570. {
  1571. "cards": [
  1572. {
  1573. "header": {
  1574. "title": "Netdata on ${host}",
  1575. "imageUrl": "${images_base_url}/images/banner-icon-144x144.png",
  1576. "imageStyle": "IMAGE"
  1577. },
  1578. "sections": [
  1579. {
  1580. "header": "<b>${host}</b>",
  1581. "widgets": [
  1582. {
  1583. "keyValue": {
  1584. "topLabel": "Status Message",
  1585. "content": "<b>${status_message}</b>",
  1586. "contentMultiline": "true",
  1587. "iconUrl": "${image}",
  1588. "onClick": {
  1589. "openLink": {
  1590. "url": "${goto_url}"
  1591. }
  1592. }
  1593. }
  1594. },
  1595. {
  1596. "keyValue": {
  1597. "topLabel": "${chart} | ${family}",
  1598. "content": "<font color=${color}>${alarm}</font>",
  1599. "contentMultiline": "true"
  1600. }
  1601. }
  1602. ]
  1603. },
  1604. {
  1605. "widgets": [
  1606. {
  1607. "textParagraph": {
  1608. "text": "<font color=\"#0057e7\">@ ${date}\n<b>${info}</b></font>"
  1609. }
  1610. }
  1611. ]
  1612. },
  1613. {
  1614. "widgets": [
  1615. {
  1616. "buttons": [
  1617. {
  1618. "textButton": {
  1619. "text": "Go to ${host}",
  1620. "onClick": {
  1621. "openLink": {
  1622. "url": "${goto_url}"
  1623. }
  1624. }
  1625. }
  1626. }
  1627. ]
  1628. }
  1629. ]
  1630. }
  1631. ]
  1632. }
  1633. ]
  1634. }
  1635. EOF
  1636. )"
  1637. httpcode=$(docurl -H "Content-Type: application/json" -X POST -d "${payload}" "${webhook}")
  1638. if [ "${httpcode}" = "200" ]; then
  1639. info "sent hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1640. sent=$((sent + 1))
  1641. else
  1642. error "failed to send hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP error code ${httpcode}."
  1643. fi
  1644. fi
  1645. done
  1646. [ ${sent} -gt 0 ] && return 0
  1647. return 1
  1648. }
  1649. # -----------------------------------------------------------------------------
  1650. # Dynatrace sender
  1651. send_dynatrace() {
  1652. [ "${SEND_DYNATRACE}" != "YES" ] && return 1
  1653. local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
  1654. local description="NetData Notification for: ${host} ${chart}.${name} is ${status}"
  1655. local payload=""
  1656. payload=$(cat <<EOF
  1657. {
  1658. "title": "NetData Alarm from ${host}",
  1659. "source" : "${DYNATRACE_ANNOTATION_TYPE}",
  1660. "description" : "${description}",
  1661. "eventType": "${DYNATRACE_EVENT}",
  1662. "attachRules":{
  1663. "tagRule":[{
  1664. "meTypes":["HOST"],
  1665. "tags":["${DYNATRACE_TAG_VALUE}"]
  1666. }]
  1667. },
  1668. "customProperties":{
  1669. "description": "${description}"
  1670. }
  1671. }
  1672. EOF
  1673. )
  1674. # echo ${payload}
  1675. httpcode=$(docurl -X POST -H "Authorization: Api-token ${DYNATRACE_TOKEN}" -H "Content-Type: application/json" -d "${payload}" ${dynatrace_url})
  1676. ret=$?
  1677. if [ ${ret} -eq 0 ]; then
  1678. if [ "${httpcode}" = "200" ]; then
  1679. info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
  1680. return 0
  1681. else
  1682. warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
  1683. return 1
  1684. fi
  1685. else
  1686. error "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to ${DYNATRACE_SERVER} with error code ${ret}."
  1687. return 1
  1688. fi
  1689. }
  1690. # -----------------------------------------------------------------------------
  1691. # prepare the content of the notification
  1692. # the url to send the user on click
  1693. urlencode "${args_host}" >/dev/null
  1694. url_host="${REPLY}"
  1695. urlencode "${chart}" >/dev/null
  1696. url_chart="${REPLY}"
  1697. urlencode "${family}" >/dev/null
  1698. url_family="${REPLY}"
  1699. urlencode "${name}" >/dev/null
  1700. url_name="${REPLY}"
  1701. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}"
  1702. GOTOCLOUD=0
  1703. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1704. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1705. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1706. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1707. fi
  1708. fi
  1709. if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1710. GOTOCLOUD=1
  1711. fi
  1712. fi
  1713. if [ ${GOTOCLOUD} -eq 0 ]; then
  1714. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  1715. else
  1716. # Temporarily disable alarm redirection, as the cloud endpoint no longer exists. This functionality will be restored after discussion on #9487. For now, just lead to netdata.cloud
  1717. #goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  1718. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}"
  1719. fi
  1720. # the severity of the alarm
  1721. severity="${status}"
  1722. # the time the alarm was raised
  1723. duration4human ${duration} >/dev/null
  1724. duration_txt="${REPLY}"
  1725. duration4human ${non_clear_duration} >/dev/null
  1726. non_clear_duration_txt="${REPLY}"
  1727. raised_for="(was ${old_status,,} for ${duration_txt})"
  1728. # the key status message
  1729. status_message="status unknown"
  1730. # the color of the alarm
  1731. color="grey"
  1732. # the alarm value
  1733. alarm="${name//_/ } = ${value_string}"
  1734. # the image of the alarm
  1735. image="${images_base_url}/images/banner-icon-144x144.png"
  1736. # prepare the title based on status
  1737. case "${status}" in
  1738. CRITICAL)
  1739. image="${images_base_url}/images/alert-128-red.png"
  1740. status_message="is critical"
  1741. color="#ca414b"
  1742. ;;
  1743. WARNING)
  1744. image="${images_base_url}/images/alert-128-orange.png"
  1745. status_message="needs attention"
  1746. color="#ffc107"
  1747. ;;
  1748. CLEAR)
  1749. image="${images_base_url}/images/check-mark-2-128-green.png"
  1750. status_message="recovered"
  1751. color="#77ca6d"
  1752. ;;
  1753. esac
  1754. if [ "${status}" = "CLEAR" ]; then
  1755. severity="Recovered from ${old_status}"
  1756. if [ ${non_clear_duration} -gt ${duration} ]; then
  1757. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  1758. fi
  1759. # don't show the value when the status is CLEAR
  1760. # for certain alarms, this value might not have any meaning
  1761. alarm="${name//_/ } ${raised_for}"
  1762. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  1763. severity="Escalated to ${status}"
  1764. if [ ${non_clear_duration} -gt ${duration} ]; then
  1765. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1766. fi
  1767. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  1768. severity="Demoted to ${status}"
  1769. if [ ${non_clear_duration} -gt ${duration} ]; then
  1770. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  1771. fi
  1772. else
  1773. raised_for=
  1774. fi
  1775. # prepare HTML versions of elements
  1776. info_html=
  1777. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  1778. raised_for_html=
  1779. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  1780. # -----------------------------------------------------------------------------
  1781. # send the slack notification
  1782. # slack aggregates posts from the same username
  1783. # so we use "${host} ${status}" as the bot username, to make them diff
  1784. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  1785. SENT_SLACK=$?
  1786. # -----------------------------------------------------------------------------
  1787. # send the hangouts notification
  1788. # hangouts aggregates posts from the same room
  1789. # so we use "${host} ${status}" as the room, to make them diff
  1790. send_hangouts "${to_hangouts}"
  1791. SENT_HANGOUTS=$?
  1792. # -----------------------------------------------------------------------------
  1793. # send the Microsoft notification
  1794. # Microsoft team aggregates posts from the same username
  1795. # so we use "${host} ${status}" as the bot username, to make them diff
  1796. send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
  1797. SENT_MSTEAM=$?
  1798. # -----------------------------------------------------------------------------
  1799. # send the rocketchat notification
  1800. # rocketchat aggregates posts from the same username
  1801. # so we use "${host} ${status}" as the bot username, to make them diff
  1802. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  1803. SENT_ROCKETCHAT=$?
  1804. # -----------------------------------------------------------------------------
  1805. # send the alerta notification
  1806. # alerta aggregates posts from the same username
  1807. # so we use "${host} ${status}" as the bot username, to make them diff
  1808. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  1809. SENT_ALERTA=$?
  1810. # -----------------------------------------------------------------------------
  1811. # send the flock notification
  1812. # flock aggregates posts from the same username
  1813. # so we use "${host} ${status}" as the bot username, to make them diff
  1814. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  1815. SENT_FLOCK=$?
  1816. # -----------------------------------------------------------------------------
  1817. # send the discord notification
  1818. # discord aggregates posts from the same username
  1819. # so we use "${host} ${status}" as the bot username, to make them diff
  1820. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  1821. SENT_DISCORD=$?
  1822. # -----------------------------------------------------------------------------
  1823. # send the pushover notification
  1824. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  1825. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  1826. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  1827. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  1828. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  1829. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  1830. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  1831. <small><small>The source of this alarm is line ${src}</small></small>
  1832. "
  1833. SENT_PUSHOVER=$?
  1834. # -----------------------------------------------------------------------------
  1835. # send the pushbullet notification
  1836. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  1837. Severity: ${severity}\\n
  1838. Chart: ${chart}\\n
  1839. Family: ${family}\\n
  1840. ${date}\\n
  1841. The source of this alarm is line ${src}"
  1842. SENT_PUSHBULLET=$?
  1843. # -----------------------------------------------------------------------------
  1844. # send the twilio SMS
  1845. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1846. Severity: ${severity}
  1847. Chart: ${chart}
  1848. Family: ${family}
  1849. ${info}"
  1850. SENT_TWILIO=$?
  1851. # -----------------------------------------------------------------------------
  1852. # send the messagebird SMS
  1853. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1854. Severity: ${severity}
  1855. Chart: ${chart}
  1856. Family: ${family}
  1857. ${info}"
  1858. SENT_MESSAGEBIRD=$?
  1859. # -----------------------------------------------------------------------------
  1860. # send the kavenegar SMS
  1861. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  1862. Severity: ${severity}
  1863. Chart: ${chart}
  1864. Family: ${family}
  1865. ${info}"
  1866. SENT_KAVENEGAR=$?
  1867. # -----------------------------------------------------------------------------
  1868. # send the telegram.org message
  1869. # https://core.telegram.org/bots/api#formatting-options
  1870. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  1871. ${chart} (${family})
  1872. <a href=\"${goto_url}\">${alarm}</a>
  1873. <i>${info}</i>"
  1874. SENT_TELEGRAM=$?
  1875. # -----------------------------------------------------------------------------
  1876. # send the kafka message
  1877. send_kafka
  1878. SENT_KAFKA=$?
  1879. # -----------------------------------------------------------------------------
  1880. # send the pagerduty.com message
  1881. send_pd "${to_pd}"
  1882. SENT_PD=$?
  1883. # -----------------------------------------------------------------------------
  1884. # send the fleep message
  1885. send_fleep "${to_fleep}"
  1886. SENT_FLEEP=$?
  1887. # -----------------------------------------------------------------------------
  1888. # send the Prowl message
  1889. send_prowl "${to_prowl}"
  1890. SENT_PROWL=$?
  1891. # -----------------------------------------------------------------------------
  1892. # send the irc message
  1893. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  1894. Severity: ${severity}
  1895. Chart: ${chart}
  1896. Family: ${family}
  1897. ${info}"
  1898. SENT_IRC=$?
  1899. # -----------------------------------------------------------------------------
  1900. # send the SMS message with smstools3
  1901. send_sms "${to_sms}"
  1902. SENT_SMS=$?
  1903. # -----------------------------------------------------------------------------
  1904. # send the custom message
  1905. send_custom() {
  1906. # is it enabled?
  1907. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  1908. # do we have any sender?
  1909. [ -z "${1}" ] && return 1
  1910. # call the custom_sender function
  1911. custom_sender "${@}"
  1912. }
  1913. send_custom "${to_custom}"
  1914. SENT_CUSTOM=$?
  1915. # -----------------------------------------------------------------------------
  1916. # send hipchat message
  1917. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  1918. ${host} ${status_message}<br/> \
  1919. <b>${alarm}</b> ${info_html}<br/> \
  1920. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  1921. <b>${date}${raised_for_html}</b><br/> \
  1922. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  1923. (source of alarm ${src}) \
  1924. "
  1925. SENT_HIPCHAT=$?
  1926. # -----------------------------------------------------------------------------
  1927. # send the Amazon SNS message
  1928. send_awssns "${to_awssns}"
  1929. SENT_AWSSNS=$?
  1930. # -----------------------------------------------------------------------------
  1931. # send the Matrix message
  1932. send_matrix "${MATRIX_HOMESERVER}" "${to_matrix}"
  1933. SENT_MATRIX=$?
  1934. # -----------------------------------------------------------------------------
  1935. # send the syslog message
  1936. send_syslog "${to_syslog}"
  1937. SENT_SYSLOG=$?
  1938. # -----------------------------------------------------------------------------
  1939. # send the email
  1940. IFS='' read -r -d '' email_plaintext_part <<EOF
  1941. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  1942. Content-Disposition: inline
  1943. Content-Transfer-Encoding: 8bit
  1944. ${host} ${status_message}
  1945. ${alarm} ${info}
  1946. ${raised_for}
  1947. Chart : ${chart}
  1948. Family : ${family}
  1949. Severity: ${severity}
  1950. URL : ${goto_url}
  1951. Source : ${src}
  1952. Date : ${date}
  1953. Notification generated on ${host}
  1954. Evaluated Expression : ${calc_expression}
  1955. Expression Variables : ${calc_param_values}
  1956. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  1957. EOF
  1958. if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
  1959. send_email <<EOF
  1960. To: ${to_email}
  1961. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  1962. MIME-Version: 1.0
  1963. Content-Type: multipart/alternative; boundary="multipart-boundary"
  1964. ${email_thread_headers}
  1965. This is a MIME-encoded multipart message
  1966. --multipart-boundary
  1967. ${email_plaintext_part}
  1968. --multipart-boundary--
  1969. EOF
  1970. else
  1971. IFS='' read -r -d '' email_html_part <<EOF
  1972. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  1973. Content-Disposition: inline
  1974. Content-Transfer-Encoding: 8bit
  1975. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  1976. <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
  1977. <body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
  1978. <table>
  1979. <tbody>
  1980. <tr>
  1981. <td style="vertical-align: top;" valign="top"></td>
  1982. <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
  1983. <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
  1984. <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
  1985. <tbody>
  1986. <tr>
  1987. <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
  1988. <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
  1989. </td>
  1990. </tr>
  1991. <tr>
  1992. <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
  1993. <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
  1994. </td>
  1995. </tr>
  1996. <tr>
  1997. <td style="vertical-align: top;" valign="top">
  1998. <div style="margin: 0; padding: 20px; max-width: 700px;">
  1999. <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
  2000. <tbody>
  2001. <tr>
  2002. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
  2003. <span>${chart}</span>
  2004. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
  2005. </td>
  2006. </tr>
  2007. <tr style="margin: 0; padding: 0;">
  2008. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2009. <span><b>${alarm}</b>${info_html}</span>
  2010. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
  2011. </td>
  2012. </tr>
  2013. <tr>
  2014. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2015. <span>${family}</span>
  2016. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
  2017. </td>
  2018. </tr>
  2019. <tr style="margin: 0; padding: 0;">
  2020. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2021. <span>${severity}</span>
  2022. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
  2023. </td>
  2024. </tr>
  2025. <tr style="margin: 0; padding: 0;">
  2026. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
  2027. <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
  2028. </td>
  2029. </tr>
  2030. <tr style="margin: 0; padding: 0;">
  2031. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2032. <span>${calc_expression}</span>
  2033. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
  2034. </td>
  2035. </tr>
  2036. <tr style="margin: 0; padding: 0;">
  2037. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2038. <span>${calc_param_values}</span>
  2039. <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
  2040. </td>
  2041. </tr>
  2042. <tr style="margin: 0; padding: 0;">
  2043. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
  2044. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  2045. </td>
  2046. </tr>
  2047. <tr style="margin: 0; padding: 0;">
  2048. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
  2049. <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
  2050. </td>
  2051. </tr>
  2052. <tr style="text-align: center; margin: 0; padding: 0;">
  2053. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
  2054. </td>
  2055. </tr>
  2056. <tr style="text-align: center; margin: 0; padding: 0;">
  2057. <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
  2058. <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
  2059. </td>
  2060. </tr>
  2061. </tbody>
  2062. </table>
  2063. </div>
  2064. </td>
  2065. </tr>
  2066. </tbody>
  2067. </table>
  2068. </div>
  2069. </td>
  2070. </tr>
  2071. </tbody>
  2072. </table>
  2073. </body>
  2074. </html>
  2075. EOF
  2076. send_email <<EOF
  2077. To: ${to_email}
  2078. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  2079. MIME-Version: 1.0
  2080. Content-Type: multipart/alternative; boundary="multipart-boundary"
  2081. ${email_thread_headers}
  2082. This is a MIME-encoded multipart message
  2083. --multipart-boundary
  2084. ${email_plaintext_part}
  2085. --multipart-boundary
  2086. ${email_html_part}
  2087. --multipart-boundary--
  2088. EOF
  2089. fi
  2090. SENT_EMAIL=$?
  2091. # -----------------------------------------------------------------------------
  2092. # send the EVENT to Dynatrace
  2093. send_dynatrace "${host}" "${chart}" "${name}" "${status}"
  2094. SENT_DYNATRACE=$?
  2095. # -----------------------------------------------------------------------------
  2096. # let netdata know
  2097. for state in "${SENT_EMAIL}" \
  2098. "${SENT_PUSHOVER}" \
  2099. "${SENT_TELEGRAM}" \
  2100. "${SENT_SLACK}" \
  2101. "${SENT_HANGOUTS}" \
  2102. "${SENT_ROCKETCHAT}" \
  2103. "${SENT_ALERTA}" \
  2104. "${SENT_FLOCK}" \
  2105. "${SENT_DISCORD}" \
  2106. "${SENT_TWILIO}" \
  2107. "${SENT_HIPCHAT}" \
  2108. "${SENT_MESSAGEBIRD}" \
  2109. "${SENT_KAVENEGAR}" \
  2110. "${SENT_PUSHBULLET}" \
  2111. "${SENT_KAFKA}" \
  2112. "${SENT_PD}" \
  2113. "${SENT_FLEEP}" \
  2114. "${SENT_PROWL}" \
  2115. "${SENT_CUSTOM}" \
  2116. "${SENT_IRC}" \
  2117. "${SENT_AWSSNS}" \
  2118. "${SENT_MATRIX}" \
  2119. "${SENT_SYSLOG}" \
  2120. "${SENT_SMS}" \
  2121. "${SENT_MSTEAM}" \
  2122. "${SENT_DYNATRACE}"; do
  2123. if [ "${state}" -eq 0 ]; then
  2124. # we sent something
  2125. exit 0
  2126. fi
  2127. done
  2128. # we did not send anything
  2129. exit 1