alarm-notify.sh.in 134 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - irc notifications by @manosf
  32. # - custom notifications by @ktsaou
  33. # - syslog messages by @Ferroin
  34. # - Microsoft Team notification by @tioumen
  35. # - RocketChat notifications by @Hermsi1337 #3777
  36. # - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
  37. # - Dynatrace Event by @illumine
  38. # - Stackpulse Event by @thiagoftsm
  39. # - Opsgenie by @thiaoftsm #9858
  40. # - Gotify by @coffeegrind123
  41. # -----------------------------------------------------------------------------
  42. # testing notifications
  43. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  44. if [ "${2}" = "test" ]; then
  45. recipient="${1}"
  46. else
  47. recipient="${2}"
  48. fi
  49. [ -z "${recipient}" ] && recipient="sysadmin"
  50. id=1
  51. last="CLEAR"
  52. test_res=0
  53. for x in "WARNING" "CRITICAL" "CLEAR"; do
  54. echo >&2
  55. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  56. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  57. #shellcheck disable=SC2181
  58. if [ $? -ne 0 ]; then
  59. echo >&2 "# FAILED"
  60. test_res=1
  61. else
  62. echo >&2 "# OK"
  63. fi
  64. last="${x}"
  65. id=$((id + 1))
  66. done
  67. exit $test_res
  68. fi
  69. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  70. export LC_ALL=C
  71. # -----------------------------------------------------------------------------
  72. PROGRAM_NAME="$(basename "${0}")"
  73. logdate() {
  74. date "+%Y-%m-%d %H:%M:%S"
  75. }
  76. log() {
  77. local status="${1}"
  78. shift
  79. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  80. }
  81. warning() {
  82. log WARNING "${@}"
  83. }
  84. error() {
  85. log ERROR "${@}"
  86. }
  87. info() {
  88. log INFO "${@}"
  89. }
  90. fatal() {
  91. log FATAL "${@}"
  92. exit 1
  93. }
  94. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  95. debug() {
  96. [ "${debug}" = "1" ] && log DEBUG "${@}"
  97. }
  98. docurl() {
  99. if [ -z "${curl}" ]; then
  100. error "${curl} is unset."
  101. return 1
  102. fi
  103. if [ "${debug}" = "1" ]; then
  104. echo >&2 "--- BEGIN curl command ---"
  105. printf >&2 "%q " ${curl} "${@}"
  106. echo >&2
  107. echo >&2 "--- END curl command ---"
  108. local out code ret
  109. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  110. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  111. ret=$?
  112. echo >&2 "--- BEGIN received response ---"
  113. cat >&2 "${out}"
  114. echo >&2
  115. echo >&2 "--- END received response ---"
  116. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  117. rm "${out}"
  118. echo "${code}"
  119. return ${ret}
  120. fi
  121. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  122. return $?
  123. }
  124. # -----------------------------------------------------------------------------
  125. # List of all the notification mechanisms we support.
  126. # Used in a couple of places to write more compact code.
  127. method_names="
  128. email
  129. pushover
  130. pushbullet
  131. telegram
  132. slack
  133. alerta
  134. flock
  135. discord
  136. hipchat
  137. twilio
  138. messagebird
  139. pd
  140. fleep
  141. syslog
  142. custom
  143. msteams
  144. kavenegar
  145. prowl
  146. irc
  147. awssns
  148. rocketchat
  149. sms
  150. hangouts
  151. dynatrace
  152. matrix
  153. "
  154. # -----------------------------------------------------------------------------
  155. # this is to be overwritten by the config file
  156. custom_sender() {
  157. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  158. }
  159. # -----------------------------------------------------------------------------
  160. # check for BASH v4+ (required for associative arrays)
  161. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  162. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  163. fi
  164. # -----------------------------------------------------------------------------
  165. # defaults to allow running this script by hand
  166. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  167. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  168. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  169. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  170. [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
  171. # -----------------------------------------------------------------------------
  172. # parse command line parameters
  173. if [[ ${1} = "unittest" ]]; then
  174. unittest=1 # enable unit testing mode
  175. roles="${2}" # the role that should be used for unit testing
  176. cfgfile="${3}" # the location of the config file to use for unit testing
  177. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  178. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  179. elif [[ ${1} = "dump_methods" ]]; then
  180. dump_methods=1
  181. status="WARNING"
  182. else
  183. roles="${1}" # the roles that should be notified for this event
  184. args_host="${2}" # the host generated this event
  185. unique_id="${3}" # the unique id of this event
  186. alarm_id="${4}" # the unique id of the alarm that generated this event
  187. event_id="${5}" # the incremental id of the event, for this alarm id
  188. when="${6}" # the timestamp this event occurred
  189. name="${7}" # the name of the alarm, as given in netdata health.d entries
  190. chart="${8}" # the name of the chart (type.id)
  191. family="${9}" # the family of the chart
  192. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  193. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  194. value="${12}" # the current value of the alarm
  195. old_value="${13}" # the previous value of the alarm
  196. src="${14}" # the line number and file the alarm has been configured
  197. duration="${15}" # the duration in seconds of the previous alarm state
  198. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  199. units="${17}" # the units of the value
  200. info="${18}" # a short description of the alarm
  201. value_string="${19}" # friendly value (with units)
  202. # shellcheck disable=SC2034
  203. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  204. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  205. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  206. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  207. total_warnings="${23}" # Total number of alarms in WARNING state
  208. total_critical="${24}" # Total number of alarms in CRITICAL state
  209. total_warn_alarms="${25}" # List of alarms in warning state
  210. total_crit_alarms="${26}" # List of alarms in critical state
  211. classification="${27}" # The class field from .conf files
  212. edit_command_line="${28}" # The command to edit the alarm, with the line number
  213. child_machine_guid="${29}" # If populated, the notification is sent for a child
  214. fi
  215. # -----------------------------------------------------------------------------
  216. # find a suitable hostname to use, if netdata did not supply a hostname
  217. if [ -z ${args_host} ]; then
  218. this_host=$(hostname -s 2>/dev/null)
  219. host="${this_host}"
  220. args_host="${this_host}"
  221. else
  222. host="${args_host}"
  223. fi
  224. # -----------------------------------------------------------------------------
  225. # screen statuses we don't need to send a notification
  226. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  227. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  228. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  229. exit 1
  230. fi
  231. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  232. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  233. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  234. exit 1
  235. fi
  236. # -----------------------------------------------------------------------------
  237. # load configuration
  238. # By default fetch images from the global public registry.
  239. # This is required by default, since all notification methods need to download
  240. # images via the Internet, and private registries might not be reachable.
  241. # This can be overwritten at the configuration file.
  242. images_base_url="https://registry.my-netdata.io"
  243. # curl options to use
  244. curl_options=""
  245. # hostname handling
  246. use_fqdn="NO"
  247. # needed commands
  248. # if empty they will be searched in the system path
  249. curl=
  250. sendmail=
  251. # enable / disable features
  252. for method_name in ${method_names^^}; do
  253. declare SEND_${method_name}="YES"
  254. declare DEFAULT_RECIPIENT_${method_name}
  255. done
  256. for method_name in ${method_names}; do
  257. declare -A role_recipients_${method_name}
  258. done
  259. # slack configs
  260. SLACK_WEBHOOK_URL=
  261. # Microsoft Teams configs
  262. MSTEAMS_WEBHOOK_URL=
  263. # Legacy Microsoft Teams configs for backwards compatibility:
  264. declare -A role_recipients_msteam
  265. # rocketchat configs
  266. ROCKETCHAT_WEBHOOK_URL=
  267. # alerta configs
  268. ALERTA_WEBHOOK_URL=
  269. ALERTA_API_KEY=
  270. # flock configs
  271. FLOCK_WEBHOOK_URL=
  272. # discord configs
  273. DISCORD_WEBHOOK_URL=
  274. # pushover configs
  275. PUSHOVER_APP_TOKEN=
  276. # pushbullet configs
  277. PUSHBULLET_ACCESS_TOKEN=
  278. PUSHBULLET_SOURCE_DEVICE=
  279. # twilio configs
  280. TWILIO_ACCOUNT_SID=
  281. TWILIO_ACCOUNT_TOKEN=
  282. TWILIO_NUMBER=
  283. # hipchat configs
  284. HIPCHAT_SERVER=
  285. HIPCHAT_AUTH_TOKEN=
  286. # messagebird configs
  287. MESSAGEBIRD_ACCESS_KEY=
  288. MESSAGEBIRD_NUMBER=
  289. # kavenegar configs
  290. KAVENEGAR_API_KEY=
  291. KAVENEGAR_SENDER=
  292. # telegram configs
  293. TELEGRAM_BOT_TOKEN=
  294. # kafka configs
  295. SEND_KAFKA="YES"
  296. KAFKA_URL=
  297. KAFKA_SENDER_IP=
  298. # pagerduty.com configs
  299. PD_SERVICE_KEY=
  300. USE_PD_VERSION=
  301. # fleep.io configs
  302. FLEEP_SENDER="${host}"
  303. # Amazon SNS configs
  304. AWSSNS_MESSAGE_FORMAT=
  305. # Matrix configs
  306. MATRIX_HOMESERVER=
  307. MATRIX_ACCESSTOKEN=
  308. # syslog configs
  309. SYSLOG_FACILITY=
  310. # email configs
  311. EMAIL_SENDER=
  312. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  313. EMAIL_THREADING=
  314. EMAIL_PLAINTEXT_ONLY=
  315. # irc configs
  316. IRC_NICKNAME=
  317. IRC_REALNAME=
  318. IRC_NETWORK=
  319. IRC_PORT=6667
  320. # hangouts configs
  321. declare -A HANGOUTS_WEBHOOK_URI
  322. declare -A HANGOUTS_WEBHOOK_THREAD
  323. # dynatrace configs
  324. DYNATRACE_SPACE=
  325. DYNATRACE_SERVER=
  326. DYNATRACE_TOKEN=
  327. DYNATRACE_TAG_VALUE=
  328. DYNATRACE_ANNOTATION_TYPE=
  329. DYNATRACE_EVENT=
  330. SEND_DYNATRACE=
  331. # stackpulse configs
  332. STACKPULSE_WEBHOOK=
  333. # gotify configs
  334. GOTIFY_APP_URL=
  335. GOTIFY_APP_TOKEN=
  336. # opsgenie configs
  337. OPSGENIE_API_KEY=
  338. # load the stock and user configuration files
  339. # these will overwrite the variables above
  340. if [ ${unittest} ]; then
  341. if source "${cfgfile}"; then
  342. error "Failed to load requested config file."
  343. exit 1
  344. fi
  345. else
  346. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  347. if [ -f "${CONFIG}" ]; then
  348. debug "Loading config file '${CONFIG}'..."
  349. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  350. else
  351. warning "Cannot find file '${CONFIG}'."
  352. fi
  353. done
  354. fi
  355. if [[ ! $curl_options =~ .*\--connect-timeout ]]; then
  356. curl_options+=" --connect-timeout 5"
  357. fi
  358. OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"}
  359. # If we didn't autodetect the character set for e-mail and it wasn't
  360. # set by the user, we need to set it to a reasonable default. UTF-8
  361. # should be correct for almost all modern UNIX systems.
  362. if [ -z ${EMAIL_CHARSET} ]; then
  363. EMAIL_CHARSET="UTF-8"
  364. fi
  365. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  366. # unless we're sending an alarm for a child system which we can't get the
  367. # FQDN of easily.
  368. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  369. host="$(hostname -f 2>/dev/null)"
  370. fi
  371. # -----------------------------------------------------------------------------
  372. # migrate old Microsoft Teams configuration keys after loading configuration
  373. msteams_migration() {
  374. SEND_MSTEAMS=${SEND_MSTEAM:-$SEND_MSTEAMS}
  375. unset -v SEND_MSTEAM
  376. DEFAULT_RECIPIENT_MSTEAMS=${DEFAULT_RECIPIENT_MSTEAM:-$DEFAULT_RECIPIENT_MSTEAMS}
  377. MSTEAMS_WEBHOOK_URL=${MSTEAM_WEBHOOK_URL:-$MSTEAMS_WEBHOOK_URL}
  378. MSTEAMS_ICON_DEFAULT=${MSTEAM_ICON_DEFAULT:-$MSTEAMS_ICON_DEFAULT}
  379. MSTEAMS_ICON_CLEAR=${MSTEAM_ICON_CLEAR:-$MSTEAMS_ICON_CLEAR}
  380. MSTEAMS_ICON_WARNING=${MSTEAM_ICON_WARNING:-$MSTEAMS_ICON_WARNING}
  381. MSTEAMS_ICON_CRITICAL=${MSTEAM_ICON_CRITICAL:-$MSTEAMS_ICON_CRITICAL}
  382. MSTEAMS_COLOR_DEFAULT=${MSTEAM_COLOR_DEFAULT:-$MSTEAMS_COLOR_DEFAULT}
  383. MSTEAMS_COLOR_CLEAR=${MSTEAM_COLOR_CLEAR:-$MSTEAMS_COLOR_CLEAR}
  384. MSTEAMS_COLOR_WARNING=${MSTEAM_COLOR_WARNING:-$MSTEAMS_COLOR_WARNING}
  385. MSTEAMS_COLOR_CRITICAL=${MSTEAM_COLOR_CRITICAL:-$MSTEAMS_COLOR_CRITICAL}
  386. # migrate role specific recipients:
  387. for key in "${!role_recipients_msteam[@]}"; do
  388. # Disable check, if role_recipients_msteams is ever used:
  389. # The role_recipients_$method are created and used programmatically
  390. # by iterating over $methods. shellcheck therefore doesn't realize
  391. # that role_recipients_msteams is actually used in the block
  392. # "find the recipients' addresses per method".
  393. # shellcheck disable=SC2034
  394. role_recipients_msteams["$key"]="${role_recipients_msteam["$key"]}"
  395. done
  396. }
  397. msteams_migration
  398. # -----------------------------------------------------------------------------
  399. # filter a recipient based on alarm event severity
  400. filter_recipient_by_criticality() {
  401. local method="${1}" x="${2}" r s
  402. shift
  403. r="${x/|*/}" # the recipient
  404. s="${x/*|/}" # the severity required for notifying this recipient
  405. # no severity filtering for this person
  406. [ "${r}" = "${s}" ] && return 0
  407. # the severity is invalid
  408. s="${s^^}"
  409. if [ "${s}" != "CRITICAL" ]; then
  410. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  411. return 0
  412. fi
  413. # create the status tracking directory for this user
  414. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  415. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  416. case "${status}" in
  417. CRITICAL)
  418. # make sure he will get future notifications for this alarm too
  419. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  420. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  421. return 0
  422. ;;
  423. WARNING)
  424. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  425. # we do not remove the file, so that he will get future notifications of this alarm
  426. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  427. return 0
  428. fi
  429. ;;
  430. *)
  431. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  432. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  433. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  434. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  435. return 0
  436. fi
  437. ;;
  438. esac
  439. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  440. return 1
  441. }
  442. # -----------------------------------------------------------------------------
  443. # verify the delivery methods supported
  444. # check slack
  445. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  446. # check rocketchat
  447. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  448. # check alerta
  449. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  450. # check flock
  451. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  452. # check discord
  453. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  454. # check pushover
  455. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  456. # check pushbullet
  457. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  458. # check twilio
  459. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  460. # check hipchat
  461. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  462. # check messagebird
  463. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  464. # check kavenegar
  465. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  466. # check telegram
  467. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  468. # check kafka
  469. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  470. # check irc
  471. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  472. # check hangouts
  473. [ ${#HANGOUTS_WEBHOOK_URI[@]} -eq 0 ] && SEND_HANGOUTS="NO"
  474. # check fleep
  475. #shellcheck disable=SC2153
  476. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  477. # check dynatrace
  478. { [ -z "${DYNATRACE_SPACE}" ] ||
  479. [ -z "${DYNATRACE_SERVER}" ] ||
  480. [ -z "${DYNATRACE_TOKEN}" ] ||
  481. [ -z "${DYNATRACE_TAG_VALUE}" ] ||
  482. [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
  483. # check opsgenie
  484. [ -z "${OPSGENIE_API_KEY}" ] && SEND_OPSGENIE="NO"
  485. # check matrix
  486. { [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
  487. # check gotify
  488. { [ -z "${GOTIFY_APP_TOKEN}" ] || [ -z "${GOTIFY_APP_URL}" ]; } && SEND_GOTIFY="NO"
  489. # check stackpulse
  490. [ -z "${STACKPULSE_WEBHOOK}" ] && SEND_STACKPULSE="NO"
  491. # check msteams
  492. [ -z "${MSTEAMS_WEBHOOK_URL}" ] && SEND_MSTEAMS="NO"
  493. # check pd
  494. [ -z "${DEFAULT_RECIPIENT_PD}" ] && SEND_PD="NO"
  495. # check prowl
  496. [ -z "${DEFAULT_RECIPIENT_PROWL}" ] && SEND_PROWL="NO"
  497. # check custom
  498. [ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"
  499. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  500. [ "${SEND_SLACK}" = "YES" ] ||
  501. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  502. [ "${SEND_ALERTA}" = "YES" ] ||
  503. [ "${SEND_PD}" = "YES" ] ||
  504. [ "${SEND_FLOCK}" = "YES" ] ||
  505. [ "${SEND_DISCORD}" = "YES" ] ||
  506. [ "${SEND_HIPCHAT}" = "YES" ] ||
  507. [ "${SEND_TWILIO}" = "YES" ] ||
  508. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  509. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  510. [ "${SEND_TELEGRAM}" = "YES" ] ||
  511. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  512. [ "${SEND_KAFKA}" = "YES" ] ||
  513. [ "${SEND_FLEEP}" = "YES" ] ||
  514. [ "${SEND_PROWL}" = "YES" ] ||
  515. [ "${SEND_HANGOUTS}" = "YES" ] ||
  516. [ "${SEND_MATRIX}" = "YES" ] ||
  517. [ "${SEND_CUSTOM}" = "YES" ] ||
  518. [ "${SEND_MSTEAMS}" = "YES" ] ||
  519. [ "${SEND_DYNATRACE}" = "YES" ] ||
  520. [ "${SEND_STACKPULSE}" = "YES" ] ||
  521. [ "${SEND_OPSGENIE}" = "YES" ] ||
  522. [ "${SEND_GOTIFY}" = "YES" ]; then
  523. # if we need curl, check for the curl command
  524. if [ -z "${curl}" ]; then
  525. curl="$(command -v curl 2>/dev/null)"
  526. fi
  527. if [ -z "${curl}" ]; then
  528. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  529. SEND_PUSHOVER="NO"
  530. SEND_PUSHBULLET="NO"
  531. SEND_TELEGRAM="NO"
  532. SEND_SLACK="NO"
  533. SEND_MSTEAMS="NO"
  534. SEND_ROCKETCHAT="NO"
  535. SEND_ALERTA="NO"
  536. SEND_PD="NO"
  537. SEND_FLOCK="NO"
  538. SEND_DISCORD="NO"
  539. SEND_TWILIO="NO"
  540. SEND_HIPCHAT="NO"
  541. SEND_MESSAGEBIRD="NO"
  542. SEND_KAVENEGAR="NO"
  543. SEND_KAFKA="NO"
  544. SEND_FLEEP="NO"
  545. SEND_PROWL="NO"
  546. SEND_HANGOUTS="NO"
  547. SEND_MATRIX="NO"
  548. SEND_CUSTOM="NO"
  549. SEND_DYNATRACE="NO"
  550. SEND_STACKPULSE="NO"
  551. SEND_OPSGENIE="NO"
  552. SEND_GOTIFY="NO"
  553. fi
  554. fi
  555. if [ "${SEND_SMS}" = "YES" ]; then
  556. if [ -z "${sendsms}" ]; then
  557. sendsms="$(command -v sendsms 2>/dev/null)"
  558. fi
  559. if [ -z "${sendsms}" ]; then
  560. SEND_SMS="NO"
  561. fi
  562. fi
  563. # if we need sendmail, check for the sendmail command
  564. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  565. sendmail="$(command -v sendmail 2>/dev/null)"
  566. if [ -z "${sendmail}" ]; then
  567. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  568. SEND_EMAIL="NO"
  569. fi
  570. fi
  571. # if we need logger, check for the logger command
  572. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  573. logger="$(command -v logger 2>/dev/null)"
  574. if [ -z "${logger}" ]; then
  575. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  576. SEND_SYSLOG="NO"
  577. fi
  578. fi
  579. # if we need aws, check for the aws command
  580. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  581. aws="$(command -v aws 2>/dev/null)"
  582. if [ -z "${aws}" ]; then
  583. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  584. SEND_AWSSNS="NO"
  585. fi
  586. fi
  587. if [ ${dump_methods} ]; then
  588. for name in "${!SEND_@}"; do
  589. if [ "${!name}" = "YES" ]; then
  590. echo "$name"
  591. fi
  592. done
  593. exit
  594. fi
  595. # -----------------------------------------------------------------------------
  596. # find the recipients' addresses per method
  597. # netdata may call us with multiple roles, and roles may have multiple but
  598. # overlapping recipients - so, here we find the unique recipients.
  599. for method_name in ${method_names}; do
  600. send_var="SEND_${method_name^^}"
  601. if [ "${!send_var}" = "NO" ]; then
  602. continue
  603. fi
  604. declare -A arr_var=()
  605. for x in ${roles//,/ }; do
  606. # the roles 'silent' and 'disabled' mean:
  607. # don't send a notification for this role
  608. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  609. continue
  610. fi
  611. role_recipients="role_recipients_${method_name}[$x]"
  612. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  613. a="${!role_recipients}"
  614. [ -z "${a}" ] && a="${!default_recipient_var}"
  615. for r in ${a//,/ }; do
  616. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  617. done
  618. done
  619. # build the list of recipients
  620. to_var="to_${method_name}"
  621. declare to_${method_name}="${!arr_var[*]}"
  622. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  623. done
  624. # -----------------------------------------------------------------------------
  625. # handle fixup of the email recipient list.
  626. fix_to_email() {
  627. to_email=
  628. while [ -n "${1}" ]; do
  629. [ -n "${to_email}" ] && to_email="${to_email}, "
  630. to_email="${to_email}${1}"
  631. shift 1
  632. done
  633. }
  634. # ${to_email} without quotes here
  635. fix_to_email ${to_email}
  636. # -----------------------------------------------------------------------------
  637. # handle output if we're running in unit test mode
  638. if [ ${unittest} ]; then
  639. for method_name in ${method_names}; do
  640. to_var="to_${method_name}"
  641. echo "results: ${method_name}: ${!to_var}"
  642. done
  643. exit 0
  644. fi
  645. # -----------------------------------------------------------------------------
  646. # check that we have at least a method enabled
  647. proceed=0
  648. for method in "${SEND_EMAIL}" \
  649. "${SEND_PUSHOVER}" \
  650. "${SEND_TELEGRAM}" \
  651. "${SEND_SLACK}" \
  652. "${SEND_ROCKETCHAT}" \
  653. "${SEND_ALERTA}" \
  654. "${SEND_FLOCK}" \
  655. "${SEND_DISCORD}" \
  656. "${SEND_TWILIO}" \
  657. "${SEND_HIPCHAT}" \
  658. "${SEND_MESSAGEBIRD}" \
  659. "${SEND_KAVENEGAR}" \
  660. "${SEND_PUSHBULLET}" \
  661. "${SEND_KAFKA}" \
  662. "${SEND_PD}" \
  663. "${SEND_FLEEP}" \
  664. "${SEND_PROWL}" \
  665. "${SEND_MATRIX}" \
  666. "${SEND_CUSTOM}" \
  667. "${SEND_IRC}" \
  668. "${SEND_HANGOUTS}" \
  669. "${SEND_AWSSNS}" \
  670. "${SEND_SYSLOG}" \
  671. "${SEND_SMS}" \
  672. "${SEND_MSTEAMS}" \
  673. "${SEND_DYNATRACE}" \
  674. "${SEND_STACKPULSE}" \
  675. "${SEND_OPSGENIE}" \
  676. "${SEND_GOTIFY}" ; do
  677. if [ "${method}" == "YES" ]; then
  678. proceed=1
  679. break
  680. fi
  681. done
  682. if [ "$proceed" -eq 0 ]; then
  683. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  684. fi
  685. # -----------------------------------------------------------------------------
  686. # get the date the alarm happened
  687. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  688. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  689. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  690. [ -z "${date}" ] && date=$(date 2>/dev/null)
  691. # -----------------------------------------------------------------------------
  692. # get the date in utc the alarm happened
  693. date_utc=$(date --date=@${when} "${date_format}" -u 2>/dev/null)
  694. [ -z "${date_utc}" ] && date_utc=$(date -u "${date_format}" 2>/dev/null)
  695. [ -z "${date_utc}" ] && date_utc=$(date -u --date=@${when} 2>/dev/null)
  696. [ -z "${date_utc}" ] && date_utc=$(date -u 2>/dev/null)
  697. # ----------------------------------------------------------------------------
  698. # prepare some extra headers if we've been asked to thread e-mails
  699. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  700. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
  701. else
  702. email_thread_headers=
  703. fi
  704. # -----------------------------------------------------------------------------
  705. # function to URL encode a string
  706. urlencode() {
  707. local string="${1}" strlen encoded pos c o
  708. strlen=${#string}
  709. for ((pos = 0; pos < strlen; pos++)); do
  710. c=${string:pos:1}
  711. case "${c}" in
  712. [-_.~a-zA-Z0-9])
  713. o="${c}"
  714. ;;
  715. *)
  716. printf -v o '%%%02x' "'${c}"
  717. ;;
  718. esac
  719. encoded+="${o}"
  720. done
  721. REPLY="${encoded}"
  722. echo "${REPLY}"
  723. }
  724. # -----------------------------------------------------------------------------
  725. # function to convert a duration in seconds, to a human readable duration
  726. # using DAYS, MINUTES, SECONDS
  727. duration4human() {
  728. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  729. d=$((s / 86400))
  730. s=$((s - (d * 86400)))
  731. h=$((s / 3600))
  732. s=$((s - (h * 3600)))
  733. m=$((s / 60))
  734. s=$((s - (m * 60)))
  735. if [ ${d} -gt 0 ]; then
  736. [ ${m} -ge 30 ] && h=$((h + 1))
  737. [ ${d} -gt 1 ] && ds="days"
  738. [ ${h} -gt 1 ] && hs="hours"
  739. if [ ${h} -gt 0 ]; then
  740. ret="${d} ${ds} and ${h} ${hs}"
  741. else
  742. ret="${d} ${ds}"
  743. fi
  744. elif [ ${h} -gt 0 ]; then
  745. [ ${s} -ge 30 ] && m=$((m + 1))
  746. [ ${h} -gt 1 ] && hs="hours"
  747. [ ${m} -gt 1 ] && ms="minutes"
  748. if [ ${m} -gt 0 ]; then
  749. ret="${h} ${hs} and ${m} ${ms}"
  750. else
  751. ret="${h} ${hs}"
  752. fi
  753. elif [ ${m} -gt 0 ]; then
  754. [ ${m} -gt 1 ] && ms="minutes"
  755. [ ${s} -gt 1 ] && ss="seconds"
  756. if [ ${s} -gt 0 ]; then
  757. ret="${m} ${ms} and ${s} ${ss}"
  758. else
  759. ret="${m} ${ms}"
  760. fi
  761. else
  762. [ ${s} -gt 1 ] && ss="seconds"
  763. ret="${s} ${ss}"
  764. fi
  765. REPLY="${ret}"
  766. echo "${REPLY}"
  767. }
  768. # -----------------------------------------------------------------------------
  769. # email sender
  770. send_email() {
  771. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  772. if [ "${SEND_EMAIL}" = "YES" ]; then
  773. if [ -n "${EMAIL_SENDER}" ]; then
  774. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  775. # the name includes double quotes
  776. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  777. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  778. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  779. # the name includes single quotes
  780. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  781. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  782. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  783. # the name does not have any quotes
  784. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  785. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  786. fi
  787. fi
  788. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  789. [ -n "${sender_name}" ] && ${sendmail} -F 2>&1 | head -1 | grep -qv "sendmail: unrecognized option: F" && opts+=(-F "${sender_name}")
  790. if [ "${debug}" = "1" ]; then
  791. echo >&2 "--- BEGIN sendmail command ---"
  792. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  793. echo >&2
  794. echo >&2 "--- END sendmail command ---"
  795. fi
  796. local cmd_output
  797. cmd_output=$("${sendmail}" -t "${opts[@]}" 2>&1)
  798. ret=$?
  799. if [ ${ret} -eq 0 ]; then
  800. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  801. return 0
  802. else
  803. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret} (${cmd_output})."
  804. return 1
  805. fi
  806. fi
  807. return 1
  808. }
  809. # -----------------------------------------------------------------------------
  810. # pushover sender
  811. send_pushover() {
  812. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  813. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  814. # https://pushover.net/api
  815. priority=-2
  816. case "${status}" in
  817. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  818. WARNING) priority=0 ;; # normal priority: respect quiet hours
  819. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  820. *) priority=-2 ;; # lowest priority: no notification at all
  821. esac
  822. for user in ${usertokens}; do
  823. httpcode=$(docurl \
  824. --form-string "token=${apptoken}" \
  825. --form-string "user=${user}" \
  826. --form-string "html=1" \
  827. --form-string "title=${title}" \
  828. --form-string "message=${message}" \
  829. --form-string "timestamp=${when}" \
  830. --form-string "url=${url}" \
  831. --form-string "url_title=Open netdata dashboard to view the alarm" \
  832. --form-string "priority=${priority}" \
  833. https://api.pushover.net/1/messages.json)
  834. if [ "${httpcode}" = "200" ]; then
  835. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  836. sent=$((sent + 1))
  837. else
  838. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  839. fi
  840. done
  841. [ ${sent} -gt 0 ] && return 0
  842. fi
  843. return 1
  844. }
  845. # -----------------------------------------------------------------------------
  846. # pushbullet sender
  847. send_pushbullet() {
  848. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 userOrChannelTag
  849. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  850. # https://docs.pushbullet.com/#create-push
  851. # Accept specification of user(s) (PushBullet account email address) and/or channel tag(s), separated by spaces.
  852. # If recipient begins with a "#" then send to channel tag, otherwise send to email recipient.
  853. for userOrChannelTag in ${recipients}; do
  854. if [ "${userOrChannelTag::1}" = "#" ]; then
  855. userOrChannelTag_type="channel_tag"
  856. userOrChannelTag="${userOrChannelTag:1}" # Remove hash from start of channel tag (required by pushbullet API)
  857. else
  858. userOrChannelTag_type="email"
  859. fi
  860. httpcode=$(docurl \
  861. --header 'Access-Token: '${userapikey}'' \
  862. --header 'Content-Type: application/json' \
  863. --data-binary @<(
  864. cat <<EOF
  865. {"title": "${title}",
  866. "type": "link",
  867. "${userOrChannelTag_type}": "${userOrChannelTag}",
  868. "body": "$(echo -n ${message})",
  869. "url": "${url}",
  870. "source_device_iden": "${source_device}"}
  871. EOF
  872. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  873. if [ "${httpcode}" = "200" ]; then
  874. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${userOrChannelTag}'"
  875. sent=$((sent + 1))
  876. else
  877. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${userOrChannelTag}' with HTTP response status code ${httpcode}."
  878. fi
  879. done
  880. [ ${sent} -gt 0 ] && return 0
  881. fi
  882. return 1
  883. }
  884. # -----------------------------------------------------------------------------
  885. # kafka sender
  886. send_kafka() {
  887. local httpcode sent=0
  888. if [ "${SEND_KAFKA}" = "YES" ]; then
  889. httpcode=$(docurl -X POST \
  890. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  891. "${KAFKA_URL}")
  892. if [ "${httpcode}" = "204" ]; then
  893. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  894. sent=$((sent + 1))
  895. else
  896. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP response status code ${httpcode}."
  897. fi
  898. [ ${sent} -gt 0 ] && return 0
  899. fi
  900. return 1
  901. }
  902. # -----------------------------------------------------------------------------
  903. # pagerduty.com sender
  904. send_pd() {
  905. local recipients="${1}" sent=0 severity current_time payload url response_code
  906. unset t
  907. case ${status} in
  908. CLEAR) t='resolve' ; severity='info' ;;
  909. WARNING) t='trigger' ; severity='warning' ;;
  910. CRITICAL) t='trigger' ; severity='critical' ;;
  911. esac
  912. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  913. if [ "$(uname)" == "Linux" ]; then
  914. current_time=$(date -d @${when} +'%Y-%m-%dT%H:%M:%S.000')
  915. else
  916. current_time=$(date -r ${when} +'%Y-%m-%dT%H:%M:%S.000')
  917. fi
  918. for PD_SERVICE_KEY in ${recipients}; do
  919. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  920. if [ ${USE_PD_VERSION} = "2" ]; then
  921. payload="$(
  922. cat <<EOF
  923. {
  924. "payload" : {
  925. "summary": "${info:0:1024}",
  926. "source" : "${args_host}",
  927. "severity" : "${severity}",
  928. "timestamp" : "${current_time}",
  929. "group" : "${family}",
  930. "class" : "${chart}",
  931. "custom_details": {
  932. "value_w_units": "${value_string}",
  933. "when": "${when}",
  934. "duration" : "${duration}",
  935. "roles": "${roles}",
  936. "alarm_id" : "${alarm_id}",
  937. "name" : "${name}",
  938. "chart" : "${chart}",
  939. "family" : "${family}",
  940. "status" : "${status}",
  941. "old_status" : "${old_status}",
  942. "value" : "${value}",
  943. "old_value" : "${old_value}",
  944. "src" : "${src}",
  945. "non_clear_duration" : "${non_clear_duration}",
  946. "units" : "${units}",
  947. "info" : "${info}"
  948. }
  949. },
  950. "routing_key": "${PD_SERVICE_KEY}",
  951. "event_action": "${t}",
  952. "dedup_key": "${unique_id}"
  953. }
  954. EOF
  955. )"
  956. url="https://events.pagerduty.com/v2/enqueue"
  957. response_code="202"
  958. else
  959. payload="$(
  960. cat <<EOF
  961. {
  962. "service_key": "${PD_SERVICE_KEY}",
  963. "event_type": "${t}",
  964. "incident_key" : "${alarm_id}",
  965. "description": "${d}",
  966. "details": {
  967. "value_w_units": "${value_string}",
  968. "when": "${when}",
  969. "duration" : "${duration}",
  970. "roles": "${roles}",
  971. "alarm_id" : "${alarm_id}",
  972. "name" : "${name}",
  973. "chart" : "${chart}",
  974. "family" : "${family}",
  975. "status" : "${status}",
  976. "old_status" : "${old_status}",
  977. "value" : "${value}",
  978. "old_value" : "${old_value}",
  979. "src" : "${src}",
  980. "non_clear_duration" : "${non_clear_duration}",
  981. "units" : "${units}",
  982. "info" : "${info}"
  983. }
  984. }
  985. EOF
  986. )"
  987. url="https://events.pagerduty.com/generic/2010-04-15/create_event.json"
  988. response_code="200"
  989. fi
  990. httpcode=$(docurl -X POST --data "${payload}" ${url})
  991. if [ "${httpcode}" = "${response_code}" ]; then
  992. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  993. sent=$((sent + 1))
  994. else
  995. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
  996. fi
  997. done
  998. [ ${sent} -gt 0 ] && return 0
  999. fi
  1000. return 1
  1001. }
  1002. # -----------------------------------------------------------------------------
  1003. # twilio sender
  1004. send_twilio() {
  1005. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  1006. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1007. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  1008. for user in ${recipients}; do
  1009. httpcode=$(docurl -X POST \
  1010. --data-urlencode "From=${twilionumber}" \
  1011. --data-urlencode "To=${user}" \
  1012. --data-urlencode "Body=${title} ${message}" \
  1013. -u "${accountsid}:${accounttoken}" \
  1014. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  1015. if [ "${httpcode}" = "201" ]; then
  1016. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1017. sent=$((sent + 1))
  1018. else
  1019. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1020. fi
  1021. done
  1022. [ ${sent} -gt 0 ] && return 0
  1023. fi
  1024. return 1
  1025. }
  1026. # -----------------------------------------------------------------------------
  1027. # hipchat sender
  1028. send_hipchat() {
  1029. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  1030. # remove <small></small> from the message
  1031. message="${message//<small>/}"
  1032. message="${message//<\/small>/}"
  1033. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  1034. # Valid values: html, text.
  1035. # Defaults to 'html'.
  1036. msg_format="html"
  1037. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  1038. case "${status}" in
  1039. WARNING) color="yellow" ;;
  1040. CRITICAL) color="red" ;;
  1041. CLEAR) color="green" ;;
  1042. *) color="gray" ;;
  1043. esac
  1044. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  1045. # Each recipient's notification preferences are taken into account.
  1046. # Defaults to false.
  1047. notify="true"
  1048. for room in ${recipients}; do
  1049. httpcode=$(docurl -X POST \
  1050. -H "Content-type: application/json" \
  1051. -H "Authorization: Bearer ${authtoken}" \
  1052. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  1053. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  1054. if [ "${httpcode}" = "204" ]; then
  1055. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1056. sent=$((sent + 1))
  1057. else
  1058. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP response status code ${httpcode}."
  1059. fi
  1060. done
  1061. [ ${sent} -gt 0 ] && return 0
  1062. fi
  1063. return 1
  1064. }
  1065. # -----------------------------------------------------------------------------
  1066. # messagebird sender
  1067. send_messagebird() {
  1068. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  1069. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1070. #https://developers.messagebird.com/docs/messaging
  1071. for user in ${recipients}; do
  1072. httpcode=$(docurl -X POST \
  1073. --data-urlencode "originator=${messagebirdnumber}" \
  1074. --data-urlencode "recipients=${user}" \
  1075. --data-urlencode "body=${title} ${message}" \
  1076. --data-urlencode "datacoding=auto" \
  1077. -H "Authorization: AccessKey ${accesskey}" \
  1078. "https://rest.messagebird.com/messages")
  1079. if [ "${httpcode}" = "201" ]; then
  1080. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1081. sent=$((sent + 1))
  1082. else
  1083. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1084. fi
  1085. done
  1086. [ ${sent} -gt 0 ] && return 0
  1087. fi
  1088. return 1
  1089. }
  1090. # -----------------------------------------------------------------------------
  1091. # kavenegar sender
  1092. send_kavenegar() {
  1093. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  1094. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1095. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1096. for user in ${recipients}; do
  1097. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  1098. --data-urlencode "sender=${kavenegarsender}" \
  1099. --data-urlencode "receptor=${user}" \
  1100. --data-urlencode "message=${title} ${message}")
  1101. if [ "${httpcode}" = "200" ]; then
  1102. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1103. sent=$((sent + 1))
  1104. else
  1105. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1106. fi
  1107. done
  1108. [ ${sent} -gt 0 ] && return 0
  1109. fi
  1110. return 1
  1111. }
  1112. # -----------------------------------------------------------------------------
  1113. # telegram sender
  1114. send_telegram() {
  1115. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  1116. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  1117. case "${status}" in
  1118. WARNING) emoji="⚠️" ;;
  1119. CRITICAL) emoji="🔴" ;;
  1120. CLEAR) emoji="✅" ;;
  1121. *) emoji="⚪️" ;;
  1122. esac
  1123. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  1124. for chatid in ${chatids}; do
  1125. # https://core.telegram.org/bots/api#sendmessage
  1126. httpcode=$(docurl ${disableNotification} \
  1127. --data-urlencode "parse_mode=HTML" \
  1128. --data-urlencode "disable_web_page_preview=true" \
  1129. --data-urlencode "text=${emoji} ${message}" \
  1130. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  1131. if [ "${httpcode}" = "200" ]; then
  1132. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  1133. sent=$((sent + 1))
  1134. elif [ "${httpcode}" = "401" ]; then
  1135. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  1136. else
  1137. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
  1138. fi
  1139. done
  1140. [ ${sent} -gt 0 ] && return 0
  1141. fi
  1142. return 1
  1143. }
  1144. # -----------------------------------------------------------------------------
  1145. # Microsoft Team sender
  1146. send_msteams() {
  1147. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1148. [ "${SEND_MSTEAMS}" != "YES" ] && return 1
  1149. case "${status}" in
  1150. WARNING) icon="${MSTEAMS_ICON_WARNING}" && color="${MSTEAMS_COLOR_WARNING}" ;;
  1151. CRITICAL) icon="${MSTEAMS_ICON_CRITICAL}" && color="${MSTEAMS_COLOR_CRITICAL}" ;;
  1152. CLEAR) icon="${MSTEAMS_ICON_CLEAR}" && color="${MSTEAMS_COLOR_CLEAR}" ;;
  1153. *) icon="${MSTEAMS_ICON_DEFAULT}" && color="${MSTEAMS_COLOR_DEFAULT}" ;;
  1154. esac
  1155. for channel in ${channels}; do
  1156. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  1157. ## Online designer : https://adaptivecards.io/designer/
  1158. payload="$(
  1159. cat <<EOF
  1160. {
  1161. "@context": "http://schema.org/extensions",
  1162. "@type": "MessageCard",
  1163. "themeColor": "${color}",
  1164. "title": "$icon Alert ${status} from netdata for ${host}",
  1165. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  1166. "potentialAction": [
  1167. {
  1168. "@type": "OpenUri",
  1169. "name": "Netdata",
  1170. "targets": [
  1171. { "os": "default", "uri": "${goto_url}" }
  1172. ]
  1173. }
  1174. ]
  1175. }
  1176. EOF
  1177. )"
  1178. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1179. cur_webhook="${webhook//CHANNEL/${channel}}"
  1180. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${cur_webhook}")
  1181. if [ "${httpcode}" = "200" ]; then
  1182. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${cur_webhook}'"
  1183. sent=$((sent + 1))
  1184. else
  1185. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${cur_webhook}', with HTTP response status code ${httpcode}."
  1186. fi
  1187. done
  1188. [ ${sent} -gt 0 ] && return 0
  1189. return 1
  1190. }
  1191. # slack sender
  1192. send_slack() {
  1193. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1194. [ "${SEND_SLACK}" != "YES" ] && return 1
  1195. case "${status}" in
  1196. WARNING) color="warning" ;;
  1197. CRITICAL) color="danger" ;;
  1198. CLEAR) color="good" ;;
  1199. *) color="#777777" ;;
  1200. esac
  1201. for channel in ${channels}; do
  1202. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1203. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1204. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1205. if [ "${channel}" = "#" ]; then
  1206. ch=""
  1207. chstr="without specifying a channel"
  1208. else
  1209. ch="\"channel\": \"${channel}\","
  1210. chstr="to '${channel}'"
  1211. fi
  1212. payload="$(
  1213. cat <<EOF
  1214. {
  1215. $ch
  1216. "username": "netdata on ${host}",
  1217. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1218. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1219. "attachments": [
  1220. {
  1221. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1222. "color": "${color}",
  1223. "title": "${alarm}",
  1224. "title_link": "${goto_url}",
  1225. "text": "${info}",
  1226. "fields": [
  1227. {
  1228. "title": "${chart}",
  1229. "short": true
  1230. },
  1231. {
  1232. "title": "${family}",
  1233. "short": true
  1234. }
  1235. ],
  1236. "thumb_url": "${image}",
  1237. "footer": "by ${host}",
  1238. "ts": ${when}
  1239. }
  1240. ]
  1241. }
  1242. EOF
  1243. )"
  1244. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1245. if [ "${httpcode}" = "200" ]; then
  1246. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1247. sent=$((sent + 1))
  1248. else
  1249. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP response status code ${httpcode}."
  1250. fi
  1251. done
  1252. [ ${sent} -gt 0 ] && return 0
  1253. return 1
  1254. }
  1255. # -----------------------------------------------------------------------------
  1256. # rocketchat sender
  1257. send_rocketchat() {
  1258. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1259. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1260. case "${status}" in
  1261. WARNING) color="warning" ;;
  1262. CRITICAL) color="danger" ;;
  1263. CLEAR) color="good" ;;
  1264. *) color="#777777" ;;
  1265. esac
  1266. for channel in ${channels}; do
  1267. payload="$(
  1268. cat <<EOF
  1269. {
  1270. "channel": "#${channel}",
  1271. "alias": "netdata on ${host}",
  1272. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1273. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1274. "attachments": [
  1275. {
  1276. "color": "${color}",
  1277. "title": "${alarm}",
  1278. "title_link": "${goto_url}",
  1279. "text": "${info}",
  1280. "fields": [
  1281. {
  1282. "title": "${chart}",
  1283. "short": true,
  1284. "value": "chart"
  1285. },
  1286. {
  1287. "title": "${family}",
  1288. "short": true,
  1289. "value": "family"
  1290. }
  1291. ],
  1292. "thumb_url": "${image}",
  1293. "ts": "${when}"
  1294. }
  1295. ]
  1296. }
  1297. EOF
  1298. )"
  1299. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1300. if [ "${httpcode}" = "200" ]; then
  1301. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1302. sent=$((sent + 1))
  1303. else
  1304. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1305. fi
  1306. done
  1307. [ ${sent} -gt 0 ] && return 0
  1308. return 1
  1309. }
  1310. # -----------------------------------------------------------------------------
  1311. # alerta sender
  1312. send_alerta() {
  1313. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1314. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1315. case "${status}" in
  1316. CRITICAL) severity="critical" ;;
  1317. WARNING) severity="warning" ;;
  1318. CLEAR) severity="cleared" ;;
  1319. *) severity="indeterminate" ;;
  1320. esac
  1321. if [[ ${chart} == httpcheck* ]]; then
  1322. resource=$chart
  1323. event=$name
  1324. else
  1325. resource="${host}:${family}"
  1326. event="${chart}.${name}"
  1327. fi
  1328. for channel in ${channels}; do
  1329. payload="$(
  1330. cat <<EOF
  1331. {
  1332. "resource": "${resource}",
  1333. "event": "${event}",
  1334. "environment": "${channel}",
  1335. "severity": "${severity}",
  1336. "service": ["Netdata"],
  1337. "group": "Performance",
  1338. "value": "${value_string}",
  1339. "text": "${info}",
  1340. "tags": ["alarm_id:${alarm_id}"],
  1341. "attributes": {
  1342. "roles": "${roles}",
  1343. "name": "${name}",
  1344. "chart": "${chart}",
  1345. "family": "${family}",
  1346. "source": "${src}",
  1347. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1348. },
  1349. "origin": "netdata/${host}",
  1350. "type": "netdataAlarm",
  1351. "rawData": "${BASH_ARGV[@]}"
  1352. }
  1353. EOF
  1354. )"
  1355. if [ -n "${ALERTA_API_KEY}" ]; then
  1356. auth="Key ${ALERTA_API_KEY}"
  1357. fi
  1358. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1359. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1360. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1361. sent=$((sent + 1))
  1362. elif [ "${httpcode}" = "202" ]; then
  1363. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1364. else
  1365. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1366. fi
  1367. done
  1368. [ ${sent} -gt 0 ] && return 0
  1369. return 1
  1370. }
  1371. # -----------------------------------------------------------------------------
  1372. # flock sender
  1373. send_flock() {
  1374. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1375. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1376. case "${status}" in
  1377. WARNING) color="warning" ;;
  1378. CRITICAL) color="danger" ;;
  1379. CLEAR) color="good" ;;
  1380. *) color="#777777" ;;
  1381. esac
  1382. for channel in ${channels}; do
  1383. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1384. \"sendAs\": {
  1385. \"name\" : \"netdata on ${host}\",
  1386. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1387. },
  1388. \"text\": \"${host} *${status_message}*\",
  1389. \"timestamp\": \"${when}\",
  1390. \"attachments\": [
  1391. {
  1392. \"description\": \"${chart} (${family}) - ${info}\",
  1393. \"color\": \"${color}\",
  1394. \"title\": \"${alarm}\",
  1395. \"url\": \"${goto_url}\",
  1396. \"text\": \"${info}\",
  1397. \"views\": {
  1398. \"image\": {
  1399. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1400. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1401. \"filename\": \"${image}\"
  1402. }
  1403. }
  1404. }
  1405. ]
  1406. }")
  1407. if [ "${httpcode}" = "200" ]; then
  1408. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1409. sent=$((sent + 1))
  1410. else
  1411. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1412. fi
  1413. done
  1414. [ ${sent} -gt 0 ] && return 0
  1415. return 1
  1416. }
  1417. # -----------------------------------------------------------------------------
  1418. # discord sender
  1419. send_discord() {
  1420. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1421. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1422. case "${status}" in
  1423. WARNING) color="warning" ;;
  1424. CRITICAL) color="danger" ;;
  1425. CLEAR) color="good" ;;
  1426. *) color="#777777" ;;
  1427. esac
  1428. for channel in ${channels}; do
  1429. username="netdata on ${host}"
  1430. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1431. payload="$(
  1432. cat <<EOF
  1433. {
  1434. "channel": "#${channel}",
  1435. "username": "${username}",
  1436. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1437. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1438. "attachments": [
  1439. {
  1440. "color": "${color}",
  1441. "title": "${alarm}",
  1442. "title_link": "${goto_url}",
  1443. "text": "${info}",
  1444. "fields": [
  1445. {
  1446. "title": "${chart}",
  1447. "value": "${family}"
  1448. }
  1449. ],
  1450. "thumb_url": "${image}",
  1451. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1452. "footer": "${host}",
  1453. "ts": ${when}
  1454. }
  1455. ]
  1456. }
  1457. EOF
  1458. )"
  1459. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1460. if [ "${httpcode}" = "200" ]; then
  1461. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1462. sent=$((sent + 1))
  1463. else
  1464. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1465. fi
  1466. done
  1467. [ ${sent} -gt 0 ] && return 0
  1468. return 1
  1469. }
  1470. # -----------------------------------------------------------------------------
  1471. # fleep sender
  1472. send_fleep() {
  1473. local httpcode sent=0 webhooks="${1}" data message
  1474. if [ "${SEND_FLEEP}" = "YES" ]; then
  1475. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1476. for hook in ${webhooks}; do
  1477. data="{ "
  1478. data="${data} 'message': '${message}', "
  1479. data="${data} 'user': '${FLEEP_SENDER}' "
  1480. data="${data} }"
  1481. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1482. if [ "${httpcode}" = "200" ]; then
  1483. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1484. sent=$((sent + 1))
  1485. else
  1486. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP response status code ${httpcode}."
  1487. fi
  1488. done
  1489. [ ${sent} -gt 0 ] && return 0
  1490. fi
  1491. return 1
  1492. }
  1493. # -----------------------------------------------------------------------------
  1494. # Prowl sender
  1495. send_prowl() {
  1496. local httpcode sent=0 data message keys prio=0 alarm_url event
  1497. if [ "${SEND_PROWL}" = "YES" ]; then
  1498. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1499. message="description=${message}"
  1500. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1501. keys="apikey=${keys}"
  1502. app="application=Netdata"
  1503. case "${status}" in
  1504. CRITICAL)
  1505. prio=2
  1506. ;;
  1507. WARNING)
  1508. prio=1
  1509. ;;
  1510. esac
  1511. prio="priority=${prio}"
  1512. alarm_url="$(urlencode ${goto_url})"
  1513. alarm_url="url=${alarm_url}"
  1514. event="$(urlencode "${host} ${status_message}")"
  1515. event="event=${event}"
  1516. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1517. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1518. if [ "${httpcode}" = "200" ]; then
  1519. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1520. sent=1
  1521. else
  1522. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1523. fi
  1524. [ ${sent} -gt 0 ] && return 0
  1525. fi
  1526. return 1
  1527. }
  1528. # -----------------------------------------------------------------------------
  1529. # irc sender
  1530. send_irc() {
  1531. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" PORT="${5}" SERVERNAME="${6}" MESSAGE="${7}" sent=0 channel color send_alarm reply_codes error
  1532. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ] && [ -n "${PORT}" ]; then
  1533. case "${status}" in
  1534. WARNING) color="warning" ;;
  1535. CRITICAL) color="danger" ;;
  1536. CLEAR) color="good" ;;
  1537. *) color="#777777" ;;
  1538. esac
  1539. SNDMESSAGE="${MESSAGE//$'\n'/", "}"
  1540. for CHANNEL in ${CHANNELS}; do
  1541. error=0
  1542. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" "${PORT}")
  1543. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1544. for code in ${reply_codes}; do
  1545. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1546. error=1
  1547. break
  1548. fi
  1549. done
  1550. if [ "${error}" -eq 0 ]; then
  1551. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1552. sent=$((sent + 1))
  1553. else
  1554. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1555. fi
  1556. done
  1557. fi
  1558. [ ${sent} -gt 0 ] && return 0
  1559. return 1
  1560. }
  1561. # -----------------------------------------------------------------------------
  1562. # Amazon SNS sender
  1563. send_awssns() {
  1564. local targets="${1}" message='' sent=0 region=''
  1565. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1566. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1567. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1568. for target in ${targets}; do
  1569. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1570. region="$(echo ${target} | cut -f 4 -d ':')"
  1571. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1572. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1573. sent=$((sent + 1))
  1574. else
  1575. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1576. fi
  1577. done
  1578. [ ${sent} -gt 0 ] && return 0
  1579. return 1
  1580. }
  1581. # -----------------------------------------------------------------------------
  1582. # Matrix sender
  1583. send_matrix() {
  1584. local homeserver="${1}" webhook accesstoken rooms="${2}" httpcode sent=0 payload
  1585. [ "${SEND_MATRIX}" != "YES" ] && return 1
  1586. [ -z "${MATRIX_ACCESSTOKEN}" ] && return 1
  1587. accesstoken="${MATRIX_ACCESSTOKEN}"
  1588. case "${status}" in
  1589. WARNING) emoji="⚠️" ;;
  1590. CRITICAL) emoji="🔴" ;;
  1591. CLEAR) emoji="✅" ;;
  1592. *) emoji="⚪️" ;;
  1593. esac
  1594. for room in ${rooms}; do
  1595. webhook="$homeserver/_matrix/client/r0/rooms/$(urlencode $room)/send/m.room.message?access_token=$accesstoken"
  1596. payload="$(
  1597. cat <<EOF
  1598. {
  1599. "msgtype": "m.notice",
  1600. "format": "org.matrix.custom.html",
  1601. "formatted_body": "${emoji} ${host} ${status_message} - <b>${name//_/ }</b><br>${chart} (${family})<br><a href=\"${goto_url}\">${alarm}</a><br><i>${info}</i>",
  1602. "body": "${emoji} ${host} ${status_message} - ${name//_/ } ${chart} (${family}) ${goto_url} ${alarm} ${info}"
  1603. }
  1604. EOF
  1605. )"
  1606. httpcode=$(docurl -X POST --data "${payload}" "${webhook}")
  1607. if [ "${httpcode}" == "200" ]; then
  1608. info "sent Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1609. sent=$((sent + 1))
  1610. else
  1611. error "failed to send Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
  1612. fi
  1613. done
  1614. [ ${sent} -gt 0 ] && return 0
  1615. return 1
  1616. }
  1617. # -----------------------------------------------------------------------------
  1618. # syslog sender
  1619. send_syslog() {
  1620. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1621. local priority='' message='' server='' port='' prefix=''
  1622. local temp1='' temp2=''
  1623. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1624. if [ "${status}" = "CRITICAL" ]; then
  1625. level='crit'
  1626. elif [ "${status}" = "WARNING" ]; then
  1627. level='warning'
  1628. fi
  1629. for target in ${targets}; do
  1630. priority="${facility}.${level}"
  1631. message=''
  1632. server=''
  1633. port=''
  1634. prefix=''
  1635. temp1=''
  1636. temp2=''
  1637. prefix=$(echo ${target} | cut -d '/' -f 2)
  1638. temp1=$(echo ${target} | cut -d '/' -f 1)
  1639. if [ ${prefix} != ${temp1} ]; then
  1640. if (echo ${temp1} | grep -q '@'); then
  1641. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1642. server=$(echo ${temp1} | cut -d '@' -f 2)
  1643. if [ ${temp2} != ${server} ]; then
  1644. priority=${temp2}
  1645. fi
  1646. port=$(echo ${server} | rev | cut -d ':' -f 1 | rev)
  1647. if (echo ${server} | grep -E -q '\[.*\]'); then
  1648. if (echo ${port} | grep -q ']'); then
  1649. port=''
  1650. else
  1651. server=$(echo ${server} | rev | cut -d ':' -f 2- | rev)
  1652. fi
  1653. else
  1654. if [ ${port} = ${server} ]; then
  1655. port=''
  1656. else
  1657. server=$(echo ${server} | cut -d ':' -f 1)
  1658. fi
  1659. fi
  1660. else
  1661. priority=${temp1}
  1662. fi
  1663. fi
  1664. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1665. if [ ${server} ]; then
  1666. logger_options="${logger_options} -n ${server}"
  1667. if [ ${port} ]; then
  1668. logger_options="${logger_options} -P ${port}"
  1669. fi
  1670. fi
  1671. ${logger} -p ${priority} ${logger_options} "${message}"
  1672. done
  1673. return $?
  1674. }
  1675. # -----------------------------------------------------------------------------
  1676. # SMS sender
  1677. send_sms() {
  1678. local recipients="${1}" errcode errmessage sent=0
  1679. # Human readable SMS
  1680. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1681. # limit it to 160 characters
  1682. msg="${msg:0:160}"
  1683. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1684. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1685. for phone in ${recipients}; do
  1686. errmessage=$($sendsms $phone "$msg" 2>&1)
  1687. errcode=$?
  1688. if [ ${errcode} -eq 0 ]; then
  1689. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1690. sent=$((sent + 1))
  1691. else
  1692. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1693. fi
  1694. done
  1695. [ ${sent} -gt 0 ] && return 0
  1696. fi
  1697. return 1
  1698. }
  1699. # -----------------------------------------------------------------------------
  1700. # hangouts sender
  1701. send_hangouts() {
  1702. local rooms="${1}" httpcode sent=0 room color payload webhook thread
  1703. [ "${SEND_HANGOUTS}" != "YES" ] && return 1
  1704. case "${status}" in
  1705. WARNING) color="#ffa700" ;;
  1706. CRITICAL) color="#d62d20" ;;
  1707. CLEAR) color="#008744" ;;
  1708. *) color="#777777" ;;
  1709. esac
  1710. for room in ${rooms}; do
  1711. if [ -z "${HANGOUTS_WEBHOOK_URI[$room]}" ] ; then
  1712. info "Can't send Hangouts notification for: ${host} ${chart}.${name} to room ${room}. HANGOUTS_WEBHOOK_URI[$room] not defined"
  1713. else
  1714. if [ -n "${HANGOUTS_WEBHOOK_THREAD[$room]}" ]; then
  1715. thread="\"name\" : \"${HANGOUTS_WEBHOOK_THREAD[$room]}\""
  1716. fi
  1717. webhook="${HANGOUTS_WEBHOOK_URI[$room]}"
  1718. payload="$(
  1719. cat <<EOF
  1720. {
  1721. "cards": [
  1722. {
  1723. "header": {
  1724. "title": "Netdata on ${host}",
  1725. "imageUrl": "${images_base_url}/images/banner-icon-144x144.png",
  1726. "imageStyle": "IMAGE"
  1727. },
  1728. "sections": [
  1729. {
  1730. "header": "<b>${host}</b>",
  1731. "widgets": [
  1732. {
  1733. "keyValue": {
  1734. "topLabel": "Status Message",
  1735. "content": "<b>${status_message}</b>",
  1736. "contentMultiline": "true",
  1737. "iconUrl": "${image}",
  1738. "onClick": {
  1739. "openLink": {
  1740. "url": "${goto_url}"
  1741. }
  1742. }
  1743. }
  1744. },
  1745. {
  1746. "keyValue": {
  1747. "topLabel": "${chart} | ${family}",
  1748. "content": "<font color=${color}>${alarm}</font>",
  1749. "contentMultiline": "true"
  1750. }
  1751. }
  1752. ]
  1753. },
  1754. {
  1755. "widgets": [
  1756. {
  1757. "textParagraph": {
  1758. "text": "<font color=\"#0057e7\">@ ${date}\n<b>${info}</b></font>"
  1759. }
  1760. }
  1761. ]
  1762. },
  1763. {
  1764. "widgets": [
  1765. {
  1766. "buttons": [
  1767. {
  1768. "textButton": {
  1769. "text": "Go to ${host}",
  1770. "onClick": {
  1771. "openLink": {
  1772. "url": "${goto_url}"
  1773. }
  1774. }
  1775. }
  1776. }
  1777. ]
  1778. }
  1779. ]
  1780. }
  1781. ]
  1782. }
  1783. ],
  1784. "thread": {
  1785. $thread
  1786. }
  1787. }
  1788. EOF
  1789. )"
  1790. httpcode=$(docurl -H "Content-Type: application/json" -X POST -d "${payload}" "${webhook}")
  1791. if [ "${httpcode}" = "200" ]; then
  1792. info "sent hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1793. sent=$((sent + 1))
  1794. else
  1795. error "failed to send hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
  1796. fi
  1797. fi
  1798. done
  1799. [ ${sent} -gt 0 ] && return 0
  1800. return 1
  1801. }
  1802. # -----------------------------------------------------------------------------
  1803. # Dynatrace sender
  1804. send_dynatrace() {
  1805. [ "${SEND_DYNATRACE}" != "YES" ] && return 1
  1806. local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
  1807. local description="Netdata Notification for: ${host} ${chart}.${name} is ${status}"
  1808. local payload=""
  1809. payload=$(cat <<EOF
  1810. {
  1811. "title": "Netdata Alarm from ${host}",
  1812. "source" : "${DYNATRACE_ANNOTATION_TYPE}",
  1813. "description" : "${description}",
  1814. "eventType": "${DYNATRACE_EVENT}",
  1815. "attachRules":{
  1816. "tagRule":[{
  1817. "meTypes":["HOST"],
  1818. "tags":["${DYNATRACE_TAG_VALUE}"]
  1819. }]
  1820. },
  1821. "customProperties":{
  1822. "description": "${description}"
  1823. }
  1824. }
  1825. EOF
  1826. )
  1827. # echo ${payload}
  1828. httpcode=$(docurl -X POST -H "Authorization: Api-token ${DYNATRACE_TOKEN}" -H "Content-Type: application/json" -d "${payload}" ${dynatrace_url})
  1829. ret=$?
  1830. if [ ${ret} -eq 0 ]; then
  1831. if [ "${httpcode}" = "200" ]; then
  1832. info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
  1833. return 0
  1834. else
  1835. warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
  1836. return 1
  1837. fi
  1838. else
  1839. error "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to ${DYNATRACE_SERVER} with error code ${ret}."
  1840. return 1
  1841. fi
  1842. }
  1843. # -----------------------------------------------------------------------------
  1844. # Stackpulse sender
  1845. send_stackpulse() {
  1846. local payload httpcode oldv currv
  1847. [ "${SEND_STACKPULSE}" != "YES" ] && return 1
  1848. # We are sending null when values are nan to avoid errors while JSON message is parsed
  1849. [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
  1850. [ "${value}" != "nan" ] && currv="${value}" || currv="null"
  1851. payload=$(cat <<EOF
  1852. {
  1853. "Node" : "${host}",
  1854. "Chart" : "${chart}",
  1855. "OldValue" : ${oldv},
  1856. "Value" : ${currv},
  1857. "Units" : "${units}",
  1858. "OldStatus" : "${old_status}",
  1859. "Status" : "${status}",
  1860. "Alarm" : "${name}",
  1861. "Date": ${when},
  1862. "Duration": ${duration},
  1863. "NonClearDuration": ${non_clear_duration},
  1864. "Description" : "${status_message}, ${info}",
  1865. "CalcExpression" : "${calc_expression}",
  1866. "CalcParamValues" : "${calc_param_values}",
  1867. "TotalWarnings" : "${total_warnings}",
  1868. "TotalCritical" : "${total_critical}",
  1869. "ID" : ${alarm_id}
  1870. }
  1871. EOF
  1872. )
  1873. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" ${STACKPULSE_WEBHOOK})
  1874. if [ "${httpcode}" = "200" ]; then
  1875. info "sent stackpulse notification for: ${host} ${chart}.${name} is ${status}"
  1876. else
  1877. error "failed to send stackpulse notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
  1878. return 1
  1879. fi
  1880. return 0
  1881. }
  1882. # -----------------------------------------------------------------------------
  1883. # Opsgenie sender
  1884. send_opsgenie() {
  1885. local payload httpcode oldv currv
  1886. [ "${SEND_OPSGENIE}" != "YES" ] && return 1
  1887. if [ -z "${OPSGENIE_API_KEY}" ] ; then
  1888. info "Can't send Opsgenie notification, because OPSGENIE_API_KEY is not defined"
  1889. return 1
  1890. fi
  1891. # We are sending null when values are nan to avoid errors while JSON message is parsed
  1892. [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
  1893. [ "${value}" != "nan" ] && currv="${value}" || currv="null"
  1894. payload=$(cat <<EOF
  1895. {
  1896. "host" : "${host}",
  1897. "unique_id" : "${unique_id}",
  1898. "alarmId" : ${alarm_id},
  1899. "eventId" : ${event_id},
  1900. "chart" : "${chart}",
  1901. "when": ${when},
  1902. "name" : "${name}",
  1903. "family" : "${family}",
  1904. "status" : "${status}",
  1905. "old_status" : "${old_status}",
  1906. "value" : ${currv},
  1907. "old_value" : ${oldv},
  1908. "duration": ${duration},
  1909. "non_clear_duration": ${non_clear_duration},
  1910. "units" : "${units}",
  1911. "info" : "${status_message}, ${info}",
  1912. "calc_expression" : "${calc_expression}",
  1913. "total_warnings" : "${total_warnings}",
  1914. "total_critical" : "${total_critical}",
  1915. "src" : "${src}"
  1916. }
  1917. EOF
  1918. )
  1919. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${OPSGENIE_API_URL}/v1/json/integrations/webhooks/netdata?apiKey=${OPSGENIE_API_KEY}")
  1920. # https://docs.opsgenie.com/docs/alert-api#create-alert
  1921. if [ "${httpcode}" = "200" ]; then
  1922. info "sent opsgenie notification for: ${host} ${chart}.${name} is ${status}"
  1923. else
  1924. error "failed to send opsgenie notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  1925. return 1
  1926. fi
  1927. return 0
  1928. }
  1929. # -----------------------------------------------------------------------------
  1930. # Gotify sender
  1931. send_gotify() {
  1932. local payload httpcode priority
  1933. [ "${SEND_GOTIFY}" != "YES" ] && return 1
  1934. if [ -z "${GOTIFY_APP_TOKEN}" ] ; then
  1935. info "Can't send Gotify notification, because GOTIFY_APP_TOKEN is not defined"
  1936. return 1
  1937. fi
  1938. # priority for Gotify Android app
  1939. case "${status}" in
  1940. CRITICAL) priority=10 ;; # sound + vibration
  1941. WARNING) priority=4 ;; # sound
  1942. *) priority=1 ;; # notification only
  1943. esac
  1944. payload=$(cat <<EOF
  1945. {
  1946. "title" : "${status}, ${name} = ${value_string}, on ${host}",
  1947. "message" : "${date}: ${chart} ${value_string}",
  1948. "priority" : ${priority}
  1949. }
  1950. EOF
  1951. )
  1952. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${GOTIFY_APP_URL}/message?token=${GOTIFY_APP_TOKEN}")
  1953. if [ "${httpcode}" = "200" ]; then
  1954. info "sent gotify notification for: ${host} ${chart}.${name} is ${status}"
  1955. else
  1956. error "failed to send gotify notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  1957. return 1
  1958. fi
  1959. return 0
  1960. }
  1961. # -----------------------------------------------------------------------------
  1962. # prepare the content of the notification
  1963. # the url to send the user on click
  1964. urlencode "${args_host}" >/dev/null
  1965. url_host="${REPLY}"
  1966. urlencode "${chart}" >/dev/null
  1967. url_chart="${REPLY}"
  1968. urlencode "${family}" >/dev/null
  1969. url_family="${REPLY}"
  1970. urlencode "${name}" >/dev/null
  1971. url_name="${REPLY}"
  1972. urlencode "${value_string}" >/dev/null
  1973. url_value_string="${REPLY}"
  1974. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}&alarm_status=${status}&alarm_chart=${chart}&alarm_value=${url_value_string}"
  1975. GOTOCLOUD=0
  1976. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1977. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1978. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1979. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1980. fi
  1981. fi
  1982. if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1983. GOTOCLOUD=1
  1984. fi
  1985. fi
  1986. if [ ${GOTOCLOUD} -eq 0 ]; then
  1987. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  1988. else
  1989. # Temporarily disable alarm redirection, as the cloud endpoint no longer exists. This functionality will be restored after discussion on #9487. For now, just lead to netdata.cloud
  1990. # Re-allow alarm redirection, for alarms 2.0, new template
  1991. if [ -z "${child_machine_guid}" ]; then
  1992. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentId=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  1993. else
  1994. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentId=${NETDATA_REGISTRY_UNIQUE_ID}&childId=${child_machine_guid}&${redirect_params}"
  1995. fi
  1996. fi
  1997. # the severity of the alarm
  1998. severity="${status}"
  1999. # the time the alarm was raised
  2000. duration4human ${duration} >/dev/null
  2001. duration_txt="${REPLY}"
  2002. duration4human ${non_clear_duration} >/dev/null
  2003. non_clear_duration_txt="${REPLY}"
  2004. raised_for="(was ${old_status,,} for ${duration_txt})"
  2005. # the key status message
  2006. status_message="status unknown"
  2007. # the color of the alarm
  2008. color="grey"
  2009. # the alarm value
  2010. alarm="${name//_/ } = ${value_string}"
  2011. # the image of the alarm
  2012. image="${images_base_url}/images/banner-icon-144x144.png"
  2013. # have a default email status, in case the following case does not catch it
  2014. status_email_subject="${status}"
  2015. # prepare the title based on status
  2016. case "${status}" in
  2017. CRITICAL)
  2018. image="${images_base_url}/images/alert-128-red.png"
  2019. alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_critical.png"
  2020. status_message="is critical"
  2021. status_email_subject="Critical"
  2022. color="#ca414b"
  2023. rich_status_raised_for="Raised to critical, for ${non_clear_duration_txt}"
  2024. background_color="#FFEBEF"
  2025. border_color="#FF4136"
  2026. text_color="#FF4136"
  2027. action_text_color="#FFFFFF"
  2028. ;;
  2029. WARNING)
  2030. image="${images_base_url}/images/alert-128-orange.png"
  2031. alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_warning.png"
  2032. status_message="needs attention"
  2033. status_email_subject="Warning"
  2034. color="#ffc107"
  2035. rich_status_raised_for="Raised to warning, for ${non_clear_duration_txt}"
  2036. background_color="#FFF8E1"
  2037. border_color="#FFC300"
  2038. text_color="#536775"
  2039. action_text_color="#35414A"
  2040. ;;
  2041. CLEAR)
  2042. image="${images_base_url}/images/check-mark-2-128-green.png"
  2043. alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_recovered.png"
  2044. status_message="recovered"
  2045. status_email_subject="Clear"
  2046. color="#77ca6d"
  2047. rich_status_raised_for=
  2048. background_color="#E5F5E8"
  2049. border_color="#68C47D"
  2050. text_color="#00AB44"
  2051. action_text_color="#FFFFFF"
  2052. ;;
  2053. esac
  2054. # the html email subject
  2055. html_email_subject="${status_email_subject}, ${name} = ${value_string}, on ${host}"
  2056. if [ "${status}" = "CLEAR" ]; then
  2057. severity="Recovered from ${old_status}"
  2058. if [ ${non_clear_duration} -gt ${duration} ]; then
  2059. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  2060. fi
  2061. rich_status_raised_for="Recovered from ${old_status,,}, ${raised_for}"
  2062. # don't show the value when the status is CLEAR
  2063. # for certain alarms, this value might not have any meaning
  2064. alarm="${name//_/ } ${raised_for}"
  2065. html_email_subject="${status_email_subject}, ${name} ${raised_for}, on ${host}"
  2066. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  2067. severity="Escalated to ${status}"
  2068. if [ ${non_clear_duration} -gt ${duration} ]; then
  2069. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  2070. fi
  2071. rich_status_raised_for="Escalated to critical, ${raised_for}"
  2072. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  2073. severity="Demoted to ${status}"
  2074. if [ ${non_clear_duration} -gt ${duration} ]; then
  2075. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  2076. fi
  2077. rich_status_raised_for="Demoted to warning, ${raised_for}"
  2078. else
  2079. raised_for=
  2080. fi
  2081. # prepare HTML versions of elements
  2082. info_html=
  2083. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  2084. raised_for_html=
  2085. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  2086. # -----------------------------------------------------------------------------
  2087. # send the slack notification
  2088. # slack aggregates posts from the same username
  2089. # so we use "${host} ${status}" as the bot username, to make them diff
  2090. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  2091. SENT_SLACK=$?
  2092. # -----------------------------------------------------------------------------
  2093. # send the hangouts notification
  2094. # hangouts aggregates posts from the same room
  2095. # so we use "${host} ${status}" as the room, to make them diff
  2096. send_hangouts "${to_hangouts}"
  2097. SENT_HANGOUTS=$?
  2098. # -----------------------------------------------------------------------------
  2099. # send the Microsoft Teams notification
  2100. # Microsoft teams aggregates posts from the same username
  2101. # so we use "${host} ${status}" as the bot username, to make them diff
  2102. send_msteams "${MSTEAMS_WEBHOOK_URL}" "${to_msteams}"
  2103. SENT_MSTEAMS=$?
  2104. # -----------------------------------------------------------------------------
  2105. # send the rocketchat notification
  2106. # rocketchat aggregates posts from the same username
  2107. # so we use "${host} ${status}" as the bot username, to make them diff
  2108. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  2109. SENT_ROCKETCHAT=$?
  2110. # -----------------------------------------------------------------------------
  2111. # send the alerta notification
  2112. # alerta aggregates posts from the same username
  2113. # so we use "${host} ${status}" as the bot username, to make them diff
  2114. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  2115. SENT_ALERTA=$?
  2116. # -----------------------------------------------------------------------------
  2117. # send the flock notification
  2118. # flock aggregates posts from the same username
  2119. # so we use "${host} ${status}" as the bot username, to make them diff
  2120. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  2121. SENT_FLOCK=$?
  2122. # -----------------------------------------------------------------------------
  2123. # send the discord notification
  2124. # discord aggregates posts from the same username
  2125. # so we use "${host} ${status}" as the bot username, to make them diff
  2126. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  2127. SENT_DISCORD=$?
  2128. # -----------------------------------------------------------------------------
  2129. # send the pushover notification
  2130. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  2131. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  2132. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  2133. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  2134. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  2135. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  2136. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  2137. <small><small>The source of this alarm is line ${src}</small></small>
  2138. "
  2139. SENT_PUSHOVER=$?
  2140. # -----------------------------------------------------------------------------
  2141. # send the pushbullet notification
  2142. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  2143. Severity: ${severity}\\n
  2144. Chart: ${chart}\\n
  2145. Family: ${family}\\n
  2146. ${date}\\n
  2147. The source of this alarm is line ${src}"
  2148. SENT_PUSHBULLET=$?
  2149. # -----------------------------------------------------------------------------
  2150. # send the twilio SMS
  2151. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2152. Severity: ${severity}
  2153. Chart: ${chart}
  2154. Family: ${family}
  2155. ${info}"
  2156. SENT_TWILIO=$?
  2157. # -----------------------------------------------------------------------------
  2158. # send the messagebird SMS
  2159. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2160. Severity: ${severity}
  2161. Chart: ${chart}
  2162. Family: ${family}
  2163. ${info}"
  2164. SENT_MESSAGEBIRD=$?
  2165. # -----------------------------------------------------------------------------
  2166. # send the kavenegar SMS
  2167. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2168. Severity: ${severity}
  2169. Chart: ${chart}
  2170. Family: ${family}
  2171. ${info}"
  2172. SENT_KAVENEGAR=$?
  2173. # -----------------------------------------------------------------------------
  2174. # send the telegram.org message
  2175. # https://core.telegram.org/bots/api#formatting-options
  2176. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  2177. ${chart} (${family})
  2178. <a href=\"${goto_url}\">${alarm}</a>
  2179. <i>${info}</i>"
  2180. SENT_TELEGRAM=$?
  2181. # -----------------------------------------------------------------------------
  2182. # send the kafka message
  2183. send_kafka
  2184. SENT_KAFKA=$?
  2185. # -----------------------------------------------------------------------------
  2186. # send the pagerduty.com message
  2187. send_pd "${to_pd}"
  2188. SENT_PD=$?
  2189. # -----------------------------------------------------------------------------
  2190. # send the fleep message
  2191. send_fleep "${to_fleep}"
  2192. SENT_FLEEP=$?
  2193. # -----------------------------------------------------------------------------
  2194. # send the Prowl message
  2195. send_prowl "${to_prowl}"
  2196. SENT_PROWL=$?
  2197. # -----------------------------------------------------------------------------
  2198. # send the irc message
  2199. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${IRC_PORT}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  2200. Severity: ${severity}
  2201. Chart: ${chart}
  2202. Family: ${family}
  2203. ${info}"
  2204. SENT_IRC=$?
  2205. # -----------------------------------------------------------------------------
  2206. # send the SMS message with smstools3
  2207. send_sms "${to_sms}"
  2208. SENT_SMS=$?
  2209. # -----------------------------------------------------------------------------
  2210. # send the custom message
  2211. send_custom() {
  2212. # is it enabled?
  2213. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  2214. # do we have any sender?
  2215. [ -z "${1}" ] && return 1
  2216. # call the custom_sender function
  2217. custom_sender "${@}"
  2218. }
  2219. send_custom "${to_custom}"
  2220. SENT_CUSTOM=$?
  2221. # -----------------------------------------------------------------------------
  2222. # send hipchat message
  2223. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  2224. ${host} ${status_message}<br/> \
  2225. <b>${alarm}</b> ${info_html}<br/> \
  2226. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  2227. <b>${date}${raised_for_html}</b><br/> \
  2228. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  2229. (source of alarm ${src}) \
  2230. "
  2231. SENT_HIPCHAT=$?
  2232. # -----------------------------------------------------------------------------
  2233. # send the Amazon SNS message
  2234. send_awssns "${to_awssns}"
  2235. SENT_AWSSNS=$?
  2236. # -----------------------------------------------------------------------------
  2237. # send the Matrix message
  2238. send_matrix "${MATRIX_HOMESERVER}" "${to_matrix}"
  2239. SENT_MATRIX=$?
  2240. # -----------------------------------------------------------------------------
  2241. # send the syslog message
  2242. send_syslog "${to_syslog}"
  2243. SENT_SYSLOG=$?
  2244. # -----------------------------------------------------------------------------
  2245. # send the email
  2246. IFS='' read -r -d '' email_plaintext_part <<EOF
  2247. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  2248. Content-Disposition: inline
  2249. Content-Transfer-Encoding: 8bit
  2250. ${host} ${status_message}
  2251. ${alarm} ${info}
  2252. ${raised_for}
  2253. Chart : ${chart}
  2254. Family : ${family}
  2255. Severity: ${severity}
  2256. URL : ${goto_url}
  2257. Source : ${src}
  2258. Date : ${date}
  2259. Notification generated on ${host}
  2260. Evaluated Expression : ${calc_expression}
  2261. Expression Variables : ${calc_param_values}
  2262. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  2263. EOF
  2264. if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
  2265. send_email <<EOF
  2266. To: ${to_email}
  2267. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  2268. MIME-Version: 1.0
  2269. Content-Type: multipart/alternative; boundary="multipart-boundary"
  2270. ${email_thread_headers}
  2271. X-Netdata-Severity: ${status,,}
  2272. X-Netdata-Alert-Name: $name
  2273. X-Netdata-Chart: $chart
  2274. X-Netdata-Family: $family
  2275. X-Netdata-Classification: $classification
  2276. X-Netdata-Host: $host
  2277. X-Netdata-Role: $roles
  2278. This is a MIME-encoded multipart message
  2279. --multipart-boundary
  2280. ${email_plaintext_part}
  2281. --multipart-boundary--
  2282. EOF
  2283. else
  2284. now=$(date "+%s")
  2285. if [ -n "$total_warn_alarms" ]; then
  2286. while read -d, -r pair; do
  2287. IFS='=' read -r key val <<<"$pair"
  2288. date_w=$(date --date=@${val} "${date_format}" 2>/dev/null)
  2289. [ -z "${date_w}" ] && date_w=$(date "${date_format}" 2>/dev/null)
  2290. [ -z "${date_w}" ] && date_w=$(date --date=@${val} 2>/dev/null)
  2291. [ -z "${date_w}" ] && date_w=$(date 2>/dev/null)
  2292. elapsed=$((now - val))
  2293. duration4human ${elapsed} >/dev/null
  2294. elapsed_txt="${REPLY}"
  2295. WARN_ALARMS+="
  2296. <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
  2297. <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
  2298. <tbody>
  2299. <tr>
  2300. <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
  2301. <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2302. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2303. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
  2304. <tbody>
  2305. <tr>
  2306. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2307. <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
  2308. </td>
  2309. </tr>
  2310. <tr>
  2311. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
  2312. <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_w}</div>
  2313. </td>
  2314. </tr>
  2315. </tbody>
  2316. </table>
  2317. </div>
  2318. <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2319. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2320. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
  2321. <tbody>
  2322. <tr>
  2323. <td style=\"vertical-align:top;padding-top:13px;\">
  2324. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
  2325. <tbody>
  2326. <tr>
  2327. <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2328. <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#555555;\"><span style=\"background-color:#FFF8E1; border: 1px solid #FFC300; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
  2329. Warning for ${elapsed_txt}
  2330. </span></div>
  2331. </td>
  2332. </tr>
  2333. </tbody>
  2334. </table>
  2335. </td>
  2336. </tr>
  2337. </tbody>
  2338. </table>
  2339. </div>
  2340. <!--[if mso | IE]></td></tr></table><![endif]-->
  2341. </td>
  2342. </tr>
  2343. </tbody>
  2344. </table>
  2345. </div>
  2346. "
  2347. done <<<"$total_warn_alarms,"
  2348. fi
  2349. if [ -n "$total_crit_alarms" ]; then
  2350. while read -d, -r pair; do
  2351. IFS='=' read -r key val <<<"$pair"
  2352. date_c=$(date --date=@${val} "${date_format}" 2>/dev/null)
  2353. [ -z "${date_c}" ] && date_c=$(date "${date_format}" 2>/dev/null)
  2354. [ -z "${date_c}" ] && date_c=$(date --date=@${val} 2>/dev/null)
  2355. [ -z "${date_c}" ] && date_c=$(date 2>/dev/null)
  2356. elapsed=$((now - val))
  2357. duration4human ${elapsed} >/dev/null
  2358. elapsed_txt="${REPLY}"
  2359. CRIT_ALARMS+="
  2360. <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
  2361. <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
  2362. <tbody>
  2363. <tr>
  2364. <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
  2365. <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2366. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2367. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
  2368. <tbody>
  2369. <tr>
  2370. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2371. <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
  2372. </td>
  2373. </tr>
  2374. <tr>
  2375. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
  2376. <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_c}</div>
  2377. </td>
  2378. </tr>
  2379. </tbody>
  2380. </table>
  2381. </div>
  2382. <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2383. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2384. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
  2385. <tbody>
  2386. <tr>
  2387. <td style=\"vertical-align:top;padding-top:13px;\">
  2388. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
  2389. <tbody>
  2390. <tr>
  2391. <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2392. <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#35414A;\"><span style=\"background-color:#FFEBEF; border: 1px solid #FF4136; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
  2393. Critical for ${elapsed_txt}
  2394. </span></div>
  2395. </td>
  2396. </tr>
  2397. </tbody>
  2398. </table>
  2399. </td>
  2400. </tr>
  2401. </tbody>
  2402. </table>
  2403. </div>
  2404. <!--[if mso | IE]></td></tr></table><![endif]-->
  2405. </td>
  2406. </tr>
  2407. </tbody>
  2408. </table>
  2409. </div>
  2410. "
  2411. done <<<"$total_crit_alarms,"
  2412. fi
  2413. if (( total_warnings + total_critical > 15 )); then
  2414. EXTRA_ALARMS_LIST_TEXT="(Showing latest 15 alerts)"
  2415. fi
  2416. if [ -n "$edit_command_line" ]; then
  2417. IFS='=' read -r edit_command line s_host <<<"$edit_command_line"
  2418. fi
  2419. IFS='' read -r -d '' email_html_part <<EOF
  2420. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  2421. Content-Disposition: inline
  2422. Content-Transfer-Encoding: 8bit
  2423. <!doctype html>
  2424. <html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office">
  2425. <head>
  2426. <title>
  2427. </title>
  2428. <!--[if !mso]><!-->
  2429. <meta http-equiv="X-UA-Compatible" content="IE=edge">
  2430. <!--<![endif]-->
  2431. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  2432. <meta name="viewport" content="width=device-width, initial-scale=1">
  2433. <style type="text/css">
  2434. #outlook a { padding:0; }
  2435. body { margin:0;padding:0;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%; }
  2436. table, td { border-collapse:collapse;mso-table-lspace:0pt;mso-table-rspace:0pt; }
  2437. img { border:0;height:auto;line-height:100%; outline:none;text-decoration:none;-ms-interpolation-mode:bicubic; }
  2438. p { display:block;margin:13px 0; }
  2439. </style>
  2440. <!--[if mso]>
  2441. <xml>
  2442. <o:OfficeDocumentSettings>
  2443. <o:AllowPNG/>
  2444. <o:PixelsPerInch>96</o:PixelsPerInch>
  2445. </o:OfficeDocumentSettings>
  2446. </xml>
  2447. <![endif]-->
  2448. <!--[if lte mso 11]>
  2449. <style type="text/css">
  2450. .mj-outlook-group-fix { width:100% !important; }
  2451. </style>
  2452. <![endif]-->
  2453. <!--[if !mso]><!-->
  2454. <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap" rel="stylesheet" type="text/css">
  2455. <link href="https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700" rel="stylesheet" type="text/css">
  2456. <style type="text/css">
  2457. @import url(https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap);
  2458. @import url(https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700);
  2459. </style>
  2460. <!--<![endif]-->
  2461. <style type="text/css">
  2462. @media only screen and (min-width:100px) {
  2463. .mj-column-px-130 { width:130px !important; max-width: 130px; }
  2464. .mj-column-per-50 { width:50% !important; max-width: 50%; }
  2465. .mj-column-per-70 { width:70% !important; max-width: 70%; }
  2466. .mj-column-per-30 { width:30% !important; max-width: 30%; }
  2467. .mj-column-per-100 { width:100% !important; max-width: 100%; }
  2468. .mj-column-px-66 { width:66px !important; max-width: 66px; }
  2469. .mj-column-px-400 { width:400px !important; max-width: 400px; }
  2470. }
  2471. </style>
  2472. <style type="text/css">
  2473. @media only screen and (max-width:100px) {
  2474. table.mj-full-width-mobile { width: 100% !important; }
  2475. td.mj-full-width-mobile { width: auto !important; }
  2476. }
  2477. </style>
  2478. </head>
  2479. <body style="word-spacing:normal;">
  2480. <div class="svgbg" style="background-image: url('https://staging.netdata.cloud/static/email/img/isotype_600.png'); background-repeat: no-repeat; background-position: top center; background-size: 600px 192px;">
  2481. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2482. <div style="margin:0px auto;max-width:600px;">
  2483. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2484. <tbody>
  2485. <tr>
  2486. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:50px;padding-left:0;text-align:left;">
  2487. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:130px;" ><![endif]-->
  2488. <div class="mj-column-px-130 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:130px;">
  2489. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2490. <tbody>
  2491. <tr>
  2492. <td align="center" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2493. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2494. <tbody>
  2495. <tr>
  2496. <td style="width:130px;">
  2497. <img alt="Netdata Logo" height="auto" src="https://app.netdata.cloud/static/email/img/full_logo.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="130">
  2498. </td>
  2499. </tr>
  2500. </tbody>
  2501. </table>
  2502. </td>
  2503. </tr>
  2504. </tbody>
  2505. </table>
  2506. </div>
  2507. <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:300px;" ><![endif]-->
  2508. <div class="mj-column-per-50 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;">
  2509. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2510. <tbody>
  2511. <tr>
  2512. <td style="vertical-align:top;padding-top:4px;">
  2513. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2514. <tbody>
  2515. <tr>
  2516. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:10px;word-break:break-word;">
  2517. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Notification</div>
  2518. </td>
  2519. </tr>
  2520. </tbody>
  2521. </table>
  2522. </td>
  2523. </tr>
  2524. </tbody>
  2525. </table>
  2526. </div>
  2527. <!--[if mso | IE]></td></tr></table><![endif]-->
  2528. </td>
  2529. </tr>
  2530. </tbody>
  2531. </table>
  2532. </div>
  2533. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="no-collapse-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2534. <div class="no-collapse" style="border-collapse: initial; margin: 0px auto; border-radius: 4px; max-width: 600px;">
  2535. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;border-radius:4px;">
  2536. <tbody>
  2537. <tr>
  2538. <td style="border:1px solid ${border_color};direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2539. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2540. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2541. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2542. <tbody>
  2543. <tr>
  2544. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2545. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:418.6px;" ><![endif]-->
  2546. <div class="mj-column-per-70 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:70%;">
  2547. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2548. <tbody>
  2549. <tr>
  2550. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:15px;word-break:break-word;">
  2551. <div style="font-family:Open Sans, sans-serif;font-size:20px;font-weight:700;line-height:1;text-align:left;color:#35414A;">${name}</div>
  2552. </td>
  2553. </tr>
  2554. </tbody>
  2555. </table>
  2556. </div>
  2557. <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:179.4px;" ><![endif]-->
  2558. <div class="mj-column-per-30 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:30%;">
  2559. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2560. <tbody>
  2561. <tr>
  2562. <td align="right" style="font-size:0px;padding:10px 25px;padding-right:25px;word-break:break-word;">
  2563. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2564. <tbody>
  2565. <tr>
  2566. <td style="width:100px;">
  2567. <img height="auto" src="${alarm_badge}" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="100"/>
  2568. </td>
  2569. </tr>
  2570. </tbody>
  2571. </table>
  2572. </td>
  2573. </tr>
  2574. </tbody>
  2575. </table>
  2576. </div>
  2577. <!--[if mso | IE]></td></tr></table><![endif]-->
  2578. </td>
  2579. </tr>
  2580. </tbody>
  2581. </table>
  2582. </div>
  2583. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2584. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2585. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2586. <tbody>
  2587. <tr>
  2588. <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
  2589. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2590. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2591. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2592. <tbody>
  2593. <tr>
  2594. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2595. <div style="font-family:IBM Plex Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">on ${host}</div>
  2596. </td>
  2597. </tr>
  2598. </tbody>
  2599. </table>
  2600. </div>
  2601. <!--[if mso | IE]></td></tr></table><![endif]-->
  2602. </td>
  2603. </tr>
  2604. </tbody>
  2605. </table>
  2606. </div>
  2607. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2608. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2609. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2610. <tbody>
  2611. <tr>
  2612. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2613. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2614. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2615. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2616. <tbody>
  2617. <tr>
  2618. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2619. <div style="font-family:Open Sans, sans-serif;font-size:26px;font-weight:700;line-height:1;text-align:left;color:#35414A;"><span style="color: ${text_color}; font-size:26px; background: ${background_color}; padding:4px 24px; border-radius: 36px">${value_string}
  2620. </span></div>
  2621. </td>
  2622. </tr>
  2623. </tbody>
  2624. </table>
  2625. </div>
  2626. <!--[if mso | IE]></td></tr></table><![endif]-->
  2627. </td>
  2628. </tr>
  2629. </tbody>
  2630. </table>
  2631. </div>
  2632. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2633. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2634. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2635. <tbody>
  2636. <tr>
  2637. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2638. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2639. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2640. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2641. <tbody>
  2642. <tr>
  2643. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2644. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:21px;text-align:left;color:#35414A;">Details: ${info}</div>
  2645. </td>
  2646. </tr>
  2647. </tbody>
  2648. </table>
  2649. </div>
  2650. <!--[if mso | IE]></td></tr></table><![endif]-->
  2651. </td>
  2652. </tr>
  2653. </tbody>
  2654. </table>
  2655. </div>
  2656. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2657. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2658. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2659. <tbody>
  2660. <tr>
  2661. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2662. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2663. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2664. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2665. <tbody>
  2666. <tr>
  2667. <td align="center" vertical-align="middle" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2668. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:separate;width:100%;line-height:100%;">
  2669. <tr>
  2670. <td
  2671. align="center" bgcolor="${border_color}" role="presentation" style="border:none;border-radius:3px;cursor:auto;height:44px;background:${border_color};" valign="middle">
  2672. <p style="display:block;background:${border_color};color:#ffffff;font-size:13px;font-weight:600;line-height:44px;margin:0;text-decoration:none;text-transform:none;mso-padding-alt:0px;border-radius:3px;">
  2673. <a href="${goto_url}" style="color: ${action_text_color}; text-decoration: none; width: 100%; display: inline-block">GO TO CHART</a>
  2674. </p>
  2675. </td>
  2676. </tr>
  2677. </table>
  2678. </td>
  2679. </tr>
  2680. </tbody>
  2681. </table>
  2682. </div>
  2683. <!--[if mso | IE]></td></tr></table><![endif]-->
  2684. </td>
  2685. </tr>
  2686. </tbody>
  2687. </table>
  2688. </div>
  2689. <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
  2690. </td>
  2691. </tr>
  2692. </tbody>
  2693. </table>
  2694. </div>
  2695. <!--[if mso | IE]></td></tr></table><![endif]-->
  2696. <div style="height:32px;line-height:32px;">&#8202;</div>
  2697. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2698. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; background: ${background_color}; background-color: ${background_color}; margin: 0px auto; border-radius: 4px; max-width: 600px;">
  2699. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:${background_color};background-color:${background_color};width:100%;border-radius:4px;">
  2700. <tbody>
  2701. <tr>
  2702. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2703. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2704. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2705. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2706. <tbody>
  2707. <tr>
  2708. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2709. <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Chart:
  2710. <span style="font-weight:700; font-size:20px">${chart}</span></div>
  2711. </td>
  2712. </tr>
  2713. <tr>
  2714. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2715. <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Family:
  2716. <span style="font-weight:700; font-size:20px">${family}</span></div>
  2717. </td>
  2718. </tr>
  2719. <tr>
  2720. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
  2721. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">${rich_status_raised_for}</div>
  2722. </td>
  2723. </tr>
  2724. <tr>
  2725. <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2726. <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
  2727. </p>
  2728. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
  2729. </td></tr></table><![endif]-->
  2730. </td>
  2731. </tr>
  2732. <tr>
  2733. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2734. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">On
  2735. <span style="font-weight:600">${date}</span></div>
  2736. </td>
  2737. </tr>
  2738. <tr>
  2739. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2740. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">By:
  2741. <span style="font-weight:600">${host}</span></div>
  2742. </td>
  2743. </tr>
  2744. <tr>
  2745. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
  2746. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">Global time:
  2747. <span style="font-weight:600">${date_utc}</span></div>
  2748. </td>
  2749. </tr>
  2750. <tr>
  2751. <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2752. <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
  2753. </p>
  2754. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
  2755. </td></tr></table><![endif]-->
  2756. </td>
  2757. </tr>
  2758. <tr>
  2759. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2760. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Classification:
  2761. <span style="font-weight:600">${classification}</span></div>
  2762. </td>
  2763. </tr>
  2764. <tr>
  2765. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2766. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Role:
  2767. <span style="font-weight:600">${roles}</span></div>
  2768. </td>
  2769. </tr>
  2770. </tbody>
  2771. </table>
  2772. </div>
  2773. <!--[if mso | IE]></td></tr></table><![endif]-->
  2774. </td>
  2775. </tr>
  2776. </tbody>
  2777. </table>
  2778. </div>
  2779. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2780. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2781. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2782. <tbody>
  2783. <tr>
  2784. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
  2785. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
  2786. <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
  2787. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2788. <tbody>
  2789. <tr>
  2790. <td style="vertical-align:top;padding:0;">
  2791. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2792. <tbody>
  2793. <tr>
  2794. <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2795. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2796. <tbody>
  2797. <tr>
  2798. <td style="width:48px;">
  2799. <img height="auto" src="https://app.netdata.cloud/static/email/img/community_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
  2800. </td>
  2801. </tr>
  2802. </tbody>
  2803. </table>
  2804. </td>
  2805. </tr>
  2806. </tbody>
  2807. </table>
  2808. </td>
  2809. </tr>
  2810. </tbody>
  2811. </table>
  2812. </div>
  2813. <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
  2814. <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
  2815. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2816. <tbody>
  2817. <tr>
  2818. <td style="vertical-align:top;padding-left:0;">
  2819. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2820. <tbody>
  2821. <tr>
  2822. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2823. <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Want to know more about this alert?</div>
  2824. </td>
  2825. </tr>
  2826. <tr>
  2827. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2828. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;">Join the troubleshooting discussion for this alert on our <a href="https://community.netdata.cloud/t/${name//[._]/-}" class="link" style="color: #00AB44; text-decoration: none;">community forums</a>.</div>
  2829. </td>
  2830. </tr>
  2831. </tbody>
  2832. </table>
  2833. </td>
  2834. </tr>
  2835. </tbody>
  2836. </table>
  2837. </div>
  2838. <!--[if mso | IE]></td></tr></table><![endif]-->
  2839. </td>
  2840. </tr>
  2841. </tbody>
  2842. </table>
  2843. </div>
  2844. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2845. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2846. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2847. <tbody>
  2848. <tr>
  2849. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
  2850. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
  2851. <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
  2852. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2853. <tbody>
  2854. <tr>
  2855. <td style="vertical-align:top;padding:0;">
  2856. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2857. <tbody>
  2858. <tr>
  2859. <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2860. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2861. <tbody>
  2862. <tr>
  2863. <td style="width:48px;">
  2864. <img height="auto" src="https://app.netdata.cloud/static/email/img/configure_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
  2865. </td>
  2866. </tr>
  2867. </tbody>
  2868. </table>
  2869. </td>
  2870. </tr>
  2871. </tbody>
  2872. </table>
  2873. </td>
  2874. </tr>
  2875. </tbody>
  2876. </table>
  2877. </div>
  2878. <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
  2879. <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
  2880. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2881. <tbody>
  2882. <tr>
  2883. <td style="vertical-align:top;padding-left:0;">
  2884. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2885. <tbody>
  2886. <tr>
  2887. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2888. <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Need to configure this alert?</div>
  2889. </td>
  2890. </tr>
  2891. <tr>
  2892. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2893. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;"><span style="color: #00AB44"><a href="https://learn.netdata.cloud/docs/agent/health/notifications#:~:text=To%20edit%20it%20on%20your,have%20one%20or%20more%20destinations" class="link" style="color: #00AB44; text-decoration: none;">Edit</a></span> this alert's configuration file by logging into $s_host and running the following command:</div>
  2894. </td>
  2895. </tr>
  2896. <tr>
  2897. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:8px;padding-left:0;word-break:break-word;">
  2898. <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1.3;text-align:left;color:#35414A;">${edit_command} <br>
  2899. <br>The alarm to edit is at line ${line}</div>
  2900. </td>
  2901. </tr>
  2902. </tbody>
  2903. </table>
  2904. </td>
  2905. </tr>
  2906. </tbody>
  2907. </table>
  2908. </div>
  2909. <!--[if mso | IE]></td></tr></table><![endif]-->
  2910. </td>
  2911. </tr>
  2912. </tbody>
  2913. </table>
  2914. </div>
  2915. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="history-wrapper-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2916. <div class="history-wrapper" style="background: #F7F8F8; background-color: #F7F8F8; margin: 0px auto; max-width: 100%;">
  2917. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:#F7F8F8;background-color:#F7F8F8;width:100%;">
  2918. <tbody>
  2919. <tr>
  2920. <td style="direction:ltr;font-size:0px;padding:0;padding-bottom:24px;text-align:center;">
  2921. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2922. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2923. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2924. <tbody>
  2925. <tr>
  2926. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:12px;text-align:center;">
  2927. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2928. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2929. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2930. <tbody>
  2931. <tr>
  2932. <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2933. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:center;color:#35414A;">The node has
  2934. <span style="font-weight:600">${total_warnings} warning</span>
  2935. and
  2936. <span style="font-weight:600">${total_critical} critical</span>
  2937. additional active alert(s)</div>
  2938. </td>
  2939. </tr>
  2940. <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2941. <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:center;color:#35414A;">${EXTRA_ALARMS_LIST_TEXT}</div>
  2942. </td>
  2943. </tr>
  2944. </tbody>
  2945. </table>
  2946. </div>
  2947. <!--[if mso | IE]></td></tr></table><![endif]-->
  2948. </td>
  2949. </tr>
  2950. </tbody>
  2951. </table>
  2952. </div>
  2953. ${CRIT_ALARMS}
  2954. ${WARN_ALARMS}
  2955. <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
  2956. </td>
  2957. </tr>
  2958. </tbody>
  2959. </table>
  2960. </div>
  2961. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2962. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2963. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2964. <tbody>
  2965. <tr>
  2966. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2967. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2968. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2969. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2970. <tbody>
  2971. <tr>
  2972. <td style="vertical-align:top;padding-top:44px;padding-bottom:12px;">
  2973. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2974. <tbody>
  2975. <tr>
  2976. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;padding-bottom:0;word-break:break-word;">
  2977. <div style="font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:center;color:#35414A;">© Netdata 2021 - The real-time performance and health monitoring</div>
  2978. </td>
  2979. </tr>
  2980. </tbody>
  2981. </table>
  2982. </td>
  2983. </tr>
  2984. </tbody>
  2985. </table>
  2986. </div>
  2987. <!--[if mso | IE]></td></tr></table><![endif]-->
  2988. </td>
  2989. </tr>
  2990. </tbody>
  2991. </table>
  2992. </div>
  2993. <!--[if mso | IE]></td></tr></table><![endif]-->
  2994. </div>
  2995. </body>
  2996. </html>
  2997. EOF
  2998. send_email <<EOF
  2999. To: ${to_email}
  3000. Subject: ${html_email_subject}
  3001. MIME-Version: 1.0
  3002. Content-Type: multipart/alternative; boundary="multipart-boundary"
  3003. ${email_thread_headers}
  3004. X-Netdata-Severity: ${status,,}
  3005. X-Netdata-Alert-Name: $name
  3006. X-Netdata-Chart: $chart
  3007. X-Netdata-Family: $family
  3008. X-Netdata-Classification: $classification
  3009. X-Netdata-Host: $host
  3010. X-Netdata-Role: $roles
  3011. This is a MIME-encoded multipart message
  3012. --multipart-boundary
  3013. ${email_plaintext_part}
  3014. --multipart-boundary
  3015. ${email_html_part}
  3016. --multipart-boundary--
  3017. EOF
  3018. fi
  3019. SENT_EMAIL=$?
  3020. # -----------------------------------------------------------------------------
  3021. # send the EVENT to Dynatrace
  3022. send_dynatrace "${host}" "${chart}" "${name}" "${status}"
  3023. SENT_DYNATRACE=$?
  3024. # -----------------------------------------------------------------------------
  3025. # send the EVENT to Stackpulse
  3026. send_stackpulse
  3027. SENT_STACKPULSE=$?
  3028. # -----------------------------------------------------------------------------
  3029. # send messages to Opsgenie
  3030. send_opsgenie
  3031. SENT_OPSGENIE=$?
  3032. # -----------------------------------------------------------------------------
  3033. # send messages to Gotify
  3034. send_gotify
  3035. SENT_GOTIFY=$?
  3036. # -----------------------------------------------------------------------------
  3037. # let netdata know
  3038. for state in "${SENT_EMAIL}" \
  3039. "${SENT_PUSHOVER}" \
  3040. "${SENT_TELEGRAM}" \
  3041. "${SENT_SLACK}" \
  3042. "${SENT_HANGOUTS}" \
  3043. "${SENT_ROCKETCHAT}" \
  3044. "${SENT_ALERTA}" \
  3045. "${SENT_FLOCK}" \
  3046. "${SENT_DISCORD}" \
  3047. "${SENT_TWILIO}" \
  3048. "${SENT_HIPCHAT}" \
  3049. "${SENT_MESSAGEBIRD}" \
  3050. "${SENT_KAVENEGAR}" \
  3051. "${SENT_PUSHBULLET}" \
  3052. "${SENT_KAFKA}" \
  3053. "${SENT_PD}" \
  3054. "${SENT_FLEEP}" \
  3055. "${SENT_PROWL}" \
  3056. "${SENT_CUSTOM}" \
  3057. "${SENT_IRC}" \
  3058. "${SENT_AWSSNS}" \
  3059. "${SENT_MATRIX}" \
  3060. "${SENT_SYSLOG}" \
  3061. "${SENT_SMS}" \
  3062. "${SENT_MSTEAMS}" \
  3063. "${SENT_DYNATRACE}" \
  3064. "${SENT_STACKPULSE}" \
  3065. "${SENT_OPSGENIE}" \
  3066. "${SENT_GOTIFY}"; do
  3067. if [ "${state}" -eq 0 ]; then
  3068. # we sent something
  3069. exit 0
  3070. fi
  3071. done
  3072. # we did not send anything
  3073. exit 1