alarm-notify.sh.in 134 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594
  1. #!/usr/bin/env bash
  2. #shellcheck source=/dev/null disable=SC2086,SC2154
  3. # netdata
  4. # real-time performance and health monitoring, done right!
  5. # (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
  6. # SPDX-License-Identifier: GPL-3.0-or-later
  7. #
  8. # Script to send alarm notifications for netdata
  9. #
  10. # Features:
  11. # - multiple notification methods
  12. # - multiple roles per alarm
  13. # - multiple recipients per role
  14. # - severity filtering per recipient
  15. #
  16. # Supported notification methods:
  17. # - emails by @ktsaou
  18. # - slack.com notifications by @ktsaou
  19. # - alerta.io notifications by @kattunga
  20. # - discordapp.com notifications by @lowfive
  21. # - pushover.net notifications by @ktsaou
  22. # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
  23. # - telegram.org notifications by @hashworks #1002
  24. # - twilio.com notifications by Levi Blaney @shadycuz #1211
  25. # - kafka notifications by @ktsaou #1342
  26. # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
  27. # - messagebird.com notifications by @tech_no_logical #1453
  28. # - hipchat notifications by @ktsaou #1561
  29. # - fleep notifications by @Ferroin
  30. # - prowlapp.com notifications by @Ferroin
  31. # - irc notifications by @manosf
  32. # - custom notifications by @ktsaou
  33. # - syslog messages by @Ferroin
  34. # - Microsoft Team notification by @tioumen
  35. # - RocketChat notifications by @Hermsi1337 #3777
  36. # - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
  37. # - Dynatrace Event by @illumine
  38. # - Stackpulse Event by @thiagoftsm
  39. # - Opsgenie by @thiaoftsm #9858
  40. # - Gotify by @coffeegrind123
  41. # -----------------------------------------------------------------------------
  42. # testing notifications
  43. if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
  44. if [ "${2}" = "test" ]; then
  45. recipient="${1}"
  46. else
  47. recipient="${2}"
  48. fi
  49. [ -z "${recipient}" ] && recipient="sysadmin"
  50. id=1
  51. last="CLEAR"
  52. test_res=0
  53. for x in "WARNING" "CRITICAL" "CLEAR"; do
  54. echo >&2
  55. echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
  56. "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
  57. #shellcheck disable=SC2181
  58. if [ $? -ne 0 ]; then
  59. echo >&2 "# FAILED"
  60. test_res=1
  61. else
  62. echo >&2 "# OK"
  63. fi
  64. last="${x}"
  65. id=$((id + 1))
  66. done
  67. exit $test_res
  68. fi
  69. export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
  70. export LC_ALL=C
  71. # -----------------------------------------------------------------------------
  72. PROGRAM_NAME="$(basename "${0}")"
  73. logdate() {
  74. date "+%Y-%m-%d %H:%M:%S"
  75. }
  76. log() {
  77. local status="${1}"
  78. shift
  79. echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
  80. }
  81. warning() {
  82. log WARNING "${@}"
  83. }
  84. error() {
  85. log ERROR "${@}"
  86. }
  87. info() {
  88. log INFO "${@}"
  89. }
  90. fatal() {
  91. log FATAL "${@}"
  92. exit 1
  93. }
  94. debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
  95. debug() {
  96. [ "${debug}" = "1" ] && log DEBUG "${@}"
  97. }
  98. docurl() {
  99. if [ -z "${curl}" ]; then
  100. error "${curl} is unset."
  101. return 1
  102. fi
  103. if [ "${debug}" = "1" ]; then
  104. echo >&2 "--- BEGIN curl command ---"
  105. printf >&2 "%q " ${curl} "${@}"
  106. echo >&2
  107. echo >&2 "--- END curl command ---"
  108. local out code ret
  109. out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
  110. code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
  111. ret=$?
  112. echo >&2 "--- BEGIN received response ---"
  113. cat >&2 "${out}"
  114. echo >&2
  115. echo >&2 "--- END received response ---"
  116. echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
  117. rm "${out}"
  118. echo "${code}"
  119. return ${ret}
  120. fi
  121. ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
  122. return $?
  123. }
  124. # -----------------------------------------------------------------------------
  125. # List of all the notification mechanisms we support.
  126. # Used in a couple of places to write more compact code.
  127. method_names="
  128. email
  129. pushover
  130. pushbullet
  131. telegram
  132. slack
  133. alerta
  134. flock
  135. discord
  136. hipchat
  137. twilio
  138. messagebird
  139. pd
  140. fleep
  141. syslog
  142. custom
  143. msteams
  144. kavenegar
  145. prowl
  146. irc
  147. awssns
  148. rocketchat
  149. sms
  150. hangouts
  151. dynatrace
  152. matrix
  153. "
  154. # -----------------------------------------------------------------------------
  155. # this is to be overwritten by the config file
  156. custom_sender() {
  157. info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
  158. }
  159. # -----------------------------------------------------------------------------
  160. # check for BASH v4+ (required for associative arrays)
  161. if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
  162. fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
  163. fi
  164. # -----------------------------------------------------------------------------
  165. # defaults to allow running this script by hand
  166. [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
  167. [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
  168. [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
  169. [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
  170. [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://api.netdata.cloud"
  171. # -----------------------------------------------------------------------------
  172. # parse command line parameters
  173. if [[ ${1} = "unittest" ]]; then
  174. unittest=1 # enable unit testing mode
  175. roles="${2}" # the role that should be used for unit testing
  176. cfgfile="${3}" # the location of the config file to use for unit testing
  177. status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  178. old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  179. elif [[ ${1} = "dump_methods" ]]; then
  180. dump_methods=1
  181. status="WARNING"
  182. else
  183. roles="${1}" # the roles that should be notified for this event
  184. args_host="${2}" # the host generated this event
  185. unique_id="${3}" # the unique id of this event
  186. alarm_id="${4}" # the unique id of the alarm that generated this event
  187. event_id="${5}" # the incremental id of the event, for this alarm id
  188. when="${6}" # the timestamp this event occurred
  189. name="${7}" # the name of the alarm, as given in netdata health.d entries
  190. chart="${8}" # the name of the chart (type.id)
  191. family="${9}" # the family of the chart
  192. status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  193. old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
  194. value="${12}" # the current value of the alarm
  195. old_value="${13}" # the previous value of the alarm
  196. src="${14}" # the line number and file the alarm has been configured
  197. duration="${15}" # the duration in seconds of the previous alarm state
  198. non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
  199. units="${17}" # the units of the value
  200. info="${18}" # a short description of the alarm
  201. value_string="${19}" # friendly value (with units)
  202. # shellcheck disable=SC2034
  203. # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
  204. old_value_string="${20}" # friendly old value (with units), previously named "old_value_string"
  205. calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
  206. calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
  207. total_warnings="${23}" # Total number of alarms in WARNING state
  208. total_critical="${24}" # Total number of alarms in CRITICAL state
  209. total_warn_alarms="${25}" # List of alarms in warning state
  210. total_crit_alarms="${26}" # List of alarms in critical state
  211. classification="${27}" # The class field from .conf files
  212. edit_command_line="${28}" # The command to edit the alarm, with the line number
  213. child_machine_guid="${29}" # If populated, the notification is sent for a child
  214. fi
  215. # -----------------------------------------------------------------------------
  216. # find a suitable hostname to use, if netdata did not supply a hostname
  217. if [ -z "${args_host}" ]; then
  218. this_host=$(hostname -s 2>/dev/null)
  219. host="${this_host}"
  220. args_host="${this_host}"
  221. else
  222. host="${args_host}"
  223. fi
  224. # -----------------------------------------------------------------------------
  225. # screen statuses we don't need to send a notification
  226. # don't do anything if this is not WARNING, CRITICAL or CLEAR
  227. if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
  228. info "not sending notification for ${status} of '${host}.${chart}.${name}'"
  229. exit 1
  230. fi
  231. # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
  232. if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
  233. info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
  234. exit 1
  235. fi
  236. # -----------------------------------------------------------------------------
  237. # load configuration
  238. # By default fetch images from the global public registry.
  239. # This is required by default, since all notification methods need to download
  240. # images via the Internet, and private registries might not be reachable.
  241. # This can be overwritten at the configuration file.
  242. images_base_url="https://registry.my-netdata.io"
  243. # curl options to use
  244. curl_options=""
  245. # hostname handling
  246. use_fqdn="NO"
  247. # needed commands
  248. # if empty they will be searched in the system path
  249. curl=
  250. sendmail=
  251. # enable / disable features
  252. for method_name in ${method_names^^}; do
  253. declare SEND_${method_name}="YES"
  254. declare DEFAULT_RECIPIENT_${method_name}
  255. done
  256. for method_name in ${method_names}; do
  257. declare -A role_recipients_${method_name}
  258. done
  259. # slack configs
  260. SLACK_WEBHOOK_URL=
  261. # Microsoft Teams configs
  262. MSTEAMS_WEBHOOK_URL=
  263. # Legacy Microsoft Teams configs for backwards compatibility:
  264. declare -A role_recipients_msteam
  265. # rocketchat configs
  266. ROCKETCHAT_WEBHOOK_URL=
  267. # alerta configs
  268. ALERTA_WEBHOOK_URL=
  269. ALERTA_API_KEY=
  270. # flock configs
  271. FLOCK_WEBHOOK_URL=
  272. # discord configs
  273. DISCORD_WEBHOOK_URL=
  274. # pushover configs
  275. PUSHOVER_APP_TOKEN=
  276. # pushbullet configs
  277. PUSHBULLET_ACCESS_TOKEN=
  278. PUSHBULLET_SOURCE_DEVICE=
  279. # twilio configs
  280. TWILIO_ACCOUNT_SID=
  281. TWILIO_ACCOUNT_TOKEN=
  282. TWILIO_NUMBER=
  283. # hipchat configs
  284. HIPCHAT_SERVER=
  285. HIPCHAT_AUTH_TOKEN=
  286. # messagebird configs
  287. MESSAGEBIRD_ACCESS_KEY=
  288. MESSAGEBIRD_NUMBER=
  289. # kavenegar configs
  290. KAVENEGAR_API_KEY=
  291. KAVENEGAR_SENDER=
  292. # telegram configs
  293. TELEGRAM_BOT_TOKEN=
  294. # kafka configs
  295. SEND_KAFKA="YES"
  296. KAFKA_URL=
  297. KAFKA_SENDER_IP=
  298. # pagerduty.com configs
  299. PD_SERVICE_KEY=
  300. USE_PD_VERSION=
  301. # fleep.io configs
  302. FLEEP_SENDER="${host}"
  303. # Amazon SNS configs
  304. AWSSNS_MESSAGE_FORMAT=
  305. # Matrix configs
  306. MATRIX_HOMESERVER=
  307. MATRIX_ACCESSTOKEN=
  308. # syslog configs
  309. SYSLOG_FACILITY=
  310. # email configs
  311. EMAIL_SENDER=
  312. EMAIL_CHARSET=$(locale charmap 2>/dev/null)
  313. EMAIL_THREADING=
  314. EMAIL_PLAINTEXT_ONLY=
  315. # irc configs
  316. IRC_NICKNAME=
  317. IRC_REALNAME=
  318. IRC_NETWORK=
  319. IRC_PORT=6667
  320. # hangouts configs
  321. declare -A HANGOUTS_WEBHOOK_URI
  322. declare -A HANGOUTS_WEBHOOK_THREAD
  323. # dynatrace configs
  324. DYNATRACE_SPACE=
  325. DYNATRACE_SERVER=
  326. DYNATRACE_TOKEN=
  327. DYNATRACE_TAG_VALUE=
  328. DYNATRACE_ANNOTATION_TYPE=
  329. DYNATRACE_EVENT=
  330. SEND_DYNATRACE=
  331. # stackpulse configs
  332. STACKPULSE_WEBHOOK=
  333. # gotify configs
  334. GOTIFY_APP_URL=
  335. GOTIFY_APP_TOKEN=
  336. # opsgenie configs
  337. OPSGENIE_API_KEY=
  338. # load the stock and user configuration files
  339. # these will overwrite the variables above
  340. if [ ${unittest} ]; then
  341. if source "${cfgfile}"; then
  342. error "Failed to load requested config file."
  343. exit 1
  344. fi
  345. else
  346. for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
  347. if [ -f "${CONFIG}" ]; then
  348. debug "Loading config file '${CONFIG}'..."
  349. source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
  350. else
  351. warning "Cannot find file '${CONFIG}'."
  352. fi
  353. done
  354. fi
  355. if [[ ! $curl_options =~ .*\--connect-timeout ]]; then
  356. curl_options+=" --connect-timeout 5"
  357. fi
  358. OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"}
  359. # If we didn't autodetect the character set for e-mail and it wasn't
  360. # set by the user, we need to set it to a reasonable default. UTF-8
  361. # should be correct for almost all modern UNIX systems.
  362. if [ -z ${EMAIL_CHARSET} ]; then
  363. EMAIL_CHARSET="UTF-8"
  364. fi
  365. # If we've been asked to use FQDN's for the URL's in the alarm, do so,
  366. # unless we're sending an alarm for a child system which we can't get the
  367. # FQDN of easily.
  368. if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
  369. host="$(hostname -f 2>/dev/null)"
  370. fi
  371. # -----------------------------------------------------------------------------
  372. # migrate old Microsoft Teams configuration keys after loading configuration
  373. msteams_migration() {
  374. SEND_MSTEAMS=${SEND_MSTEAM:-$SEND_MSTEAMS}
  375. unset -v SEND_MSTEAM
  376. DEFAULT_RECIPIENT_MSTEAMS=${DEFAULT_RECIPIENT_MSTEAM:-$DEFAULT_RECIPIENT_MSTEAMS}
  377. MSTEAMS_WEBHOOK_URL=${MSTEAM_WEBHOOK_URL:-$MSTEAMS_WEBHOOK_URL}
  378. MSTEAMS_ICON_DEFAULT=${MSTEAM_ICON_DEFAULT:-$MSTEAMS_ICON_DEFAULT}
  379. MSTEAMS_ICON_CLEAR=${MSTEAM_ICON_CLEAR:-$MSTEAMS_ICON_CLEAR}
  380. MSTEAMS_ICON_WARNING=${MSTEAM_ICON_WARNING:-$MSTEAMS_ICON_WARNING}
  381. MSTEAMS_ICON_CRITICAL=${MSTEAM_ICON_CRITICAL:-$MSTEAMS_ICON_CRITICAL}
  382. MSTEAMS_COLOR_DEFAULT=${MSTEAM_COLOR_DEFAULT:-$MSTEAMS_COLOR_DEFAULT}
  383. MSTEAMS_COLOR_CLEAR=${MSTEAM_COLOR_CLEAR:-$MSTEAMS_COLOR_CLEAR}
  384. MSTEAMS_COLOR_WARNING=${MSTEAM_COLOR_WARNING:-$MSTEAMS_COLOR_WARNING}
  385. MSTEAMS_COLOR_CRITICAL=${MSTEAM_COLOR_CRITICAL:-$MSTEAMS_COLOR_CRITICAL}
  386. # migrate role specific recipients:
  387. for key in "${!role_recipients_msteam[@]}"; do
  388. # Disable check, if role_recipients_msteams is ever used:
  389. # The role_recipients_$method are created and used programmatically
  390. # by iterating over $methods. shellcheck therefore doesn't realize
  391. # that role_recipients_msteams is actually used in the block
  392. # "find the recipients' addresses per method".
  393. # shellcheck disable=SC2034
  394. role_recipients_msteams["$key"]="${role_recipients_msteam["$key"]}"
  395. done
  396. }
  397. msteams_migration
  398. # -----------------------------------------------------------------------------
  399. # filter a recipient based on alarm event severity
  400. filter_recipient_by_criticality() {
  401. local method="${1}" x="${2}" r s
  402. shift
  403. r="${x/|*/}" # the recipient
  404. s="${x/*|/}" # the severity required for notifying this recipient
  405. # no severity filtering for this person
  406. [ "${r}" = "${s}" ] && return 0
  407. # the severity is invalid
  408. s="${s^^}"
  409. if [ "${s}" != "CRITICAL" ]; then
  410. error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
  411. return 0
  412. fi
  413. # create the status tracking directory for this user
  414. [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
  415. mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
  416. case "${status}" in
  417. CRITICAL)
  418. # make sure he will get future notifications for this alarm too
  419. touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  420. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
  421. return 0
  422. ;;
  423. WARNING)
  424. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  425. # we do not remove the file, so that he will get future notifications of this alarm
  426. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
  427. return 0
  428. fi
  429. ;;
  430. *)
  431. if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
  432. # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
  433. rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
  434. debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
  435. return 0
  436. fi
  437. ;;
  438. esac
  439. debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
  440. return 1
  441. }
  442. # -----------------------------------------------------------------------------
  443. # verify the delivery methods supported
  444. # check slack
  445. [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
  446. # check rocketchat
  447. [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
  448. # check alerta
  449. [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
  450. # check flock
  451. [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
  452. # check discord
  453. [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
  454. # check pushover
  455. [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
  456. # check pushbullet
  457. [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
  458. # check twilio
  459. { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
  460. # check hipchat
  461. [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
  462. # check messagebird
  463. { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
  464. # check kavenegar
  465. { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
  466. # check telegram
  467. [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
  468. # check kafka
  469. { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
  470. # check irc
  471. [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
  472. # check hangouts
  473. [ ${#HANGOUTS_WEBHOOK_URI[@]} -eq 0 ] && SEND_HANGOUTS="NO"
  474. # check fleep
  475. #shellcheck disable=SC2153
  476. { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
  477. # check dynatrace
  478. { [ -z "${DYNATRACE_SPACE}" ] ||
  479. [ -z "${DYNATRACE_SERVER}" ] ||
  480. [ -z "${DYNATRACE_TOKEN}" ] ||
  481. [ -z "${DYNATRACE_TAG_VALUE}" ] ||
  482. [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
  483. # check opsgenie
  484. [ -z "${OPSGENIE_API_KEY}" ] && SEND_OPSGENIE="NO"
  485. # check matrix
  486. { [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
  487. # check gotify
  488. { [ -z "${GOTIFY_APP_TOKEN}" ] || [ -z "${GOTIFY_APP_URL}" ]; } && SEND_GOTIFY="NO"
  489. # check stackpulse
  490. [ -z "${STACKPULSE_WEBHOOK}" ] && SEND_STACKPULSE="NO"
  491. # check msteams
  492. [ -z "${MSTEAMS_WEBHOOK_URL}" ] && SEND_MSTEAMS="NO"
  493. # check pd
  494. [ -z "${DEFAULT_RECIPIENT_PD}" ] && SEND_PD="NO"
  495. # check prowl
  496. [ -z "${DEFAULT_RECIPIENT_PROWL}" ] && SEND_PROWL="NO"
  497. # check custom
  498. [ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"
  499. if [ "${SEND_PUSHOVER}" = "YES" ] ||
  500. [ "${SEND_SLACK}" = "YES" ] ||
  501. [ "${SEND_ROCKETCHAT}" = "YES" ] ||
  502. [ "${SEND_ALERTA}" = "YES" ] ||
  503. [ "${SEND_PD}" = "YES" ] ||
  504. [ "${SEND_FLOCK}" = "YES" ] ||
  505. [ "${SEND_DISCORD}" = "YES" ] ||
  506. [ "${SEND_HIPCHAT}" = "YES" ] ||
  507. [ "${SEND_TWILIO}" = "YES" ] ||
  508. [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
  509. [ "${SEND_KAVENEGAR}" = "YES" ] ||
  510. [ "${SEND_TELEGRAM}" = "YES" ] ||
  511. [ "${SEND_PUSHBULLET}" = "YES" ] ||
  512. [ "${SEND_KAFKA}" = "YES" ] ||
  513. [ "${SEND_FLEEP}" = "YES" ] ||
  514. [ "${SEND_PROWL}" = "YES" ] ||
  515. [ "${SEND_HANGOUTS}" = "YES" ] ||
  516. [ "${SEND_MATRIX}" = "YES" ] ||
  517. [ "${SEND_CUSTOM}" = "YES" ] ||
  518. [ "${SEND_MSTEAMS}" = "YES" ] ||
  519. [ "${SEND_DYNATRACE}" = "YES" ] ||
  520. [ "${SEND_STACKPULSE}" = "YES" ] ||
  521. [ "${SEND_OPSGENIE}" = "YES" ] ||
  522. [ "${SEND_GOTIFY}" = "YES" ]; then
  523. # if we need curl, check for the curl command
  524. if [ -z "${curl}" ]; then
  525. curl="$(command -v curl 2>/dev/null)"
  526. fi
  527. if [ -z "${curl}" ]; then
  528. error "Cannot find curl command in the system path. Disabling all curl based notifications."
  529. SEND_PUSHOVER="NO"
  530. SEND_PUSHBULLET="NO"
  531. SEND_TELEGRAM="NO"
  532. SEND_SLACK="NO"
  533. SEND_MSTEAMS="NO"
  534. SEND_ROCKETCHAT="NO"
  535. SEND_ALERTA="NO"
  536. SEND_PD="NO"
  537. SEND_FLOCK="NO"
  538. SEND_DISCORD="NO"
  539. SEND_TWILIO="NO"
  540. SEND_HIPCHAT="NO"
  541. SEND_MESSAGEBIRD="NO"
  542. SEND_KAVENEGAR="NO"
  543. SEND_KAFKA="NO"
  544. SEND_FLEEP="NO"
  545. SEND_PROWL="NO"
  546. SEND_HANGOUTS="NO"
  547. SEND_MATRIX="NO"
  548. SEND_CUSTOM="NO"
  549. SEND_DYNATRACE="NO"
  550. SEND_STACKPULSE="NO"
  551. SEND_OPSGENIE="NO"
  552. SEND_GOTIFY="NO"
  553. fi
  554. fi
  555. if [ "${SEND_SMS}" = "YES" ]; then
  556. if [ -z "${sendsms}" ]; then
  557. sendsms="$(command -v sendsms 2>/dev/null)"
  558. fi
  559. if [ -z "${sendsms}" ]; then
  560. SEND_SMS="NO"
  561. fi
  562. fi
  563. # if we need sendmail, check for the sendmail command
  564. if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
  565. sendmail="$(command -v sendmail 2>/dev/null)"
  566. if [ -z "${sendmail}" ]; then
  567. debug "Cannot find sendmail command in the system path. Disabling email notifications."
  568. SEND_EMAIL="NO"
  569. fi
  570. fi
  571. # if we need logger, check for the logger command
  572. if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
  573. logger="$(command -v logger 2>/dev/null)"
  574. if [ -z "${logger}" ]; then
  575. debug "Cannot find logger command in the system path. Disabling syslog notifications."
  576. SEND_SYSLOG="NO"
  577. fi
  578. fi
  579. # if we need aws, check for the aws command
  580. if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
  581. aws="$(command -v aws 2>/dev/null)"
  582. if [ -z "${aws}" ]; then
  583. debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
  584. SEND_AWSSNS="NO"
  585. fi
  586. fi
  587. if [ ${dump_methods} ]; then
  588. for name in "${!SEND_@}"; do
  589. if [ "${!name}" = "YES" ]; then
  590. echo "$name"
  591. fi
  592. done
  593. exit
  594. fi
  595. # -----------------------------------------------------------------------------
  596. # find the recipients' addresses per method
  597. # netdata may call us with multiple roles, and roles may have multiple but
  598. # overlapping recipients - so, here we find the unique recipients.
  599. for method_name in ${method_names}; do
  600. send_var="SEND_${method_name^^}"
  601. if [ "${!send_var}" = "NO" ]; then
  602. continue
  603. fi
  604. declare -A arr_var=()
  605. for x in ${roles//,/ }; do
  606. # the roles 'silent' and 'disabled' mean:
  607. # don't send a notification for this role
  608. if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
  609. continue
  610. fi
  611. role_recipients="role_recipients_${method_name}[$x]"
  612. default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
  613. a="${!role_recipients}"
  614. [ -z "${a}" ] && a="${!default_recipient_var}"
  615. for r in ${a//,/ }; do
  616. [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
  617. done
  618. done
  619. # build the list of recipients
  620. to_var="to_${method_name}"
  621. declare to_${method_name}="${!arr_var[*]}"
  622. [ -z "${!to_var}" ] && declare ${send_var}="NO"
  623. done
  624. # -----------------------------------------------------------------------------
  625. # handle fixup of the email recipient list.
  626. fix_to_email() {
  627. to_email=
  628. while [ -n "${1}" ]; do
  629. [ -n "${to_email}" ] && to_email="${to_email}, "
  630. to_email="${to_email}${1}"
  631. shift 1
  632. done
  633. }
  634. # ${to_email} without quotes here
  635. fix_to_email ${to_email}
  636. # -----------------------------------------------------------------------------
  637. # handle output if we're running in unit test mode
  638. if [ ${unittest} ]; then
  639. for method_name in ${method_names}; do
  640. to_var="to_${method_name}"
  641. echo "results: ${method_name}: ${!to_var}"
  642. done
  643. exit 0
  644. fi
  645. # -----------------------------------------------------------------------------
  646. # check that we have at least a method enabled
  647. proceed=0
  648. for method in "${SEND_EMAIL}" \
  649. "${SEND_PUSHOVER}" \
  650. "${SEND_TELEGRAM}" \
  651. "${SEND_SLACK}" \
  652. "${SEND_ROCKETCHAT}" \
  653. "${SEND_ALERTA}" \
  654. "${SEND_FLOCK}" \
  655. "${SEND_DISCORD}" \
  656. "${SEND_TWILIO}" \
  657. "${SEND_HIPCHAT}" \
  658. "${SEND_MESSAGEBIRD}" \
  659. "${SEND_KAVENEGAR}" \
  660. "${SEND_PUSHBULLET}" \
  661. "${SEND_KAFKA}" \
  662. "${SEND_PD}" \
  663. "${SEND_FLEEP}" \
  664. "${SEND_PROWL}" \
  665. "${SEND_MATRIX}" \
  666. "${SEND_CUSTOM}" \
  667. "${SEND_IRC}" \
  668. "${SEND_HANGOUTS}" \
  669. "${SEND_AWSSNS}" \
  670. "${SEND_SYSLOG}" \
  671. "${SEND_SMS}" \
  672. "${SEND_MSTEAMS}" \
  673. "${SEND_DYNATRACE}" \
  674. "${SEND_STACKPULSE}" \
  675. "${SEND_OPSGENIE}" \
  676. "${SEND_GOTIFY}" ; do
  677. if [ "${method}" == "YES" ]; then
  678. proceed=1
  679. break
  680. fi
  681. done
  682. if [ "$proceed" -eq 0 ]; then
  683. fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
  684. fi
  685. # -----------------------------------------------------------------------------
  686. # get the date the alarm happened
  687. date=$(date --date=@${when} "${date_format}" 2>/dev/null)
  688. [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
  689. [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
  690. [ -z "${date}" ] && date=$(date 2>/dev/null)
  691. # -----------------------------------------------------------------------------
  692. # get the date in utc the alarm happened
  693. date_utc=$(date --date=@${when} "${date_format}" -u 2>/dev/null)
  694. [ -z "${date_utc}" ] && date_utc=$(date -u "${date_format}" 2>/dev/null)
  695. [ -z "${date_utc}" ] && date_utc=$(date -u --date=@${when} 2>/dev/null)
  696. [ -z "${date_utc}" ] && date_utc=$(date -u 2>/dev/null)
  697. # ----------------------------------------------------------------------------
  698. # prepare some extra headers if we've been asked to thread e-mails
  699. if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
  700. email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
  701. else
  702. email_thread_headers=
  703. fi
  704. # -----------------------------------------------------------------------------
  705. # function to URL encode a string
  706. urlencode() {
  707. local string="${1}" strlen encoded pos c o
  708. strlen=${#string}
  709. for ((pos = 0; pos < strlen; pos++)); do
  710. c=${string:pos:1}
  711. case "${c}" in
  712. [-_.~a-zA-Z0-9])
  713. o="${c}"
  714. ;;
  715. *)
  716. printf -v o '%%%02x' "'${c}"
  717. ;;
  718. esac
  719. encoded+="${o}"
  720. done
  721. REPLY="${encoded}"
  722. echo "${REPLY}"
  723. }
  724. # -----------------------------------------------------------------------------
  725. # function to convert a duration in seconds, to a human readable duration
  726. # using DAYS, MINUTES, SECONDS
  727. duration4human() {
  728. local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
  729. d=$((s / 86400))
  730. s=$((s - (d * 86400)))
  731. h=$((s / 3600))
  732. s=$((s - (h * 3600)))
  733. m=$((s / 60))
  734. s=$((s - (m * 60)))
  735. if [ ${d} -gt 0 ]; then
  736. [ ${m} -ge 30 ] && h=$((h + 1))
  737. [ ${d} -gt 1 ] && ds="days"
  738. [ ${h} -gt 1 ] && hs="hours"
  739. if [ ${h} -gt 0 ]; then
  740. ret="${d} ${ds} and ${h} ${hs}"
  741. else
  742. ret="${d} ${ds}"
  743. fi
  744. elif [ ${h} -gt 0 ]; then
  745. [ ${s} -ge 30 ] && m=$((m + 1))
  746. [ ${h} -gt 1 ] && hs="hours"
  747. [ ${m} -gt 1 ] && ms="minutes"
  748. if [ ${m} -gt 0 ]; then
  749. ret="${h} ${hs} and ${m} ${ms}"
  750. else
  751. ret="${h} ${hs}"
  752. fi
  753. elif [ ${m} -gt 0 ]; then
  754. [ ${m} -gt 1 ] && ms="minutes"
  755. [ ${s} -gt 1 ] && ss="seconds"
  756. if [ ${s} -gt 0 ]; then
  757. ret="${m} ${ms} and ${s} ${ss}"
  758. else
  759. ret="${m} ${ms}"
  760. fi
  761. else
  762. [ ${s} -gt 1 ] && ss="seconds"
  763. ret="${s} ${ss}"
  764. fi
  765. REPLY="${ret}"
  766. echo "${REPLY}"
  767. }
  768. # -----------------------------------------------------------------------------
  769. # email sender
  770. send_email() {
  771. local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
  772. if [ "${SEND_EMAIL}" = "YES" ]; then
  773. if [ -n "${EMAIL_SENDER}" ]; then
  774. if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
  775. # the name includes double quotes
  776. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  777. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
  778. elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
  779. # the name includes single quotes
  780. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  781. sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
  782. elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
  783. # the name does not have any quotes
  784. sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
  785. sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
  786. fi
  787. fi
  788. [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
  789. [ -n "${sender_name}" ] && ${sendmail} -F 2>&1 | head -1 | grep -qv "sendmail: unrecognized option: F" && opts+=(-F "${sender_name}")
  790. if [ "${debug}" = "1" ]; then
  791. echo >&2 "--- BEGIN sendmail command ---"
  792. printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
  793. echo >&2
  794. echo >&2 "--- END sendmail command ---"
  795. fi
  796. local cmd_output
  797. cmd_output=$("${sendmail}" -t "${opts[@]}" 2>&1)
  798. ret=$?
  799. if [ ${ret} -eq 0 ]; then
  800. info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
  801. return 0
  802. else
  803. error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret} (${cmd_output})."
  804. return 1
  805. fi
  806. fi
  807. return 1
  808. }
  809. # -----------------------------------------------------------------------------
  810. # pushover sender
  811. send_pushover() {
  812. local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
  813. if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
  814. # https://pushover.net/api
  815. priority=-2
  816. case "${status}" in
  817. CLEAR) priority=-1 ;; # low priority: no sound or vibration
  818. WARNING) priority=0 ;; # normal priority: respect quiet hours
  819. CRITICAL) priority=1 ;; # high priority: bypass quiet hours
  820. *) priority=-2 ;; # lowest priority: no notification at all
  821. esac
  822. for user in ${usertokens}; do
  823. httpcode=$(docurl \
  824. --form-string "token=${apptoken}" \
  825. --form-string "user=${user}" \
  826. --form-string "html=1" \
  827. --form-string "title=${title}" \
  828. --form-string "message=${message}" \
  829. --form-string "timestamp=${when}" \
  830. --form-string "url=${url}" \
  831. --form-string "url_title=Open netdata dashboard to view the alarm" \
  832. --form-string "priority=${priority}" \
  833. https://api.pushover.net/1/messages.json)
  834. if [ "${httpcode}" = "200" ]; then
  835. info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
  836. sent=$((sent + 1))
  837. else
  838. error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  839. fi
  840. done
  841. [ ${sent} -gt 0 ] && return 0
  842. fi
  843. return 1
  844. }
  845. # -----------------------------------------------------------------------------
  846. # pushbullet sender
  847. send_pushbullet() {
  848. local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 userOrChannelTag
  849. if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  850. # https://docs.pushbullet.com/#create-push
  851. # Accept specification of user(s) (PushBullet account email address) and/or channel tag(s), separated by spaces.
  852. # If recipient begins with a "#" then send to channel tag, otherwise send to email recipient.
  853. for userOrChannelTag in ${recipients}; do
  854. if [ "${userOrChannelTag::1}" = "#" ]; then
  855. userOrChannelTag_type="channel_tag"
  856. userOrChannelTag="${userOrChannelTag:1}" # Remove hash from start of channel tag (required by pushbullet API)
  857. else
  858. userOrChannelTag_type="email"
  859. fi
  860. httpcode=$(docurl \
  861. --header 'Access-Token: '${userapikey}'' \
  862. --header 'Content-Type: application/json' \
  863. --data-binary @<(
  864. cat <<EOF
  865. {"title": "${title}",
  866. "type": "link",
  867. "${userOrChannelTag_type}": "${userOrChannelTag}",
  868. "body": "$(echo -n ${message})",
  869. "url": "${url}",
  870. "source_device_iden": "${source_device}"}
  871. EOF
  872. ) "https://api.pushbullet.com/v2/pushes" -X POST)
  873. if [ "${httpcode}" = "200" ]; then
  874. info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${userOrChannelTag}'"
  875. sent=$((sent + 1))
  876. else
  877. error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${userOrChannelTag}' with HTTP response status code ${httpcode}."
  878. fi
  879. done
  880. [ ${sent} -gt 0 ] && return 0
  881. fi
  882. return 1
  883. }
  884. # -----------------------------------------------------------------------------
  885. # kafka sender
  886. send_kafka() {
  887. local httpcode sent=0
  888. if [ "${SEND_KAFKA}" = "YES" ]; then
  889. httpcode=$(docurl -X POST \
  890. --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
  891. "${KAFKA_URL}")
  892. if [ "${httpcode}" = "204" ]; then
  893. info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
  894. sent=$((sent + 1))
  895. else
  896. error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP response status code ${httpcode}."
  897. fi
  898. [ ${sent} -gt 0 ] && return 0
  899. fi
  900. return 1
  901. }
  902. # -----------------------------------------------------------------------------
  903. # pagerduty.com sender
  904. send_pd() {
  905. local recipients="${1}" sent=0 severity current_time payload url response_code
  906. unset t
  907. case ${status} in
  908. CLEAR) t='resolve' ; severity='info' ;;
  909. WARNING) t='trigger' ; severity='warning' ;;
  910. CRITICAL) t='trigger' ; severity='critical' ;;
  911. esac
  912. if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
  913. if [ "$(uname)" == "Linux" ]; then
  914. current_time=$(date -d @${when} +'%Y-%m-%dT%H:%M:%S.000')
  915. else
  916. current_time=$(date -r ${when} +'%Y-%m-%dT%H:%M:%S.000')
  917. fi
  918. for PD_SERVICE_KEY in ${recipients}; do
  919. d="${status} ${name} = ${value_string} - ${host}, ${family}"
  920. if [ ${USE_PD_VERSION} = "2" ]; then
  921. payload="$(
  922. cat <<EOF
  923. {
  924. "payload" : {
  925. "summary": "${info:0:1024}",
  926. "source" : "${args_host}",
  927. "severity" : "${severity}",
  928. "timestamp" : "${current_time}",
  929. "group" : "${family}",
  930. "class" : "${chart}",
  931. "custom_details": {
  932. "value_w_units": "${value_string}",
  933. "when": "${when}",
  934. "duration" : "${duration}",
  935. "roles": "${roles}",
  936. "alarm_id" : "${alarm_id}",
  937. "name" : "${name}",
  938. "chart" : "${chart}",
  939. "family" : "${family}",
  940. "status" : "${status}",
  941. "old_status" : "${old_status}",
  942. "value" : "${value}",
  943. "old_value" : "${old_value}",
  944. "src" : "${src}",
  945. "non_clear_duration" : "${non_clear_duration}",
  946. "units" : "${units}",
  947. "info" : "${info}"
  948. }
  949. },
  950. "routing_key": "${PD_SERVICE_KEY}",
  951. "event_action": "${t}",
  952. "dedup_key": "${unique_id}"
  953. }
  954. EOF
  955. )"
  956. url="https://events.pagerduty.com/v2/enqueue"
  957. response_code="202"
  958. else
  959. payload="$(
  960. cat <<EOF
  961. {
  962. "service_key": "${PD_SERVICE_KEY}",
  963. "event_type": "${t}",
  964. "incident_key" : "${alarm_id}",
  965. "description": "${d}",
  966. "details": {
  967. "value_w_units": "${value_string}",
  968. "when": "${when}",
  969. "duration" : "${duration}",
  970. "roles": "${roles}",
  971. "alarm_id" : "${alarm_id}",
  972. "name" : "${name}",
  973. "chart" : "${chart}",
  974. "family" : "${family}",
  975. "status" : "${status}",
  976. "old_status" : "${old_status}",
  977. "value" : "${value}",
  978. "old_value" : "${old_value}",
  979. "src" : "${src}",
  980. "non_clear_duration" : "${non_clear_duration}",
  981. "units" : "${units}",
  982. "info" : "${info}"
  983. }
  984. }
  985. EOF
  986. )"
  987. url="https://events.pagerduty.com/generic/2010-04-15/create_event.json"
  988. response_code="200"
  989. fi
  990. httpcode=$(docurl -X POST --data "${payload}" ${url})
  991. if [ "${httpcode}" = "${response_code}" ]; then
  992. info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
  993. sent=$((sent + 1))
  994. else
  995. error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
  996. fi
  997. done
  998. [ ${sent} -gt 0 ] && return 0
  999. fi
  1000. return 1
  1001. }
  1002. # -----------------------------------------------------------------------------
  1003. # twilio sender
  1004. send_twilio() {
  1005. local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
  1006. if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1007. #https://www.twilio.com/packages/labs/code/bash/twilio-sms
  1008. for user in ${recipients}; do
  1009. httpcode=$(docurl -X POST \
  1010. --data-urlencode "From=${twilionumber}" \
  1011. --data-urlencode "To=${user}" \
  1012. --data-urlencode "Body=${title} ${message}" \
  1013. -u "${accountsid}:${accounttoken}" \
  1014. "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
  1015. if [ "${httpcode}" = "201" ]; then
  1016. info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1017. sent=$((sent + 1))
  1018. else
  1019. error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1020. fi
  1021. done
  1022. [ ${sent} -gt 0 ] && return 0
  1023. fi
  1024. return 1
  1025. }
  1026. # -----------------------------------------------------------------------------
  1027. # hipchat sender
  1028. send_hipchat() {
  1029. local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
  1030. # remove <small></small> from the message
  1031. message="${message//<small>/}"
  1032. message="${message//<\/small>/}"
  1033. if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
  1034. # Valid values: html, text.
  1035. # Defaults to 'html'.
  1036. msg_format="html"
  1037. # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
  1038. case "${status}" in
  1039. WARNING) color="yellow" ;;
  1040. CRITICAL) color="red" ;;
  1041. CLEAR) color="green" ;;
  1042. *) color="gray" ;;
  1043. esac
  1044. # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
  1045. # Each recipient's notification preferences are taken into account.
  1046. # Defaults to false.
  1047. notify="true"
  1048. for room in ${recipients}; do
  1049. httpcode=$(docurl -X POST \
  1050. -H "Content-type: application/json" \
  1051. -H "Authorization: Bearer ${authtoken}" \
  1052. -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
  1053. "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
  1054. if [ "${httpcode}" = "204" ]; then
  1055. info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1056. sent=$((sent + 1))
  1057. else
  1058. error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP response status code ${httpcode}."
  1059. fi
  1060. done
  1061. [ ${sent} -gt 0 ] && return 0
  1062. fi
  1063. return 1
  1064. }
  1065. # -----------------------------------------------------------------------------
  1066. # messagebird sender
  1067. send_messagebird() {
  1068. local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  1069. if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1070. #https://developers.messagebird.com/docs/messaging
  1071. for user in ${recipients}; do
  1072. httpcode=$(docurl -X POST \
  1073. --data-urlencode "originator=${messagebirdnumber}" \
  1074. --data-urlencode "recipients=${user}" \
  1075. --data-urlencode "body=${title} ${message}" \
  1076. --data-urlencode "datacoding=auto" \
  1077. -H "Authorization: AccessKey ${accesskey}" \
  1078. "https://rest.messagebird.com/messages")
  1079. if [ "${httpcode}" = "201" ]; then
  1080. info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1081. sent=$((sent + 1))
  1082. else
  1083. error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1084. fi
  1085. done
  1086. [ ${sent} -gt 0 ] && return 0
  1087. fi
  1088. return 1
  1089. }
  1090. # -----------------------------------------------------------------------------
  1091. # kavenegar sender
  1092. send_kavenegar() {
  1093. local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
  1094. if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
  1095. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1096. for user in ${recipients}; do
  1097. httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
  1098. --data-urlencode "sender=${kavenegarsender}" \
  1099. --data-urlencode "receptor=${user}" \
  1100. --data-urlencode "message=${title} ${message}")
  1101. if [ "${httpcode}" = "200" ]; then
  1102. info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1103. sent=$((sent + 1))
  1104. else
  1105. error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
  1106. fi
  1107. done
  1108. [ ${sent} -gt 0 ] && return 0
  1109. fi
  1110. return 1
  1111. }
  1112. # -----------------------------------------------------------------------------
  1113. # telegram sender
  1114. send_telegram() {
  1115. local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
  1116. if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
  1117. case "${status}" in
  1118. WARNING) emoji="⚠️" ;;
  1119. CRITICAL) emoji="🔴" ;;
  1120. CLEAR) emoji="✅" ;;
  1121. *) emoji="⚪️" ;;
  1122. esac
  1123. if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
  1124. for chatid in ${chatids}; do
  1125. notify_telegram=1
  1126. notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0}
  1127. while [ ${notify_telegram} -eq 1 ]; do
  1128. # https://core.telegram.org/bots/api#sendmessage
  1129. httpcode=$(docurl ${disableNotification} \
  1130. --data-urlencode "parse_mode=HTML" \
  1131. --data-urlencode "disable_web_page_preview=true" \
  1132. --data-urlencode "text=${emoji} ${message}" \
  1133. "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
  1134. notify_telegram=0
  1135. if [ "${httpcode}" = "200" ]; then
  1136. info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
  1137. sent=$((sent + 1))
  1138. elif [ "${httpcode}" = "401" ]; then
  1139. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
  1140. elif [ "${httpcode}" = "429" ]; then
  1141. if [ "$notify_retries" -gt 0 ]; then
  1142. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s."
  1143. notify_retries=$((notify_retries - 1))
  1144. notify_telegram=1
  1145. sleep 1
  1146. else
  1147. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded."
  1148. fi
  1149. else
  1150. error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
  1151. fi
  1152. done
  1153. done
  1154. [ ${sent} -gt 0 ] && return 0
  1155. fi
  1156. return 1
  1157. }
  1158. # -----------------------------------------------------------------------------
  1159. # Microsoft Team sender
  1160. send_msteams() {
  1161. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1162. [ "${SEND_MSTEAMS}" != "YES" ] && return 1
  1163. case "${status}" in
  1164. WARNING) icon="${MSTEAMS_ICON_WARNING}" && color="${MSTEAMS_COLOR_WARNING}" ;;
  1165. CRITICAL) icon="${MSTEAMS_ICON_CRITICAL}" && color="${MSTEAMS_COLOR_CRITICAL}" ;;
  1166. CLEAR) icon="${MSTEAMS_ICON_CLEAR}" && color="${MSTEAMS_COLOR_CLEAR}" ;;
  1167. *) icon="${MSTEAMS_ICON_DEFAULT}" && color="${MSTEAMS_COLOR_DEFAULT}" ;;
  1168. esac
  1169. for channel in ${channels}; do
  1170. ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
  1171. ## Online designer : https://adaptivecards.io/designer/
  1172. payload="$(
  1173. cat <<EOF
  1174. {
  1175. "@context": "http://schema.org/extensions",
  1176. "@type": "MessageCard",
  1177. "themeColor": "${color}",
  1178. "title": "$icon Alert ${status} from netdata for ${host}",
  1179. "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
  1180. "potentialAction": [
  1181. {
  1182. "@type": "OpenUri",
  1183. "name": "Netdata",
  1184. "targets": [
  1185. { "os": "default", "uri": "${goto_url}" }
  1186. ]
  1187. }
  1188. ]
  1189. }
  1190. EOF
  1191. )"
  1192. # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
  1193. cur_webhook="${webhook//CHANNEL/${channel}}"
  1194. httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${cur_webhook}")
  1195. if [ "${httpcode}" = "200" ]; then
  1196. info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${cur_webhook}'"
  1197. sent=$((sent + 1))
  1198. else
  1199. error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${cur_webhook}', with HTTP response status code ${httpcode}."
  1200. fi
  1201. done
  1202. [ ${sent} -gt 0 ] && return 0
  1203. return 1
  1204. }
  1205. # slack sender
  1206. send_slack() {
  1207. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1208. [ "${SEND_SLACK}" != "YES" ] && return 1
  1209. case "${status}" in
  1210. WARNING) color="warning" ;;
  1211. CRITICAL) color="danger" ;;
  1212. CLEAR) color="good" ;;
  1213. *) color="#777777" ;;
  1214. esac
  1215. for channel in ${channels}; do
  1216. # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
  1217. if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
  1218. # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
  1219. if [ "${channel}" = "#" ]; then
  1220. ch=""
  1221. chstr="without specifying a channel"
  1222. else
  1223. ch="\"channel\": \"${channel}\","
  1224. chstr="to '${channel}'"
  1225. fi
  1226. payload="$(
  1227. cat <<EOF
  1228. {
  1229. $ch
  1230. "username": "netdata on ${host}",
  1231. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1232. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1233. "attachments": [
  1234. {
  1235. "fallback": "${alarm} - ${chart} (${family}) - ${info}",
  1236. "color": "${color}",
  1237. "title": "${alarm}",
  1238. "title_link": "${goto_url}",
  1239. "text": "${info}",
  1240. "fields": [
  1241. {
  1242. "title": "${chart}",
  1243. "short": true
  1244. },
  1245. {
  1246. "title": "${family}",
  1247. "short": true
  1248. }
  1249. ],
  1250. "thumb_url": "${image}",
  1251. "footer": "by ${host}",
  1252. "ts": ${when}
  1253. }
  1254. ]
  1255. }
  1256. EOF
  1257. )"
  1258. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1259. if [ "${httpcode}" = "200" ]; then
  1260. info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
  1261. sent=$((sent + 1))
  1262. else
  1263. error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP response status code ${httpcode}."
  1264. fi
  1265. done
  1266. [ ${sent} -gt 0 ] && return 0
  1267. return 1
  1268. }
  1269. # -----------------------------------------------------------------------------
  1270. # rocketchat sender
  1271. send_rocketchat() {
  1272. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1273. [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
  1274. case "${status}" in
  1275. WARNING) color="warning" ;;
  1276. CRITICAL) color="danger" ;;
  1277. CLEAR) color="good" ;;
  1278. *) color="#777777" ;;
  1279. esac
  1280. for channel in ${channels}; do
  1281. payload="$(
  1282. cat <<EOF
  1283. {
  1284. "channel": "#${channel}",
  1285. "alias": "netdata on ${host}",
  1286. "avatar": "${images_base_url}/images/banner-icon-144x144.png",
  1287. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1288. "attachments": [
  1289. {
  1290. "color": "${color}",
  1291. "title": "${alarm}",
  1292. "title_link": "${goto_url}",
  1293. "text": "${info}",
  1294. "fields": [
  1295. {
  1296. "title": "${chart}",
  1297. "short": true,
  1298. "value": "chart"
  1299. },
  1300. {
  1301. "title": "${family}",
  1302. "short": true,
  1303. "value": "family"
  1304. }
  1305. ],
  1306. "thumb_url": "${image}",
  1307. "ts": "${when}"
  1308. }
  1309. ]
  1310. }
  1311. EOF
  1312. )"
  1313. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1314. if [ "${httpcode}" = "200" ]; then
  1315. info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1316. sent=$((sent + 1))
  1317. else
  1318. error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1319. fi
  1320. done
  1321. [ ${sent} -gt 0 ] && return 0
  1322. return 1
  1323. }
  1324. # -----------------------------------------------------------------------------
  1325. # alerta sender
  1326. send_alerta() {
  1327. local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
  1328. [ "${SEND_ALERTA}" != "YES" ] && return 1
  1329. case "${status}" in
  1330. CRITICAL) severity="critical" ;;
  1331. WARNING) severity="warning" ;;
  1332. CLEAR) severity="cleared" ;;
  1333. *) severity="indeterminate" ;;
  1334. esac
  1335. if [[ ${chart} == httpcheck* ]]; then
  1336. resource=$chart
  1337. event=$name
  1338. else
  1339. resource="${host}:${family}"
  1340. event="${chart}.${name}"
  1341. fi
  1342. for channel in ${channels}; do
  1343. payload="$(
  1344. cat <<EOF
  1345. {
  1346. "resource": "${resource}",
  1347. "event": "${event}",
  1348. "environment": "${channel}",
  1349. "severity": "${severity}",
  1350. "service": ["Netdata"],
  1351. "group": "Performance",
  1352. "value": "${value_string}",
  1353. "text": "${info}",
  1354. "tags": ["alarm_id:${alarm_id}"],
  1355. "attributes": {
  1356. "roles": "${roles}",
  1357. "name": "${name}",
  1358. "chart": "${chart}",
  1359. "family": "${family}",
  1360. "source": "${src}",
  1361. "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
  1362. },
  1363. "origin": "netdata/${host}",
  1364. "type": "netdataAlarm",
  1365. "rawData": "${BASH_ARGV[@]}"
  1366. }
  1367. EOF
  1368. )"
  1369. if [ -n "${ALERTA_API_KEY}" ]; then
  1370. auth="Key ${ALERTA_API_KEY}"
  1371. fi
  1372. httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
  1373. if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
  1374. info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1375. sent=$((sent + 1))
  1376. elif [ "${httpcode}" = "202" ]; then
  1377. info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1378. else
  1379. error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1380. fi
  1381. done
  1382. [ ${sent} -gt 0 ] && return 0
  1383. return 1
  1384. }
  1385. # -----------------------------------------------------------------------------
  1386. # flock sender
  1387. send_flock() {
  1388. local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
  1389. [ "${SEND_FLOCK}" != "YES" ] && return 1
  1390. case "${status}" in
  1391. WARNING) color="warning" ;;
  1392. CRITICAL) color="danger" ;;
  1393. CLEAR) color="good" ;;
  1394. *) color="#777777" ;;
  1395. esac
  1396. for channel in ${channels}; do
  1397. httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
  1398. \"sendAs\": {
  1399. \"name\" : \"netdata on ${host}\",
  1400. \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
  1401. },
  1402. \"text\": \"${host} *${status_message}*\",
  1403. \"timestamp\": \"${when}\",
  1404. \"attachments\": [
  1405. {
  1406. \"description\": \"${chart} (${family}) - ${info}\",
  1407. \"color\": \"${color}\",
  1408. \"title\": \"${alarm}\",
  1409. \"url\": \"${goto_url}\",
  1410. \"text\": \"${info}\",
  1411. \"views\": {
  1412. \"image\": {
  1413. \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
  1414. \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
  1415. \"filename\": \"${image}\"
  1416. }
  1417. }
  1418. }
  1419. ]
  1420. }")
  1421. if [ "${httpcode}" = "200" ]; then
  1422. info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1423. sent=$((sent + 1))
  1424. else
  1425. error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1426. fi
  1427. done
  1428. [ ${sent} -gt 0 ] && return 0
  1429. return 1
  1430. }
  1431. # -----------------------------------------------------------------------------
  1432. # discord sender
  1433. send_discord() {
  1434. local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
  1435. [ "${SEND_DISCORD}" != "YES" ] && return 1
  1436. case "${status}" in
  1437. WARNING) color="warning" ;;
  1438. CRITICAL) color="danger" ;;
  1439. CLEAR) color="good" ;;
  1440. *) color="#777777" ;;
  1441. esac
  1442. for channel in ${channels}; do
  1443. username="netdata on ${host}"
  1444. [ ${#username} -gt 32 ] && username="${username:0:29}..."
  1445. payload="$(
  1446. cat <<EOF
  1447. {
  1448. "channel": "#${channel}",
  1449. "username": "${username}",
  1450. "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
  1451. "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
  1452. "attachments": [
  1453. {
  1454. "color": "${color}",
  1455. "title": "${alarm}",
  1456. "title_link": "${goto_url}",
  1457. "text": "${info}",
  1458. "fields": [
  1459. {
  1460. "title": "${chart}",
  1461. "value": "${family}"
  1462. }
  1463. ],
  1464. "thumb_url": "${image}",
  1465. "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
  1466. "footer": "${host}",
  1467. "ts": ${when}
  1468. }
  1469. ]
  1470. }
  1471. EOF
  1472. )"
  1473. httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
  1474. if [ "${httpcode}" = "200" ]; then
  1475. info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
  1476. sent=$((sent + 1))
  1477. else
  1478. error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
  1479. fi
  1480. done
  1481. [ ${sent} -gt 0 ] && return 0
  1482. return 1
  1483. }
  1484. # -----------------------------------------------------------------------------
  1485. # fleep sender
  1486. send_fleep() {
  1487. local httpcode sent=0 webhooks="${1}" data message
  1488. if [ "${SEND_FLEEP}" = "YES" ]; then
  1489. message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
  1490. for hook in ${webhooks}; do
  1491. data="{ "
  1492. data="${data} 'message': '${message}', "
  1493. data="${data} 'user': '${FLEEP_SENDER}' "
  1494. data="${data} }"
  1495. httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
  1496. if [ "${httpcode}" = "200" ]; then
  1497. info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
  1498. sent=$((sent + 1))
  1499. else
  1500. error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP response status code ${httpcode}."
  1501. fi
  1502. done
  1503. [ ${sent} -gt 0 ] && return 0
  1504. fi
  1505. return 1
  1506. }
  1507. # -----------------------------------------------------------------------------
  1508. # Prowl sender
  1509. send_prowl() {
  1510. local httpcode sent=0 data message keys prio=0 alarm_url event
  1511. if [ "${SEND_PROWL}" = "YES" ]; then
  1512. message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
  1513. message="description=${message}"
  1514. keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
  1515. keys="apikey=${keys}"
  1516. app="application=Netdata"
  1517. case "${status}" in
  1518. CRITICAL)
  1519. prio=2
  1520. ;;
  1521. WARNING)
  1522. prio=1
  1523. ;;
  1524. esac
  1525. prio="priority=${prio}"
  1526. alarm_url="$(urlencode ${goto_url})"
  1527. alarm_url="url=${alarm_url}"
  1528. event="$(urlencode "${host} ${status_message}")"
  1529. event="event=${event}"
  1530. data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
  1531. httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
  1532. if [ "${httpcode}" = "200" ]; then
  1533. info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
  1534. sent=1
  1535. else
  1536. error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
  1537. fi
  1538. [ ${sent} -gt 0 ] && return 0
  1539. fi
  1540. return 1
  1541. }
  1542. # -----------------------------------------------------------------------------
  1543. # irc sender
  1544. send_irc() {
  1545. local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" PORT="${5}" SERVERNAME="${6}" MESSAGE="${7}" sent=0 channel color send_alarm reply_codes error
  1546. if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ] && [ -n "${PORT}" ]; then
  1547. case "${status}" in
  1548. WARNING) color="warning" ;;
  1549. CRITICAL) color="danger" ;;
  1550. CLEAR) color="good" ;;
  1551. *) color="#777777" ;;
  1552. esac
  1553. SNDMESSAGE="${MESSAGE//$'\n'/", "}"
  1554. for CHANNEL in ${CHANNELS}; do
  1555. error=0
  1556. send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \ | nc "${NETWORK}" "${PORT}")
  1557. reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
  1558. for code in ${reply_codes}; do
  1559. if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
  1560. error=1
  1561. break
  1562. fi
  1563. done
  1564. if [ "${error}" -eq 0 ]; then
  1565. info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
  1566. sent=$((sent + 1))
  1567. else
  1568. error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
  1569. fi
  1570. done
  1571. fi
  1572. [ ${sent} -gt 0 ] && return 0
  1573. return 1
  1574. }
  1575. # -----------------------------------------------------------------------------
  1576. # Amazon SNS sender
  1577. send_awssns() {
  1578. local targets="${1}" message='' sent=0 region=''
  1579. local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
  1580. [ "${SEND_AWSSNS}" = "YES" ] || return 1
  1581. message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
  1582. for target in ${targets}; do
  1583. # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
  1584. region="$(echo ${target} | cut -f 4 -d ':')"
  1585. if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
  1586. info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1587. sent=$((sent + 1))
  1588. else
  1589. error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
  1590. fi
  1591. done
  1592. [ ${sent} -gt 0 ] && return 0
  1593. return 1
  1594. }
  1595. # -----------------------------------------------------------------------------
  1596. # Matrix sender
  1597. send_matrix() {
  1598. local homeserver="${1}" webhook accesstoken rooms="${2}" httpcode sent=0 payload
  1599. [ "${SEND_MATRIX}" != "YES" ] && return 1
  1600. [ -z "${MATRIX_ACCESSTOKEN}" ] && return 1
  1601. accesstoken="${MATRIX_ACCESSTOKEN}"
  1602. case "${status}" in
  1603. WARNING) emoji="⚠️" ;;
  1604. CRITICAL) emoji="🔴" ;;
  1605. CLEAR) emoji="✅" ;;
  1606. *) emoji="⚪️" ;;
  1607. esac
  1608. for room in ${rooms}; do
  1609. webhook="$homeserver/_matrix/client/r0/rooms/$(urlencode $room)/send/m.room.message?access_token=$accesstoken"
  1610. payload="$(
  1611. cat <<EOF
  1612. {
  1613. "msgtype": "m.notice",
  1614. "format": "org.matrix.custom.html",
  1615. "formatted_body": "${emoji} ${host} ${status_message} - <b>${name//_/ }</b><br>${chart} (${family})<br><a href=\"${goto_url}\">${alarm}</a><br><i>${info}</i>",
  1616. "body": "${emoji} ${host} ${status_message} - ${name//_/ } ${chart} (${family}) ${goto_url} ${alarm} ${info}"
  1617. }
  1618. EOF
  1619. )"
  1620. httpcode=$(docurl -X POST --data "${payload}" "${webhook}")
  1621. if [ "${httpcode}" == "200" ]; then
  1622. info "sent Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1623. sent=$((sent + 1))
  1624. else
  1625. error "failed to send Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
  1626. fi
  1627. done
  1628. [ ${sent} -gt 0 ] && return 0
  1629. return 1
  1630. }
  1631. # -----------------------------------------------------------------------------
  1632. # syslog sender
  1633. send_syslog() {
  1634. local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
  1635. local priority='' message='' server='' port='' prefix=''
  1636. local temp1='' temp2=''
  1637. [ "${SEND_SYSLOG}" = "YES" ] || return 1
  1638. if [ "${status}" = "CRITICAL" ]; then
  1639. level='crit'
  1640. elif [ "${status}" = "WARNING" ]; then
  1641. level='warning'
  1642. fi
  1643. for target in ${targets}; do
  1644. priority="${facility}.${level}"
  1645. message=''
  1646. server=''
  1647. port=''
  1648. prefix=''
  1649. temp1=''
  1650. temp2=''
  1651. prefix=$(echo ${target} | cut -d '/' -f 2)
  1652. temp1=$(echo ${target} | cut -d '/' -f 1)
  1653. if [ ${prefix} != ${temp1} ]; then
  1654. if (echo ${temp1} | grep -q '@'); then
  1655. temp2=$(echo ${temp1} | cut -d '@' -f 1)
  1656. server=$(echo ${temp1} | cut -d '@' -f 2)
  1657. if [ ${temp2} != ${server} ]; then
  1658. priority=${temp2}
  1659. fi
  1660. port=$(echo ${server} | rev | cut -d ':' -f 1 | rev)
  1661. if (echo ${server} | grep -E -q '\[.*\]'); then
  1662. if (echo ${port} | grep -q ']'); then
  1663. port=''
  1664. else
  1665. server=$(echo ${server} | rev | cut -d ':' -f 2- | rev)
  1666. fi
  1667. else
  1668. if [ ${port} = ${server} ]; then
  1669. port=''
  1670. else
  1671. server=$(echo ${server} | cut -d ':' -f 1)
  1672. fi
  1673. fi
  1674. else
  1675. priority=${temp1}
  1676. fi
  1677. fi
  1678. message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
  1679. if [ ${server} ]; then
  1680. logger_options="${logger_options} -n ${server}"
  1681. if [ ${port} ]; then
  1682. logger_options="${logger_options} -P ${port}"
  1683. fi
  1684. fi
  1685. ${logger} -p ${priority} ${logger_options} "${message}"
  1686. done
  1687. return $?
  1688. }
  1689. # -----------------------------------------------------------------------------
  1690. # SMS sender
  1691. send_sms() {
  1692. local recipients="${1}" errcode errmessage sent=0
  1693. # Human readable SMS
  1694. local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
  1695. # limit it to 160 characters
  1696. msg="${msg:0:160}"
  1697. if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
  1698. # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
  1699. for phone in ${recipients}; do
  1700. errmessage=$($sendsms $phone "$msg" 2>&1)
  1701. errcode=$?
  1702. if [ ${errcode} -eq 0 ]; then
  1703. info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
  1704. sent=$((sent + 1))
  1705. else
  1706. error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
  1707. fi
  1708. done
  1709. [ ${sent} -gt 0 ] && return 0
  1710. fi
  1711. return 1
  1712. }
  1713. # -----------------------------------------------------------------------------
  1714. # hangouts sender
  1715. send_hangouts() {
  1716. local rooms="${1}" httpcode sent=0 room color payload webhook thread
  1717. [ "${SEND_HANGOUTS}" != "YES" ] && return 1
  1718. case "${status}" in
  1719. WARNING) color="#ffa700" ;;
  1720. CRITICAL) color="#d62d20" ;;
  1721. CLEAR) color="#008744" ;;
  1722. *) color="#777777" ;;
  1723. esac
  1724. for room in ${rooms}; do
  1725. if [ -z "${HANGOUTS_WEBHOOK_URI[$room]}" ] ; then
  1726. info "Can't send Hangouts notification for: ${host} ${chart}.${name} to room ${room}. HANGOUTS_WEBHOOK_URI[$room] not defined"
  1727. else
  1728. if [ -n "${HANGOUTS_WEBHOOK_THREAD[$room]}" ]; then
  1729. thread="\"name\" : \"${HANGOUTS_WEBHOOK_THREAD[$room]}\""
  1730. fi
  1731. webhook="${HANGOUTS_WEBHOOK_URI[$room]}"
  1732. payload="$(
  1733. cat <<EOF
  1734. {
  1735. "cards": [
  1736. {
  1737. "header": {
  1738. "title": "Netdata on ${host}",
  1739. "imageUrl": "${images_base_url}/images/banner-icon-144x144.png",
  1740. "imageStyle": "IMAGE"
  1741. },
  1742. "sections": [
  1743. {
  1744. "header": "<b>${host}</b>",
  1745. "widgets": [
  1746. {
  1747. "keyValue": {
  1748. "topLabel": "Status Message",
  1749. "content": "<b>${status_message}</b>",
  1750. "contentMultiline": "true",
  1751. "iconUrl": "${image}",
  1752. "onClick": {
  1753. "openLink": {
  1754. "url": "${goto_url}"
  1755. }
  1756. }
  1757. }
  1758. },
  1759. {
  1760. "keyValue": {
  1761. "topLabel": "${chart} | ${family}",
  1762. "content": "<font color=${color}>${alarm}</font>",
  1763. "contentMultiline": "true"
  1764. }
  1765. }
  1766. ]
  1767. },
  1768. {
  1769. "widgets": [
  1770. {
  1771. "textParagraph": {
  1772. "text": "<font color=\"#0057e7\">@ ${date}\n<b>${info}</b></font>"
  1773. }
  1774. }
  1775. ]
  1776. },
  1777. {
  1778. "widgets": [
  1779. {
  1780. "buttons": [
  1781. {
  1782. "textButton": {
  1783. "text": "Go to ${host}",
  1784. "onClick": {
  1785. "openLink": {
  1786. "url": "${goto_url}"
  1787. }
  1788. }
  1789. }
  1790. }
  1791. ]
  1792. }
  1793. ]
  1794. }
  1795. ]
  1796. }
  1797. ],
  1798. "thread": {
  1799. $thread
  1800. }
  1801. }
  1802. EOF
  1803. )"
  1804. httpcode=$(docurl -H "Content-Type: application/json" -X POST -d "${payload}" "${webhook}")
  1805. if [ "${httpcode}" = "200" ]; then
  1806. info "sent hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
  1807. sent=$((sent + 1))
  1808. else
  1809. error "failed to send hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
  1810. fi
  1811. fi
  1812. done
  1813. [ ${sent} -gt 0 ] && return 0
  1814. return 1
  1815. }
  1816. # -----------------------------------------------------------------------------
  1817. # Dynatrace sender
  1818. send_dynatrace() {
  1819. [ "${SEND_DYNATRACE}" != "YES" ] && return 1
  1820. local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
  1821. local description="Netdata Notification for: ${host} ${chart}.${name} is ${status}"
  1822. local payload=""
  1823. payload=$(cat <<EOF
  1824. {
  1825. "title": "Netdata Alarm from ${host}",
  1826. "source" : "${DYNATRACE_ANNOTATION_TYPE}",
  1827. "description" : "${description}",
  1828. "eventType": "${DYNATRACE_EVENT}",
  1829. "attachRules":{
  1830. "tagRule":[{
  1831. "meTypes":["HOST"],
  1832. "tags":["${DYNATRACE_TAG_VALUE}"]
  1833. }]
  1834. },
  1835. "customProperties":{
  1836. "description": "${description}"
  1837. }
  1838. }
  1839. EOF
  1840. )
  1841. # echo ${payload}
  1842. httpcode=$(docurl -X POST -H "Authorization: Api-token ${DYNATRACE_TOKEN}" -H "Content-Type: application/json" -d "${payload}" ${dynatrace_url})
  1843. ret=$?
  1844. if [ ${ret} -eq 0 ]; then
  1845. if [ "${httpcode}" = "200" ]; then
  1846. info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
  1847. return 0
  1848. else
  1849. warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
  1850. return 1
  1851. fi
  1852. else
  1853. error "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to ${DYNATRACE_SERVER} with error code ${ret}."
  1854. return 1
  1855. fi
  1856. }
  1857. # -----------------------------------------------------------------------------
  1858. # Stackpulse sender
  1859. send_stackpulse() {
  1860. local payload httpcode oldv currv
  1861. [ "${SEND_STACKPULSE}" != "YES" ] && return 1
  1862. # We are sending null when values are nan to avoid errors while JSON message is parsed
  1863. [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
  1864. [ "${value}" != "nan" ] && currv="${value}" || currv="null"
  1865. payload=$(cat <<EOF
  1866. {
  1867. "Node" : "${host}",
  1868. "Chart" : "${chart}",
  1869. "OldValue" : ${oldv},
  1870. "Value" : ${currv},
  1871. "Units" : "${units}",
  1872. "OldStatus" : "${old_status}",
  1873. "Status" : "${status}",
  1874. "Alarm" : "${name}",
  1875. "Date": ${when},
  1876. "Duration": ${duration},
  1877. "NonClearDuration": ${non_clear_duration},
  1878. "Description" : "${status_message}, ${info}",
  1879. "CalcExpression" : "${calc_expression}",
  1880. "CalcParamValues" : "${calc_param_values}",
  1881. "TotalWarnings" : "${total_warnings}",
  1882. "TotalCritical" : "${total_critical}",
  1883. "ID" : ${alarm_id}
  1884. }
  1885. EOF
  1886. )
  1887. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" ${STACKPULSE_WEBHOOK})
  1888. if [ "${httpcode}" = "200" ]; then
  1889. info "sent stackpulse notification for: ${host} ${chart}.${name} is ${status}"
  1890. else
  1891. error "failed to send stackpulse notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
  1892. return 1
  1893. fi
  1894. return 0
  1895. }
  1896. # -----------------------------------------------------------------------------
  1897. # Opsgenie sender
  1898. send_opsgenie() {
  1899. local payload httpcode oldv currv
  1900. [ "${SEND_OPSGENIE}" != "YES" ] && return 1
  1901. if [ -z "${OPSGENIE_API_KEY}" ] ; then
  1902. info "Can't send Opsgenie notification, because OPSGENIE_API_KEY is not defined"
  1903. return 1
  1904. fi
  1905. # We are sending null when values are nan to avoid errors while JSON message is parsed
  1906. [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
  1907. [ "${value}" != "nan" ] && currv="${value}" || currv="null"
  1908. payload=$(cat <<EOF
  1909. {
  1910. "host" : "${host}",
  1911. "unique_id" : "${unique_id}",
  1912. "alarmId" : ${alarm_id},
  1913. "eventId" : ${event_id},
  1914. "chart" : "${chart}",
  1915. "when": ${when},
  1916. "name" : "${name}",
  1917. "family" : "${family}",
  1918. "status" : "${status}",
  1919. "old_status" : "${old_status}",
  1920. "value" : ${currv},
  1921. "old_value" : ${oldv},
  1922. "duration": ${duration},
  1923. "non_clear_duration": ${non_clear_duration},
  1924. "units" : "${units}",
  1925. "info" : "${status_message}, ${info}",
  1926. "calc_expression" : "${calc_expression}",
  1927. "total_warnings" : "${total_warnings}",
  1928. "total_critical" : "${total_critical}",
  1929. "src" : "${src}"
  1930. }
  1931. EOF
  1932. )
  1933. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${OPSGENIE_API_URL}/v1/json/integrations/webhooks/netdata?apiKey=${OPSGENIE_API_KEY}")
  1934. # https://docs.opsgenie.com/docs/alert-api#create-alert
  1935. if [ "${httpcode}" = "200" ]; then
  1936. info "sent opsgenie notification for: ${host} ${chart}.${name} is ${status}"
  1937. else
  1938. error "failed to send opsgenie notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  1939. return 1
  1940. fi
  1941. return 0
  1942. }
  1943. # -----------------------------------------------------------------------------
  1944. # Gotify sender
  1945. send_gotify() {
  1946. local payload httpcode priority
  1947. [ "${SEND_GOTIFY}" != "YES" ] && return 1
  1948. if [ -z "${GOTIFY_APP_TOKEN}" ] ; then
  1949. info "Can't send Gotify notification, because GOTIFY_APP_TOKEN is not defined"
  1950. return 1
  1951. fi
  1952. # priority for Gotify Android app
  1953. case "${status}" in
  1954. CRITICAL) priority=10 ;; # sound + vibration
  1955. WARNING) priority=4 ;; # sound
  1956. *) priority=1 ;; # notification only
  1957. esac
  1958. payload=$(cat <<EOF
  1959. {
  1960. "title" : "${status}, ${name} = ${value_string}, on ${host}",
  1961. "message" : "${date}: ${chart} ${value_string}",
  1962. "priority" : ${priority}
  1963. }
  1964. EOF
  1965. )
  1966. httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${GOTIFY_APP_URL}/message?token=${GOTIFY_APP_TOKEN}")
  1967. if [ "${httpcode}" = "200" ]; then
  1968. info "sent gotify notification for: ${host} ${chart}.${name} is ${status}"
  1969. else
  1970. error "failed to send gotify notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
  1971. return 1
  1972. fi
  1973. return 0
  1974. }
  1975. # -----------------------------------------------------------------------------
  1976. # prepare the content of the notification
  1977. # the url to send the user on click
  1978. urlencode "${args_host}" >/dev/null
  1979. url_host="${REPLY}"
  1980. urlencode "${chart}" >/dev/null
  1981. url_chart="${REPLY}"
  1982. urlencode "${family}" >/dev/null
  1983. url_family="${REPLY}"
  1984. urlencode "${name}" >/dev/null
  1985. url_name="${REPLY}"
  1986. urlencode "${value_string}" >/dev/null
  1987. url_value_string="${REPLY}"
  1988. redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}&alarm_status=${status}&alarm_chart=${chart}&alarm_value=${url_value_string}"
  1989. GOTOCLOUD=0
  1990. if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
  1991. if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1992. if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
  1993. NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
  1994. fi
  1995. fi
  1996. if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
  1997. GOTOCLOUD=1
  1998. fi
  1999. fi
  2000. if [ ${GOTOCLOUD} -eq 0 ]; then
  2001. goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
  2002. else
  2003. # Temporarily disable alarm redirection, as the cloud endpoint no longer exists. This functionality will be restored after discussion on #9487. For now, just lead to netdata.cloud
  2004. # Re-allow alarm redirection, for alarms 2.0, new template
  2005. if [ -z "${child_machine_guid}" ]; then
  2006. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentId=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
  2007. else
  2008. goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentId=${NETDATA_REGISTRY_UNIQUE_ID}&childId=${child_machine_guid}&${redirect_params}"
  2009. fi
  2010. fi
  2011. # the severity of the alarm
  2012. severity="${status}"
  2013. # the time the alarm was raised
  2014. duration4human ${duration} >/dev/null
  2015. duration_txt="${REPLY}"
  2016. duration4human ${non_clear_duration} >/dev/null
  2017. non_clear_duration_txt="${REPLY}"
  2018. raised_for="(was ${old_status,,} for ${duration_txt})"
  2019. # the key status message
  2020. status_message="status unknown"
  2021. # the color of the alarm
  2022. color="grey"
  2023. # the alarm value
  2024. alarm="${name//_/ } = ${value_string}"
  2025. # the image of the alarm
  2026. image="${images_base_url}/images/banner-icon-144x144.png"
  2027. # have a default email status, in case the following case does not catch it
  2028. status_email_subject="${status}"
  2029. # prepare the title based on status
  2030. case "${status}" in
  2031. CRITICAL)
  2032. image="${images_base_url}/images/alert-128-red.png"
  2033. alarm_badge="https://app.netdata.cloud/static/email/img/label_critical.png"
  2034. status_message="is critical"
  2035. status_email_subject="Critical"
  2036. color="#ca414b"
  2037. rich_status_raised_for="Raised to critical, for ${non_clear_duration_txt}"
  2038. background_color="#FFEBEF"
  2039. border_color="#FF4136"
  2040. text_color="#FF4136"
  2041. action_text_color="#FFFFFF"
  2042. ;;
  2043. WARNING)
  2044. image="${images_base_url}/images/alert-128-orange.png"
  2045. alarm_badge="https://app.netdata.cloud/static/email/img/label_warning.png"
  2046. status_message="needs attention"
  2047. status_email_subject="Warning"
  2048. color="#ffc107"
  2049. rich_status_raised_for="Raised to warning, for ${non_clear_duration_txt}"
  2050. background_color="#FFF8E1"
  2051. border_color="#FFC300"
  2052. text_color="#536775"
  2053. action_text_color="#35414A"
  2054. ;;
  2055. CLEAR)
  2056. image="${images_base_url}/images/check-mark-2-128-green.png"
  2057. alarm_badge="https://app.netdata.cloud/static/email/img/label_recovered.png"
  2058. status_message="recovered"
  2059. status_email_subject="Clear"
  2060. color="#77ca6d"
  2061. rich_status_raised_for=
  2062. background_color="#E5F5E8"
  2063. border_color="#68C47D"
  2064. text_color="#00AB44"
  2065. action_text_color="#FFFFFF"
  2066. ;;
  2067. esac
  2068. # the html email subject
  2069. html_email_subject="${status_email_subject}, ${name} = ${value_string}, on ${host}"
  2070. if [ "${status}" = "CLEAR" ]; then
  2071. severity="Recovered from ${old_status}"
  2072. if [ ${non_clear_duration} -gt ${duration} ]; then
  2073. raised_for="(alarm was raised for ${non_clear_duration_txt})"
  2074. fi
  2075. rich_status_raised_for="Recovered from ${old_status,,}, ${raised_for}"
  2076. # don't show the value when the status is CLEAR
  2077. # for certain alarms, this value might not have any meaning
  2078. alarm="${name//_/ } ${raised_for}"
  2079. html_email_subject="${status_email_subject}, ${name} ${raised_for}, on ${host}"
  2080. elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
  2081. severity="Escalated to ${status}"
  2082. if [ ${non_clear_duration} -gt ${duration} ]; then
  2083. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  2084. fi
  2085. rich_status_raised_for="Escalated to critical, ${raised_for}"
  2086. elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
  2087. severity="Demoted to ${status}"
  2088. if [ ${non_clear_duration} -gt ${duration} ]; then
  2089. raised_for="(alarm is raised for ${non_clear_duration_txt})"
  2090. fi
  2091. rich_status_raised_for="Demoted to warning, ${raised_for}"
  2092. else
  2093. raised_for=
  2094. fi
  2095. # prepare HTML versions of elements
  2096. info_html=
  2097. [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
  2098. raised_for_html=
  2099. [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
  2100. # -----------------------------------------------------------------------------
  2101. # send the slack notification
  2102. # slack aggregates posts from the same username
  2103. # so we use "${host} ${status}" as the bot username, to make them diff
  2104. send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
  2105. SENT_SLACK=$?
  2106. # -----------------------------------------------------------------------------
  2107. # send the hangouts notification
  2108. # hangouts aggregates posts from the same room
  2109. # so we use "${host} ${status}" as the room, to make them diff
  2110. send_hangouts "${to_hangouts}"
  2111. SENT_HANGOUTS=$?
  2112. # -----------------------------------------------------------------------------
  2113. # send the Microsoft Teams notification
  2114. # Microsoft teams aggregates posts from the same username
  2115. # so we use "${host} ${status}" as the bot username, to make them diff
  2116. send_msteams "${MSTEAMS_WEBHOOK_URL}" "${to_msteams}"
  2117. SENT_MSTEAMS=$?
  2118. # -----------------------------------------------------------------------------
  2119. # send the rocketchat notification
  2120. # rocketchat aggregates posts from the same username
  2121. # so we use "${host} ${status}" as the bot username, to make them diff
  2122. send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
  2123. SENT_ROCKETCHAT=$?
  2124. # -----------------------------------------------------------------------------
  2125. # send the alerta notification
  2126. # alerta aggregates posts from the same username
  2127. # so we use "${host} ${status}" as the bot username, to make them diff
  2128. send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
  2129. SENT_ALERTA=$?
  2130. # -----------------------------------------------------------------------------
  2131. # send the flock notification
  2132. # flock aggregates posts from the same username
  2133. # so we use "${host} ${status}" as the bot username, to make them diff
  2134. send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
  2135. SENT_FLOCK=$?
  2136. # -----------------------------------------------------------------------------
  2137. # send the discord notification
  2138. # discord aggregates posts from the same username
  2139. # so we use "${host} ${status}" as the bot username, to make them diff
  2140. send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
  2141. SENT_DISCORD=$?
  2142. # -----------------------------------------------------------------------------
  2143. # send the pushover notification
  2144. send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
  2145. <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
  2146. <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
  2147. <small><b>${family}</b><br/>Family<br/>&nbsp;</small>
  2148. <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
  2149. <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
  2150. <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
  2151. <small><small>The source of this alarm is line ${src}</small></small>
  2152. "
  2153. SENT_PUSHOVER=$?
  2154. # -----------------------------------------------------------------------------
  2155. # send the pushbullet notification
  2156. send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
  2157. Severity: ${severity}\\n
  2158. Chart: ${chart}\\n
  2159. Family: ${family}\\n
  2160. ${date}\\n
  2161. The source of this alarm is line ${src}"
  2162. SENT_PUSHBULLET=$?
  2163. # -----------------------------------------------------------------------------
  2164. # send the twilio SMS
  2165. send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2166. Severity: ${severity}
  2167. Chart: ${chart}
  2168. Family: ${family}
  2169. ${info}"
  2170. SENT_TWILIO=$?
  2171. # -----------------------------------------------------------------------------
  2172. # send the messagebird SMS
  2173. send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2174. Severity: ${severity}
  2175. Chart: ${chart}
  2176. Family: ${family}
  2177. ${info}"
  2178. SENT_MESSAGEBIRD=$?
  2179. # -----------------------------------------------------------------------------
  2180. # send the kavenegar SMS
  2181. send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
  2182. Severity: ${severity}
  2183. Chart: ${chart}
  2184. Family: ${family}
  2185. ${info}"
  2186. SENT_KAVENEGAR=$?
  2187. # -----------------------------------------------------------------------------
  2188. # send the telegram.org message
  2189. # https://core.telegram.org/bots/api#formatting-options
  2190. send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
  2191. ${chart} (${family})
  2192. <a href=\"${goto_url}\">${alarm}</a>
  2193. <i>${info}</i>"
  2194. SENT_TELEGRAM=$?
  2195. # -----------------------------------------------------------------------------
  2196. # send the kafka message
  2197. send_kafka
  2198. SENT_KAFKA=$?
  2199. # -----------------------------------------------------------------------------
  2200. # send the pagerduty.com message
  2201. send_pd "${to_pd}"
  2202. SENT_PD=$?
  2203. # -----------------------------------------------------------------------------
  2204. # send the fleep message
  2205. send_fleep "${to_fleep}"
  2206. SENT_FLEEP=$?
  2207. # -----------------------------------------------------------------------------
  2208. # send the Prowl message
  2209. send_prowl "${to_prowl}"
  2210. SENT_PROWL=$?
  2211. # -----------------------------------------------------------------------------
  2212. # send the irc message
  2213. send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${IRC_PORT}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
  2214. Severity: ${severity}
  2215. Chart: ${chart}
  2216. Family: ${family}
  2217. ${info}"
  2218. SENT_IRC=$?
  2219. # -----------------------------------------------------------------------------
  2220. # send the SMS message with smstools3
  2221. send_sms "${to_sms}"
  2222. SENT_SMS=$?
  2223. # -----------------------------------------------------------------------------
  2224. # send the custom message
  2225. send_custom() {
  2226. # is it enabled?
  2227. [ "${SEND_CUSTOM}" != "YES" ] && return 1
  2228. # do we have any sender?
  2229. [ -z "${1}" ] && return 1
  2230. # call the custom_sender function
  2231. custom_sender "${@}"
  2232. }
  2233. send_custom "${to_custom}"
  2234. SENT_CUSTOM=$?
  2235. # -----------------------------------------------------------------------------
  2236. # send hipchat message
  2237. send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
  2238. ${host} ${status_message}<br/> \
  2239. <b>${alarm}</b> ${info_html}<br/> \
  2240. <b>${chart}</b> (family <b>${family}</b>)<br/> \
  2241. <b>${date}${raised_for_html}</b><br/> \
  2242. <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
  2243. (source of alarm ${src}) \
  2244. "
  2245. SENT_HIPCHAT=$?
  2246. # -----------------------------------------------------------------------------
  2247. # send the Amazon SNS message
  2248. send_awssns "${to_awssns}"
  2249. SENT_AWSSNS=$?
  2250. # -----------------------------------------------------------------------------
  2251. # send the Matrix message
  2252. send_matrix "${MATRIX_HOMESERVER}" "${to_matrix}"
  2253. SENT_MATRIX=$?
  2254. # -----------------------------------------------------------------------------
  2255. # send the syslog message
  2256. send_syslog "${to_syslog}"
  2257. SENT_SYSLOG=$?
  2258. # -----------------------------------------------------------------------------
  2259. # send the email
  2260. IFS='' read -r -d '' email_plaintext_part <<EOF
  2261. Content-Type: text/plain; encoding=${EMAIL_CHARSET}
  2262. Content-Disposition: inline
  2263. Content-Transfer-Encoding: 8bit
  2264. ${host} ${status_message}
  2265. ${alarm} ${info}
  2266. ${raised_for}
  2267. Chart : ${chart}
  2268. Family : ${family}
  2269. Severity: ${severity}
  2270. URL : ${goto_url}
  2271. Source : ${src}
  2272. Date : ${date}
  2273. Notification generated on ${host}
  2274. Evaluated Expression : ${calc_expression}
  2275. Expression Variables : ${calc_param_values}
  2276. The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
  2277. EOF
  2278. if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
  2279. send_email <<EOF
  2280. To: ${to_email}
  2281. Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
  2282. MIME-Version: 1.0
  2283. Content-Type: multipart/alternative; boundary="multipart-boundary"
  2284. ${email_thread_headers}
  2285. X-Netdata-Severity: ${status,,}
  2286. X-Netdata-Alert-Name: $name
  2287. X-Netdata-Chart: $chart
  2288. X-Netdata-Family: $family
  2289. X-Netdata-Classification: $classification
  2290. X-Netdata-Host: $host
  2291. X-Netdata-Role: $roles
  2292. This is a MIME-encoded multipart message
  2293. --multipart-boundary
  2294. ${email_plaintext_part}
  2295. --multipart-boundary--
  2296. EOF
  2297. else
  2298. now=$(date "+%s")
  2299. if [ -n "$total_warn_alarms" ]; then
  2300. while read -d, -r pair; do
  2301. IFS='=' read -r key val <<<"$pair"
  2302. date_w=$(date --date=@${val} "${date_format}" 2>/dev/null)
  2303. [ -z "${date_w}" ] && date_w=$(date "${date_format}" 2>/dev/null)
  2304. [ -z "${date_w}" ] && date_w=$(date --date=@${val} 2>/dev/null)
  2305. [ -z "${date_w}" ] && date_w=$(date 2>/dev/null)
  2306. elapsed=$((now - val))
  2307. duration4human ${elapsed} >/dev/null
  2308. elapsed_txt="${REPLY}"
  2309. WARN_ALARMS+="
  2310. <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
  2311. <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
  2312. <tbody>
  2313. <tr>
  2314. <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
  2315. <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2316. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2317. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
  2318. <tbody>
  2319. <tr>
  2320. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2321. <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
  2322. </td>
  2323. </tr>
  2324. <tr>
  2325. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
  2326. <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_w}</div>
  2327. </td>
  2328. </tr>
  2329. </tbody>
  2330. </table>
  2331. </div>
  2332. <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2333. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2334. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
  2335. <tbody>
  2336. <tr>
  2337. <td style=\"vertical-align:top;padding-top:13px;\">
  2338. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
  2339. <tbody>
  2340. <tr>
  2341. <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2342. <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#555555;\"><span style=\"background-color:#FFF8E1; border: 1px solid #FFC300; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
  2343. Warning for ${elapsed_txt}
  2344. </span></div>
  2345. </td>
  2346. </tr>
  2347. </tbody>
  2348. </table>
  2349. </td>
  2350. </tr>
  2351. </tbody>
  2352. </table>
  2353. </div>
  2354. <!--[if mso | IE]></td></tr></table><![endif]-->
  2355. </td>
  2356. </tr>
  2357. </tbody>
  2358. </table>
  2359. </div>
  2360. "
  2361. done <<<"$total_warn_alarms,"
  2362. fi
  2363. if [ -n "$total_crit_alarms" ]; then
  2364. while read -d, -r pair; do
  2365. IFS='=' read -r key val <<<"$pair"
  2366. date_c=$(date --date=@${val} "${date_format}" 2>/dev/null)
  2367. [ -z "${date_c}" ] && date_c=$(date "${date_format}" 2>/dev/null)
  2368. [ -z "${date_c}" ] && date_c=$(date --date=@${val} 2>/dev/null)
  2369. [ -z "${date_c}" ] && date_c=$(date 2>/dev/null)
  2370. elapsed=$((now - val))
  2371. duration4human ${elapsed} >/dev/null
  2372. elapsed_txt="${REPLY}"
  2373. CRIT_ALARMS+="
  2374. <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
  2375. <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
  2376. <tbody>
  2377. <tr>
  2378. <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
  2379. <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2380. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2381. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
  2382. <tbody>
  2383. <tr>
  2384. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2385. <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
  2386. </td>
  2387. </tr>
  2388. <tr>
  2389. <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
  2390. <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_c}</div>
  2391. </td>
  2392. </tr>
  2393. </tbody>
  2394. </table>
  2395. </div>
  2396. <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
  2397. <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
  2398. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
  2399. <tbody>
  2400. <tr>
  2401. <td style=\"vertical-align:top;padding-top:13px;\">
  2402. <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
  2403. <tbody>
  2404. <tr>
  2405. <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
  2406. <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#35414A;\"><span style=\"background-color:#FFEBEF; border: 1px solid #FF4136; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
  2407. Critical for ${elapsed_txt}
  2408. </span></div>
  2409. </td>
  2410. </tr>
  2411. </tbody>
  2412. </table>
  2413. </td>
  2414. </tr>
  2415. </tbody>
  2416. </table>
  2417. </div>
  2418. <!--[if mso | IE]></td></tr></table><![endif]-->
  2419. </td>
  2420. </tr>
  2421. </tbody>
  2422. </table>
  2423. </div>
  2424. "
  2425. done <<<"$total_crit_alarms,"
  2426. fi
  2427. if (( total_warnings + total_critical > 15 )); then
  2428. EXTRA_ALARMS_LIST_TEXT="(Showing latest 15 alerts)"
  2429. fi
  2430. if [ -n "$edit_command_line" ]; then
  2431. IFS='=' read -r edit_command line s_host <<<"$edit_command_line"
  2432. fi
  2433. IFS='' read -r -d '' email_html_part <<EOF
  2434. Content-Type: text/html; encoding=${EMAIL_CHARSET}
  2435. Content-Disposition: inline
  2436. Content-Transfer-Encoding: 8bit
  2437. <!doctype html>
  2438. <html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office">
  2439. <head>
  2440. <title>
  2441. </title>
  2442. <!--[if !mso]><!-->
  2443. <meta http-equiv="X-UA-Compatible" content="IE=edge">
  2444. <!--<![endif]-->
  2445. <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  2446. <meta name="viewport" content="width=device-width, initial-scale=1">
  2447. <style type="text/css">
  2448. #outlook a { padding:0; }
  2449. body { margin:0;padding:0;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%; }
  2450. table, td { border-collapse:collapse;mso-table-lspace:0pt;mso-table-rspace:0pt; }
  2451. img { border:0;height:auto;line-height:100%; outline:none;text-decoration:none;-ms-interpolation-mode:bicubic; }
  2452. p { display:block;margin:13px 0; }
  2453. </style>
  2454. <!--[if mso]>
  2455. <xml>
  2456. <o:OfficeDocumentSettings>
  2457. <o:AllowPNG/>
  2458. <o:PixelsPerInch>96</o:PixelsPerInch>
  2459. </o:OfficeDocumentSettings>
  2460. </xml>
  2461. <![endif]-->
  2462. <!--[if lte mso 11]>
  2463. <style type="text/css">
  2464. .mj-outlook-group-fix { width:100% !important; }
  2465. </style>
  2466. <![endif]-->
  2467. <!--[if !mso]><!-->
  2468. <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap" rel="stylesheet" type="text/css">
  2469. <link href="https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700" rel="stylesheet" type="text/css">
  2470. <style type="text/css">
  2471. @import url(https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap);
  2472. @import url(https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700);
  2473. </style>
  2474. <!--<![endif]-->
  2475. <style type="text/css">
  2476. @media only screen and (min-width:100px) {
  2477. .mj-column-px-130 { width:130px !important; max-width: 130px; }
  2478. .mj-column-per-50 { width:50% !important; max-width: 50%; }
  2479. .mj-column-per-70 { width:70% !important; max-width: 70%; }
  2480. .mj-column-per-30 { width:30% !important; max-width: 30%; }
  2481. .mj-column-per-100 { width:100% !important; max-width: 100%; }
  2482. .mj-column-px-66 { width:66px !important; max-width: 66px; }
  2483. .mj-column-px-400 { width:400px !important; max-width: 400px; }
  2484. }
  2485. </style>
  2486. <style type="text/css">
  2487. @media only screen and (max-width:100px) {
  2488. table.mj-full-width-mobile { width: 100% !important; }
  2489. td.mj-full-width-mobile { width: auto !important; }
  2490. }
  2491. </style>
  2492. </head>
  2493. <body style="word-spacing:normal;">
  2494. <div class="svgbg" style="background-image: url('https://staging.netdata.cloud/static/email/img/isotype_600.png'); background-repeat: no-repeat; background-position: top center; background-size: 600px 192px;">
  2495. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2496. <div style="margin:0px auto;max-width:600px;">
  2497. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2498. <tbody>
  2499. <tr>
  2500. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:50px;padding-left:0;text-align:left;">
  2501. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:130px;" ><![endif]-->
  2502. <div class="mj-column-px-130 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:130px;">
  2503. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2504. <tbody>
  2505. <tr>
  2506. <td align="center" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2507. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2508. <tbody>
  2509. <tr>
  2510. <td style="width:130px;">
  2511. <img alt="Netdata Logo" height="auto" src="https://app.netdata.cloud/static/email/img/full_logo.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="130">
  2512. </td>
  2513. </tr>
  2514. </tbody>
  2515. </table>
  2516. </td>
  2517. </tr>
  2518. </tbody>
  2519. </table>
  2520. </div>
  2521. <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:300px;" ><![endif]-->
  2522. <div class="mj-column-per-50 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;">
  2523. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2524. <tbody>
  2525. <tr>
  2526. <td style="vertical-align:top;padding-top:4px;">
  2527. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2528. <tbody>
  2529. <tr>
  2530. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:10px;word-break:break-word;">
  2531. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Notification</div>
  2532. </td>
  2533. </tr>
  2534. </tbody>
  2535. </table>
  2536. </td>
  2537. </tr>
  2538. </tbody>
  2539. </table>
  2540. </div>
  2541. <!--[if mso | IE]></td></tr></table><![endif]-->
  2542. </td>
  2543. </tr>
  2544. </tbody>
  2545. </table>
  2546. </div>
  2547. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="no-collapse-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2548. <div class="no-collapse" style="border-collapse: initial; margin: 0px auto; border-radius: 4px; max-width: 600px;">
  2549. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;border-radius:4px;">
  2550. <tbody>
  2551. <tr>
  2552. <td style="border:1px solid ${border_color};direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2553. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2554. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2555. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2556. <tbody>
  2557. <tr>
  2558. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2559. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:418.6px;" ><![endif]-->
  2560. <div class="mj-column-per-70 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:70%;">
  2561. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2562. <tbody>
  2563. <tr>
  2564. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:15px;word-break:break-word;">
  2565. <div style="font-family:Open Sans, sans-serif;font-size:20px;font-weight:700;line-height:1;text-align:left;color:#35414A;">${name}</div>
  2566. </td>
  2567. </tr>
  2568. </tbody>
  2569. </table>
  2570. </div>
  2571. <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:179.4px;" ><![endif]-->
  2572. <div class="mj-column-per-30 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:30%;">
  2573. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2574. <tbody>
  2575. <tr>
  2576. <td align="right" style="font-size:0px;padding:10px 25px;padding-right:25px;word-break:break-word;">
  2577. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2578. <tbody>
  2579. <tr>
  2580. <td style="width:100px;">
  2581. <img height="auto" src="${alarm_badge}" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="100"/>
  2582. </td>
  2583. </tr>
  2584. </tbody>
  2585. </table>
  2586. </td>
  2587. </tr>
  2588. </tbody>
  2589. </table>
  2590. </div>
  2591. <!--[if mso | IE]></td></tr></table><![endif]-->
  2592. </td>
  2593. </tr>
  2594. </tbody>
  2595. </table>
  2596. </div>
  2597. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2598. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2599. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2600. <tbody>
  2601. <tr>
  2602. <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
  2603. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2604. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2605. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2606. <tbody>
  2607. <tr>
  2608. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2609. <div style="font-family:IBM Plex Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">on ${host}</div>
  2610. </td>
  2611. </tr>
  2612. </tbody>
  2613. </table>
  2614. </div>
  2615. <!--[if mso | IE]></td></tr></table><![endif]-->
  2616. </td>
  2617. </tr>
  2618. </tbody>
  2619. </table>
  2620. </div>
  2621. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2622. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2623. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2624. <tbody>
  2625. <tr>
  2626. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2627. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2628. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2629. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2630. <tbody>
  2631. <tr>
  2632. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2633. <div style="font-family:Open Sans, sans-serif;font-size:26px;font-weight:700;line-height:1;text-align:left;color:#35414A;"><span style="color: ${text_color}; font-size:26px; background: ${background_color}; padding:4px 24px; border-radius: 36px">${value_string}
  2634. </span></div>
  2635. </td>
  2636. </tr>
  2637. </tbody>
  2638. </table>
  2639. </div>
  2640. <!--[if mso | IE]></td></tr></table><![endif]-->
  2641. </td>
  2642. </tr>
  2643. </tbody>
  2644. </table>
  2645. </div>
  2646. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2647. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2648. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2649. <tbody>
  2650. <tr>
  2651. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2652. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2653. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2654. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2655. <tbody>
  2656. <tr>
  2657. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2658. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:21px;text-align:left;color:#35414A;">Details: ${info}</div>
  2659. </td>
  2660. </tr>
  2661. </tbody>
  2662. </table>
  2663. </div>
  2664. <!--[if mso | IE]></td></tr></table><![endif]-->
  2665. </td>
  2666. </tr>
  2667. </tbody>
  2668. </table>
  2669. </div>
  2670. <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2671. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
  2672. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2673. <tbody>
  2674. <tr>
  2675. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
  2676. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
  2677. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2678. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2679. <tbody>
  2680. <tr>
  2681. <td align="center" vertical-align="middle" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2682. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:separate;width:100%;line-height:100%;">
  2683. <tr>
  2684. <td
  2685. align="center" bgcolor="${border_color}" role="presentation" style="border:none;border-radius:3px;cursor:auto;height:44px;background:${border_color};" valign="middle">
  2686. <p style="display:block;background:${border_color};color:#ffffff;font-size:13px;font-weight:600;line-height:44px;margin:0;text-decoration:none;text-transform:none;mso-padding-alt:0px;border-radius:3px;">
  2687. <a href="${goto_url}" style="color: ${action_text_color}; text-decoration: none; width: 100%; display: inline-block">GO TO CHART</a>
  2688. </p>
  2689. </td>
  2690. </tr>
  2691. </table>
  2692. </td>
  2693. </tr>
  2694. </tbody>
  2695. </table>
  2696. </div>
  2697. <!--[if mso | IE]></td></tr></table><![endif]-->
  2698. </td>
  2699. </tr>
  2700. </tbody>
  2701. </table>
  2702. </div>
  2703. <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
  2704. </td>
  2705. </tr>
  2706. </tbody>
  2707. </table>
  2708. </div>
  2709. <!--[if mso | IE]></td></tr></table><![endif]-->
  2710. <div style="height:32px;line-height:32px;">&#8202;</div>
  2711. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2712. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; background: ${background_color}; background-color: ${background_color}; margin: 0px auto; border-radius: 4px; max-width: 600px;">
  2713. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:${background_color};background-color:${background_color};width:100%;border-radius:4px;">
  2714. <tbody>
  2715. <tr>
  2716. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2717. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2718. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2719. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2720. <tbody>
  2721. <tr>
  2722. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2723. <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Chart:
  2724. <span style="font-weight:700; font-size:20px">${chart}</span></div>
  2725. </td>
  2726. </tr>
  2727. <tr>
  2728. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2729. <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Family:
  2730. <span style="font-weight:700; font-size:20px">${family}</span></div>
  2731. </td>
  2732. </tr>
  2733. <tr>
  2734. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
  2735. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">${rich_status_raised_for}</div>
  2736. </td>
  2737. </tr>
  2738. <tr>
  2739. <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2740. <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
  2741. </p>
  2742. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
  2743. </td></tr></table><![endif]-->
  2744. </td>
  2745. </tr>
  2746. <tr>
  2747. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2748. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">On
  2749. <span style="font-weight:600">${date}</span></div>
  2750. </td>
  2751. </tr>
  2752. <tr>
  2753. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2754. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">By:
  2755. <span style="font-weight:600">${host}</span></div>
  2756. </td>
  2757. </tr>
  2758. <tr>
  2759. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
  2760. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">Global time:
  2761. <span style="font-weight:600">${date_utc}</span></div>
  2762. </td>
  2763. </tr>
  2764. <tr>
  2765. <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2766. <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
  2767. </p>
  2768. <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
  2769. </td></tr></table><![endif]-->
  2770. </td>
  2771. </tr>
  2772. <tr>
  2773. <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
  2774. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Classification:
  2775. <span style="font-weight:600">${classification}</span></div>
  2776. </td>
  2777. </tr>
  2778. <tr>
  2779. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
  2780. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Role:
  2781. <span style="font-weight:600">${roles}</span></div>
  2782. </td>
  2783. </tr>
  2784. </tbody>
  2785. </table>
  2786. </div>
  2787. <!--[if mso | IE]></td></tr></table><![endif]-->
  2788. </td>
  2789. </tr>
  2790. </tbody>
  2791. </table>
  2792. </div>
  2793. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2794. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2795. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2796. <tbody>
  2797. <tr>
  2798. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
  2799. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
  2800. <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
  2801. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2802. <tbody>
  2803. <tr>
  2804. <td style="vertical-align:top;padding:0;">
  2805. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2806. <tbody>
  2807. <tr>
  2808. <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2809. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2810. <tbody>
  2811. <tr>
  2812. <td style="width:48px;">
  2813. <img height="auto" src="https://app.netdata.cloud/static/email/img/community_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
  2814. </td>
  2815. </tr>
  2816. </tbody>
  2817. </table>
  2818. </td>
  2819. </tr>
  2820. </tbody>
  2821. </table>
  2822. </td>
  2823. </tr>
  2824. </tbody>
  2825. </table>
  2826. </div>
  2827. <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
  2828. <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
  2829. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2830. <tbody>
  2831. <tr>
  2832. <td style="vertical-align:top;padding-left:0;">
  2833. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2834. <tbody>
  2835. <tr>
  2836. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2837. <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Want to know more about this alert?</div>
  2838. </td>
  2839. </tr>
  2840. <tr>
  2841. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2842. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;">Join the troubleshooting discussion for this alert on our <a href="https://community.netdata.cloud/t/${name//[._]/-}" class="link" style="color: #00AB44; text-decoration: none;">community forums</a>.</div>
  2843. </td>
  2844. </tr>
  2845. </tbody>
  2846. </table>
  2847. </td>
  2848. </tr>
  2849. </tbody>
  2850. </table>
  2851. </div>
  2852. <!--[if mso | IE]></td></tr></table><![endif]-->
  2853. </td>
  2854. </tr>
  2855. </tbody>
  2856. </table>
  2857. </div>
  2858. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2859. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2860. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2861. <tbody>
  2862. <tr>
  2863. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
  2864. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
  2865. <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
  2866. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2867. <tbody>
  2868. <tr>
  2869. <td style="vertical-align:top;padding:0;">
  2870. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2871. <tbody>
  2872. <tr>
  2873. <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
  2874. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
  2875. <tbody>
  2876. <tr>
  2877. <td style="width:48px;">
  2878. <img height="auto" src="https://app.netdata.cloud/static/email/img/configure_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
  2879. </td>
  2880. </tr>
  2881. </tbody>
  2882. </table>
  2883. </td>
  2884. </tr>
  2885. </tbody>
  2886. </table>
  2887. </td>
  2888. </tr>
  2889. </tbody>
  2890. </table>
  2891. </div>
  2892. <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
  2893. <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
  2894. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2895. <tbody>
  2896. <tr>
  2897. <td style="vertical-align:top;padding-left:0;">
  2898. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2899. <tbody>
  2900. <tr>
  2901. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2902. <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Need to configure this alert?</div>
  2903. </td>
  2904. </tr>
  2905. <tr>
  2906. <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
  2907. <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;"><span style="color: #00AB44"><a href="https://learn.netdata.cloud/docs/agent/health/notifications#:~:text=To%20edit%20it%20on%20your,have%20one%20or%20more%20destinations" class="link" style="color: #00AB44; text-decoration: none;">Edit</a></span> this alert's configuration file by logging into $s_host and running the following command:</div>
  2908. </td>
  2909. </tr>
  2910. <tr>
  2911. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:8px;padding-left:0;word-break:break-word;">
  2912. <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1.3;text-align:left;color:#35414A;">${edit_command} <br>
  2913. <br>The alarm to edit is at line ${line}</div>
  2914. </td>
  2915. </tr>
  2916. </tbody>
  2917. </table>
  2918. </td>
  2919. </tr>
  2920. </tbody>
  2921. </table>
  2922. </div>
  2923. <!--[if mso | IE]></td></tr></table><![endif]-->
  2924. </td>
  2925. </tr>
  2926. </tbody>
  2927. </table>
  2928. </div>
  2929. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="history-wrapper-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2930. <div class="history-wrapper" style="background: #F7F8F8; background-color: #F7F8F8; margin: 0px auto; max-width: 100%;">
  2931. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:#F7F8F8;background-color:#F7F8F8;width:100%;">
  2932. <tbody>
  2933. <tr>
  2934. <td style="direction:ltr;font-size:0px;padding:0;padding-bottom:24px;text-align:center;">
  2935. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2936. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2937. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2938. <tbody>
  2939. <tr>
  2940. <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:12px;text-align:center;">
  2941. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2942. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2943. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
  2944. <tbody>
  2945. <tr>
  2946. <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2947. <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:center;color:#35414A;">The node has
  2948. <span style="font-weight:600">${total_warnings} warning</span>
  2949. and
  2950. <span style="font-weight:600">${total_critical} critical</span>
  2951. additional active alert(s)</div>
  2952. </td>
  2953. </tr>
  2954. <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
  2955. <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:center;color:#35414A;">${EXTRA_ALARMS_LIST_TEXT}</div>
  2956. </td>
  2957. </tr>
  2958. </tbody>
  2959. </table>
  2960. </div>
  2961. <!--[if mso | IE]></td></tr></table><![endif]-->
  2962. </td>
  2963. </tr>
  2964. </tbody>
  2965. </table>
  2966. </div>
  2967. ${CRIT_ALARMS}
  2968. ${WARN_ALARMS}
  2969. <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
  2970. </td>
  2971. </tr>
  2972. </tbody>
  2973. </table>
  2974. </div>
  2975. <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
  2976. <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
  2977. <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
  2978. <tbody>
  2979. <tr>
  2980. <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
  2981. <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
  2982. <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
  2983. <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
  2984. <tbody>
  2985. <tr>
  2986. <td style="vertical-align:top;padding-top:44px;padding-bottom:12px;">
  2987. <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
  2988. <tbody>
  2989. <tr>
  2990. <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;padding-bottom:0;word-break:break-word;">
  2991. <div style="font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:center;color:#35414A;">© Netdata 2021 - The real-time performance and health monitoring</div>
  2992. </td>
  2993. </tr>
  2994. </tbody>
  2995. </table>
  2996. </td>
  2997. </tr>
  2998. </tbody>
  2999. </table>
  3000. </div>
  3001. <!--[if mso | IE]></td></tr></table><![endif]-->
  3002. </td>
  3003. </tr>
  3004. </tbody>
  3005. </table>
  3006. </div>
  3007. <!--[if mso | IE]></td></tr></table><![endif]-->
  3008. </div>
  3009. </body>
  3010. </html>
  3011. EOF
  3012. send_email <<EOF
  3013. To: ${to_email}
  3014. Subject: ${html_email_subject}
  3015. MIME-Version: 1.0
  3016. Content-Type: multipart/alternative; boundary="multipart-boundary"
  3017. ${email_thread_headers}
  3018. X-Netdata-Severity: ${status,,}
  3019. X-Netdata-Alert-Name: $name
  3020. X-Netdata-Chart: $chart
  3021. X-Netdata-Family: $family
  3022. X-Netdata-Classification: $classification
  3023. X-Netdata-Host: $host
  3024. X-Netdata-Role: $roles
  3025. This is a MIME-encoded multipart message
  3026. --multipart-boundary
  3027. ${email_plaintext_part}
  3028. --multipart-boundary
  3029. ${email_html_part}
  3030. --multipart-boundary--
  3031. EOF
  3032. fi
  3033. SENT_EMAIL=$?
  3034. # -----------------------------------------------------------------------------
  3035. # send the EVENT to Dynatrace
  3036. send_dynatrace "${host}" "${chart}" "${name}" "${status}"
  3037. SENT_DYNATRACE=$?
  3038. # -----------------------------------------------------------------------------
  3039. # send the EVENT to Stackpulse
  3040. send_stackpulse
  3041. SENT_STACKPULSE=$?
  3042. # -----------------------------------------------------------------------------
  3043. # send messages to Opsgenie
  3044. send_opsgenie
  3045. SENT_OPSGENIE=$?
  3046. # -----------------------------------------------------------------------------
  3047. # send messages to Gotify
  3048. send_gotify
  3049. SENT_GOTIFY=$?
  3050. # -----------------------------------------------------------------------------
  3051. # let netdata know
  3052. for state in "${SENT_EMAIL}" \
  3053. "${SENT_PUSHOVER}" \
  3054. "${SENT_TELEGRAM}" \
  3055. "${SENT_SLACK}" \
  3056. "${SENT_HANGOUTS}" \
  3057. "${SENT_ROCKETCHAT}" \
  3058. "${SENT_ALERTA}" \
  3059. "${SENT_FLOCK}" \
  3060. "${SENT_DISCORD}" \
  3061. "${SENT_TWILIO}" \
  3062. "${SENT_HIPCHAT}" \
  3063. "${SENT_MESSAGEBIRD}" \
  3064. "${SENT_KAVENEGAR}" \
  3065. "${SENT_PUSHBULLET}" \
  3066. "${SENT_KAFKA}" \
  3067. "${SENT_PD}" \
  3068. "${SENT_FLEEP}" \
  3069. "${SENT_PROWL}" \
  3070. "${SENT_CUSTOM}" \
  3071. "${SENT_IRC}" \
  3072. "${SENT_AWSSNS}" \
  3073. "${SENT_MATRIX}" \
  3074. "${SENT_SYSLOG}" \
  3075. "${SENT_SMS}" \
  3076. "${SENT_MSTEAMS}" \
  3077. "${SENT_DYNATRACE}" \
  3078. "${SENT_STACKPULSE}" \
  3079. "${SENT_OPSGENIE}" \
  3080. "${SENT_GOTIFY}"; do
  3081. if [ "${state}" -eq 0 ]; then
  3082. # we sent something
  3083. exit 0
  3084. fi
  3085. done
  3086. # we did not send anything
  3087. exit 1