netdata/health/health.d/web_log.conf

# -----------------------------------------------------------------------------
# high level response code alarms

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: 1m_requests
       on: web_log.response_statuses
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned
     calc: ($this == 0)?(1):($this)
    units: requests
    every: 10s
     info: number of HTTP requests in the last minute

 template: 1m_successful
       on: web_log.response_statuses
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of successful_requests
     calc: $this * 100 / $1m_requests
    units: %
    every: 10s
     warn: ($1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
     crit: ($1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
       to: webmaster

 template: 1m_redirects
       on: web_log.response_statuses
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of redirects
     calc: $this * 100 / $1m_requests
    units: %
    every: 10s
     warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING ) ? (  1 ) : ( 20 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of redirection HTTP requests over the last minute (3xx except 304)
       to: webmaster

 template: 1m_bad_requests
       on: web_log.response_statuses
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of bad_requests
     calc: $this * 100 / $1m_requests
    units: %
    every: 10s
     warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 10 ) : ( 30 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of client error HTTP requests over the last minute (4xx except 401)
       to: webmaster

 template: 1m_internal_errors
       on: web_log.response_statuses
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of server_errors
     calc: $this * 100 / $1m_requests
    units: %
    every: 10s
     warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 1 ) : ( 2 )) ) : ( 0 )
     crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of server error HTTP requests over the last minute (5xx)
       to: webmaster

# unmatched lines

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_total_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: 1m_total_requests
       on: web_log.response_codes
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned
     calc: ($this == 0)?(1):($this)
    units: requests
    every: 10s
     info: number of HTTP requests over the last minute

 template: 1m_unmatched
       on: web_log.response_codes
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of unmatched
     calc: $this * 100 / $1m_total_requests
    units: %
    every: 10s
     warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 )
    delay: up 1m down 5m multiplier 1.5 max 1h
     info: percentage of unparsed log lines over the last minute
       to: webmaster

# -----------------------------------------------------------------------------
# web slow

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: 10m_response_time
       on: web_log.response_time
    class: Latency
     type: System
component: Web log
 families: *
   lookup: average -10m unaligned of avg
    units: ms
    every: 30s
     info: average HTTP response time over the last 10 minutes

 template: web_slow
       on: web_log.response_time
    class: Latency
     type: Web Server
component: Web log
 families: *
   lookup: average -1m unaligned of avg
    units: ms
    every: 10s
    green: 500
      red: 1000
     warn: ($1m_requests > 120) ? ($this > $green && $this > ($10m_response_time * 2) ) : ( 0 )
     crit: ($1m_requests > 120) ? ($this > $red   && $this > ($10m_response_time * 4) ) : ( 0 )
    delay: down 15m multiplier 1.5 max 1h
     info: average HTTP response time over the last minute
  options: no-clear-notification
       to: webmaster

# -----------------------------------------------------------------------------
# web too many or too few requests

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $5m_successful_old > 120
#
# i.e. when there were at least 120 requests during the 5 minutes starting
#      at -10m and ending at -5m

 template: 5m_successful_old
       on: web_log.response_statuses
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: average -5m at -5m unaligned of successful_requests
    units: requests/s
    every: 30s
     info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago

 template: 5m_successful
       on: web_log.response_statuses
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: average -5m unaligned of successful_requests
    units: requests/s
    every: 30s
     info: average number of successful HTTP requests over the last 5 minutes

 template: 5m_requests_ratio
       on: web_log.response_codes
    class: Workload
     type: Web Server
component: Web log
 families: *
     calc: ($5m_successful_old > 0)?($5m_successful * 100 / $5m_successful_old):(100)
    units: %
    every: 30s
     warn: ($5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
     crit: ($5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
    delay: down 15m multiplier 1.5 max 1h
  options: no-clear-notification
     info: ratio of successful HTTP requests over the last 5 minutes, \
           compared with the previous 5 minutes \
           (clear notification for this alarm will not be sent)
       to: webmaster



# ---------------------------------------------------GO-VERSION---------------------------------------------------------

# unmatched lines

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_total_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: web_log_1m_total_requests
       on: web_log.requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned
     calc: ($this == 0)?(1):($this)
    units: requests
    every: 10s
     info: number of HTTP requests in the last minute

 template: web_log_1m_unmatched
       on: web_log.excluded_requests
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of unmatched
     calc: $this * 100 / $web_log_1m_total_requests
    units: %
    every: 10s
     warn: ($web_log_1m_total_requests > 120) ? ($this > 1) : ( 0 )
    delay: up 1m down 5m multiplier 1.5 max 1h
     info: percentage of unparsed log lines over the last minute
       to: webmaster

# -----------------------------------------------------------------------------
# high level response code alarms

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: web_log_1m_requests
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned
     calc: ($this == 0)?(1):($this)
    units: requests
    every: 10s
     info: number of HTTP requests in the last minute

 template: web_log_1m_successful
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of success
     calc: $this * 100 / $web_log_1m_requests
    units: %
    every: 10s
     warn: ($web_log_1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
     crit: ($web_log_1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of successful HTTP requests over the last minute (1xx, 2xx, 304, 401)
       to: webmaster

 template: web_log_1m_redirects
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of redirect
     calc: $this * 100 / $web_log_1m_requests
    units: %
    every: 10s
     warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING ) ? (  1 ) : ( 20 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of redirection HTTP requests over the last minute (3xx except 304)
       to: webmaster

 template: web_log_1m_bad_requests
       on: web_log.type_requests
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of bad
     calc: $this * 100 / $web_log_1m_requests
    units: %
    every: 10s
     warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 10 ) : ( 30 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of client error HTTP requests over the last minute (4xx except 401)
       to: webmaster

 template: web_log_1m_internal_errors
       on: web_log.type_requests
    class: Errors
     type: Web Server
component: Web log
 families: *
   lookup: sum -1m unaligned of error
     calc: $this * 100 / $web_log_1m_requests
    units: %
    every: 10s
     warn: ($web_log_1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 1 ) : ( 2 )) ) : ( 0 )
     crit: ($web_log_1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
    delay: up 2m down 15m multiplier 1.5 max 1h
     info: ratio of server error HTTP requests over the last minute (5xx)
       to: webmaster

# -----------------------------------------------------------------------------
# web slow

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $1m_requests > 120
#
# i.e. when there are at least 120 requests during the last minute

 template: web_log_10m_response_time
       on: web_log.request_processing_time
    class: Latency
     type: System
component: Web log
 families: *
   lookup: average -10m unaligned of avg
    units: ms
    every: 30s
     info: average HTTP response time over the last 10 minutes

 template: web_log_web_slow
       on: web_log.request_processing_time
    class: Latency
     type: Web Server
component: Web log
 families: *
   lookup: average -1m unaligned of avg
    units: ms
    every: 10s
    green: 500
      red: 1000
     warn: ($web_log_1m_requests > 120) ? ($this > $green && $this > ($web_log_10m_response_time * 2) ) : ( 0 )
     crit: ($web_log_1m_requests > 120) ? ($this > $red   && $this > ($web_log_10m_response_time * 4) ) : ( 0 )
    delay: down 15m multiplier 1.5 max 1h
     info: average HTTP response time over the last 1 minute
  options: no-clear-notification
       to: webmaster

# -----------------------------------------------------------------------------
# web too many or too few requests

# the following alarms trigger only when there are enough data.
# we assume there are enough data when:
#
#  $5m_successful_old > 120
#
# i.e. when there were at least 120 requests during the 5 minutes starting
#      at -10m and ending at -5m

 template: web_log_5m_successful_old
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: average -5m at -5m unaligned of success
    units: requests/s
    every: 30s
     info: average number of successful HTTP requests for the 5 minutes starting 10 minutes ago

 template: web_log_5m_successful
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
   lookup: average -5m unaligned of success
    units: requests/s
    every: 30s
     info: average number of successful HTTP requests over the last 5 minutes

 template: web_log_5m_requests_ratio
       on: web_log.type_requests
    class: Workload
     type: Web Server
component: Web log
 families: *
     calc: ($web_log_5m_successful_old > 0)?($web_log_5m_successful * 100 / $web_log_5m_successful_old):(100)
    units: %
    every: 30s
     warn: ($web_log_5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
     crit: ($web_log_5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
    delay: down 15m multiplier 1.5 max 1h
  options: no-clear-notification
     info: ratio of successful HTTP requests over over the last 5 minutes, \
           compared with the previous 5 minutes \
           (clear notification for this alarm will not be sent)
       to: webmaster