Merge pull request #3782 from l2isbad/litespeed_module

Litespeed module

conf.d/Makefile.am

@@ -49,6 +49,7 @@ dist_pythonconfig_DATA = \
     python.d/icecast.conf \
     python.d/ipfs.conf \
     python.d/isc_dhcpd.conf \
+    python.d/litespeed.conf \
     python.d/mdstat.conf \
     python.d/memcached.conf \
     python.d/mongodb.conf \

conf.d/python.d.conf

@@ -49,6 +49,7 @@ go_expvar: no
 # icecast: yes
 # ipfs: yes
 # isc_dhcpd: yes
+# litespeed: yes
 # mdstat: yes
 # memcached: yes
 # mongodb: yes

conf.d/python.d/litespeed.conf

@@ -0,0 +1,74 @@
+# netdata python.d.plugin configuration for litespeed
+#
+# This file is in YaML format. Generally the format is:
+#
+# name: value
+#
+# There are 2 sections:
+#  - global variables
+#  - one or more JOBS
+#
+# JOBS allow you to collect values from multiple sources.
+# Each source will have its own set of charts.
+#
+# JOB parameters have to be indented (using spaces only, example below).
+
+# ----------------------------------------------------------------------
+# Global Variables
+# These variables set the defaults for all JOBs, however each JOB
+# may define its own, overriding the defaults.
+
+# update_every sets the default data collection frequency.
+# If unset, the python.d.plugin default is used.
+# update_every: 1
+
+# priority controls the order of charts at the netdata dashboard.
+# Lower numbers move the charts towards the top of the page.
+# If unset, the default for python.d.plugin is used.
+# priority: 60000
+
+# retries sets the number of retries to be made in case of failures.
+# If unset, the default for python.d.plugin is used.
+# Attempts to restore the service are made once every update_every
+# and only if the module has collected values in the past.
+# retries: 60
+
+# autodetection_retry sets the job re-check interval in seconds.
+# The job is not deleted if check fails.
+# Attempts to start the job are made once every autodetection_retry.
+# This feature is disabled by default.
+# autodetection_retry: 0
+
+# ----------------------------------------------------------------------
+# JOBS (data collection sources)
+#
+# The default JOBS share the same *name*. JOBS with the same name
+# are mutually exclusive. Only one of them will be allowed running at
+# any time. This allows autodetection to try several alternatives and
+# pick the one that works.
+#
+# Any number of jobs is supported.
+#
+# All python.d.plugin JOBS (for all its modules) support a set of
+# predefined parameters. These are:
+#
+# job_name:
+#     name: myname            # the JOB's name as it will appear at the
+#                             # dashboard (by default is the job_name)
+#                             # JOBs sharing a name are mutually exclusive
+#     update_every: 1         # the JOB's data collection frequency
+#     priority: 60000         # the JOB's order on the dashboard
+#     retries: 60             # the JOB's number of restoration attempts
+#     autodetection_retry: 0  # the JOB's re-check interval in seconds
+#
+# Additionally to the above, lightspeed also supports the following:
+#
+#     path: 'PATH'       # path to lightspeed stats files directory
+#
+# ----------------------------------------------------------------------
+# AUTO-DETECTION JOBS
+# only one of them will run (they have the same name)
+
+localhost:
+  name : 'local'
+  path  : '/tmp/lshttpd/'

installer/functions.sh

@@ -824,7 +824,7 @@ download_netdata_conf() {
 # -----------------------------------------------------------------------------
 # add netdata user and group
 
-NETDATA_WANTED_GROUPS="docker nginx varnish haproxy adm nsd proxy squid ceph"
+NETDATA_WANTED_GROUPS="docker nginx varnish haproxy adm nsd proxy squid ceph nobody"
 NETDATA_ADDED_TO_GROUPS=""
 add_netdata_user_and_group() {
     local homedir="${1}" g

python.d/Makefile.am

@@ -37,6 +37,7 @@ dist_python_DATA = \
     icecast.chart.py \
     ipfs.chart.py \
     isc_dhcpd.chart.py \
+    litespeed.chart.py \
     mdstat.chart.py \
     memcached.chart.py \
     mongodb.chart.py \

python.d/README.md

@@ -982,6 +982,53 @@ The module will not work If no configuration is given.
 
 ---
 
+# litespeed
+
+Module monitor litespeed web server performance metrics.
+
+It produces:
+
+1. **Network Throughput HTTP** in kilobits/s
+ * in
+ * out
+
+2. **Network Throughput HTTPS** in kilobits/s
+ * in
+ * out
+
+3. **Connections HTTP** in connections
+ * free
+ * used
+
+4. **Connections HTTPS** in connections
+ * free
+ * used
+
+5. **Requests** in requests/s
+ * requests
+
+6. **Requests In Processing** in requests
+ * processing
+ 
+7. **Public Cache Hits** in hits/s
+ * hits
+ 
+8. **Private Cache Hits** in hits/s
+ * hits
+ 
+9. **Static Hits** in hits/s
+ * hits 
+
+
+### configuration
+```yaml
+local:
+  path  : 'PATH'
+```
+
+If no configuration is given, module will use "/tmp/lshttpd/".
+
+---
 
 # mdstat
 

python.d/litespeed.chart.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Description: litespeed netdata python.d module
+# Author: Ilya Maschenko (l2isbad)
+
+import glob
+import re
+import os
+
+from collections import namedtuple
+
+from bases.FrameworkServices.SimpleService import SimpleService
+
+
+update_every = 10
+
+# charts order (can be overridden if you want less charts, or different order)
+ORDER = [
+    'net_throughput_http', 'net_throughput_https',  # net throughput
+    'connections_http', 'connections_https',        # connections
+    'requests', 'requests_processing',              # requests
+    'pub_cache_hits', 'private_cache_hits',         # cache
+    'static_hits'                                   # static
+]
+
+CHARTS = {
+    'net_throughput_http': {
+        'options': [
+            None, 'Network Throughput HTTP', 'kilobits/s', 'net throughput', 'litespeed.net_throughput', 'area'],
+        'lines': [
+            ["bps_in", "in", "absolute"],
+            ["bps_out", "out", "absolute", -1]
+        ]},
+    'net_throughput_https': {
+        'options': [
+            None, 'Network Throughput HTTPS', 'kilobits/s', 'net throughput', 'litespeed.net_throughput', 'area'],
+        'lines': [
+            ["ssl_bps_in", "in", "absolute"],
+            ["ssl_bps_out", "out", "absolute", -1]
+        ]},
+    'connections_http': {
+        'options': [
+            None, 'Connections HTTP', 'conns', 'connections', 'litespeed.connections', 'stacked'],
+        'lines': [
+            ["conn_free", "free", "absolute"],
+            ["conn_used", "used", "absolute"]
+        ]},
+    'connections_https': {
+        'options': [
+            None, 'Connections HTTPS', 'conns', 'connections', 'litespeed.connections', 'stacked'],
+        'lines': [
+            ["ssl_conn_free", "free", "absolute"],
+            ["ssl_conn_used", "used", "absolute"]
+        ]},
+    'requests': {
+        'options': [None, 'Requests', 'requests/s', 'requests', 'litespeed.requests', 'line'],
+        'lines': [
+            ["requests", None, "absolute", 1, 100]
+        ]},
+    'requests_processing': {
+        'options': [None, 'Requests In Processing', 'requests', 'requests', 'litespeed.requests_processing', 'line'],
+        'lines': [
+            ["requests_processing", "processing", "absolute"]
+        ]},
+    'pub_cache_hits': {
+        'options': [None, 'Public Cache Hits', 'hits/s', 'cache', 'litespeed.cache', 'line'],
+        'lines': [
+            ["pub_cache_hits", "hits", "absolute", 1, 100]
+        ]},
+    'private_cache_hits': {
+        'options': [None, 'Private Cache Hits', 'hits/s', 'cache', 'litespeed.cache', 'line'],
+        'lines': [
+            ["private_cache_hits", "hits", "absolute", 1, 100]
+        ]},
+    'static_hits': {
+        'options': [None, 'Static Hits', 'hits/s', 'static', 'litespeed.static', 'line'],
+        'lines': [
+            ["static_hits", "hits", "absolute", 1, 100]
+        ]},
+}
+
+t = namedtuple("T", ["key", "id", "mul"])
+
+T = [
+    t("BPS_IN", "bps_in", 8),
+    t("BPS_OUT", "bps_out", 8),
+    t("SSL_BPS_IN", "ssl_bps_in", 8),
+    t("SSL_BPS_OUT", "ssl_bps_out", 8),
+    t("REQ_PER_SEC", "requests", 100),
+    t("REQ_PROCESSING", "requests_processing", 1),
+    t("PUB_CACHE_HITS_PER_SEC", "pub_cache_hits", 100),
+    t("PRIVATE_CACHE_HITS_PER_SEC", "private_cache_hits", 100),
+    t("STATIC_HITS_PER_SEC", "static_hits", 100),
+    t("PLAINCONN", "conn_used", 1),
+    t("AVAILCONN", "conn_free", 1),
+    t("SSLCONN", "ssl_conn_used", 1),
+    t("AVAILSSL", "ssl_conn_free", 1),
+]
+
+RE = re.compile(r'([A-Z_]+): ([0-9.]+)')
+
+ZERO_DATA = {
+    "bps_in": 0,
+    "bps_out": 0,
+    "ssl_bps_in": 0,
+    "ssl_bps_out": 0,
+    "requests": 0,
+    "requests_processing": 0,
+    "pub_cache_hits": 0,
+    "private_cache_hits": 0,
+    "static_hits": 0,
+    "conn_used": 0,
+    "conn_free": 0,
+    "ssl_conn_used": 0,
+    "ssl_conn_free": 0,
+    }
+
+
+class Service(SimpleService):
+    def __init__(self, configuration=None, name=None):
+        SimpleService.__init__(self, configuration=configuration, name=name)
+        self.order = ORDER
+        self.definitions = CHARTS
+        self.path = self.configuration.get('path', "/tmp/lshttpd/")
+        self.files = list()
+
+    def check(self):
+        if not self.path:
+            self.error('"path" not specified')
+            return False
+
+        fs = glob.glob(os.path.join(self.path, ".rtreport*"))
+
+        if not fs:
+            self.error('"{0}" has no "rtreport" files or dir is not readable'.format(self.path))
+            return None
+
+        self.debug("stats files:", fs)
+
+        for f in fs:
+            if not is_readable_file(f):
+                self.error("{0} is not readable".format(f))
+                continue
+            self.files.append(f)
+
+        return bool(self.files)
+
+    def get_data(self):
+        """
+        Format data received from http request
+        :return: dict
+        """
+        data = dict(ZERO_DATA)
+
+        for f in self.files:
+            try:
+                with open(f) as b:
+                    lines = b.readlines()
+            except (OSError, IOError) as err:
+                self.error(err)
+                return None
+            else:
+                parse_file(data, lines)
+
+        return data
+
+
+def parse_file(data, lines):
+    for line in lines:
+        if not line.startswith(("BPS_IN:", "MAXCONN:", "REQ_RATE []:")):
+            continue
+        m = dict(RE.findall(line))
+        for v in T:
+            if v.key in m:
+                data[v.id] += float(m[v.key]) * v.mul
+
+
+def is_readable_file(v):
+    return os.path.isfile(v) and os.access(v, os.R_OK)