Merge pull request #3832 from l2isbad/redis_pika

redis: refactor + pika support

python.d/python_modules/bases/FrameworkServices/SocketService.py

@@ -172,7 +172,7 @@ class SocketService(SimpleService):
                 pass
             self._sock = None
 
-    def _send(self):
+    def _send(self, request=None):
         """
         Send request.
         :return: boolean
@@ -180,8 +180,8 @@ class SocketService(SimpleService):
         # Send request if it is needed
         if self.request != self.__empty_request:
             try:
-                self.debug('sending request: {0}'.format(self.request))
-                self._sock.send(self.request)
+                self.debug('sending request: {0}'.format(request or self.request))
+                self._sock.send(request or self.request)
             except Exception as error:
                 self._socket_error('error sending request: {0}'.format(error))
                 self._disconnect()
@@ -222,7 +222,7 @@ class SocketService(SimpleService):
         self.debug('final response: {0}'.format(data))
         return data
 
-    def _get_raw_data(self, raw=False):
+    def _get_raw_data(self, raw=False, request=None):
         """
         Get raw data with low-level "socket" module.
         :param raw: set `True` to return bytes
@@ -236,7 +236,7 @@ class SocketService(SimpleService):
                 return None
 
         # Send request if it is needed
-        if not self._send():
+        if not self._send(request):
             return None
 
         data = self._receive(raw)

python.d/redis.chart.py

@@ -1,75 +1,105 @@
 # -*- coding: utf-8 -*-
 # Description: redis netdata python.d module
 # Author: Pawel Krupa (paulfantom)
+# Author: Ilya Mashchenko (l2isbad)
 # SPDX-License-Identifier: GPL-3.0+
 
-from bases.FrameworkServices.SocketService import SocketService
+import re
+
+from copy import deepcopy
 
-# default module values (can be overridden per job in `config`)
-priority = 60000
-retries = 60
+from bases.FrameworkServices.SocketService import SocketService
 
-# default job configuration (overridden by python.d.plugin)
-# config = {'local': {
-#             'update_every': update_every,
-#             'retries': retries,
-#             'priority': priority,
-#             'host': 'localhost',
-#             'port': 6379,
-#             'unix_socket': None
-#          }}
+REDIS_ORDER = [
+    'operations',
+    'hit_rate',
+    'memory',
+    'keys_redis',
+    'eviction',
+    'net',
+    'connections',
+    'clients',
+    'slaves',
+    'persistence',
+    'bgsave_now',
+    'bgsave_health',
+    'uptime',
+]
+
+PIKA_ORDER = [
+    'operations',
+    'hit_rate',
+    'memory',
+    'keys_pika',
+    'connections',
+    'clients',
+    'slaves',
+    'uptime',
+]
 
-ORDER = ['operations', 'hit_rate', 'memory', 'keys', 'net', 'connections', 'clients', 'slaves', 'persistence',
-         'bgsave_now', 'bgsave_health']
 
 CHARTS = {
     'operations': {
-        'options': [None, 'Redis Operations', 'operations/s', 'operations', 'redis.operations', 'line'],
+        'options': [None, 'Operations', 'operations/s', 'operations', 'redis.operations', 'line'],
         'lines': [
             ['total_commands_processed', 'commands', 'incremental'],
             ['instantaneous_ops_per_sec', 'operations', 'absolute']
         ]},
     'hit_rate': {
-        'options': [None, 'Redis Hit rate', 'percent', 'hits', 'redis.hit_rate', 'line'],
+        'options': [None, 'Hit rate', 'percent', 'hits', 'redis.hit_rate', 'line'],
         'lines': [
             ['hit_rate', 'rate', 'absolute']
         ]},
     'memory': {
-        'options': [None, 'Redis Memory utilization', 'kilobytes', 'memory', 'redis.memory', 'line'],
+        'options': [None, 'Memory utilization', 'kilobytes', 'memory', 'redis.memory', 'line'],
         'lines': [
             ['used_memory', 'total', 'absolute', 1, 1024],
             ['used_memory_lua', 'lua', 'absolute', 1, 1024]
         ]},
     'net': {
-        'options': [None, 'Redis Bandwidth', 'kilobits/s', 'network', 'redis.net', 'area'],
+        'options': [None, 'Bandwidth', 'kilobits/s', 'network', 'redis.net', 'area'],
         'lines': [
             ['total_net_input_bytes', 'in', 'incremental', 8, 1024],
             ['total_net_output_bytes', 'out', 'incremental', -8, 1024]
         ]},
-    'keys': {
-        'options': [None, 'Redis Keys per Database', 'keys', 'keys', 'redis.keys', 'line'],
+    'keys_redis': {
+        'options': [None, 'Keys per Database', 'keys', 'keys', 'redis.keys', 'line'],
         'lines': [
             # lines are created dynamically in `check()` method
         ]},
+    'keys_pika': {
+        'options': [None, 'Keys', 'keys', 'keys', 'redis.keys', 'line'],
+        'lines': [
+            ['kv_keys', 'kv', 'absolute'],
+            ['hash_keys', 'hash', 'absolute'],
+            ['list_keys', 'list', 'absolute'],
+            ['zset_keys', 'zset', 'absolute'],
+            ['set_keys', 'set', 'absolute']
+        ]},
+    'eviction': {
+        'options': [None, 'Evicted Keys', 'keys', 'keys', 'redis.eviction', 'line'],
+        'lines': [
+            ['evicted_keys', 'evicted', 'absolute']
+        ]},
     'connections': {
-        'options': [None, 'Redis Connections', 'connections/s', 'connections', 'redis.connections', 'line'],
+        'options': [None, 'Connections', 'connections/s', 'connections', 'redis.connections', 'line'],
         'lines': [
             ['total_connections_received', 'received', 'incremental', 1],
             ['rejected_connections', 'rejected', 'incremental', -1]
         ]},
     'clients': {
-        'options': [None, 'Redis Clients', 'clients', 'connections', 'redis.clients', 'line'],
+        'options': [None, 'Clients', 'clients', 'connections', 'redis.clients', 'line'],
         'lines': [
             ['connected_clients', 'connected', 'absolute', 1],
             ['blocked_clients', 'blocked', 'absolute', -1]
         ]},
     'slaves': {
-        'options': [None, 'Redis Slaves', 'slaves', 'replication', 'redis.slaves', 'line'],
+        'options': [None, 'Slaves', 'slaves', 'replication', 'redis.slaves', 'line'],
         'lines': [
             ['connected_slaves', 'connected', 'absolute']
         ]},
     'persistence': {
-        'options': [None, 'Redis Persistence Changes Since Last Save', 'changes', 'persistence',
+        'options': [None, 'Persistence Changes Since Last Save', 'changes', 'persistence',
                     'redis.rdb_changes', 'line'],
         'lines': [
             ['rdb_changes_since_last_save', 'changes', 'absolute']
@@ -85,85 +115,117 @@ CHARTS = {
                     'redis.bgsave_health', 'line'],
         'lines': [
             ['rdb_last_bgsave_status', 'rdb save', 'absolute']
+        ]},
+    'uptime': {
+        'options': [None, 'Uptime', 'seconds', 'uptime', 'redis.uptime', 'line'],
+        'lines': [
+            ['uptime_in_seconds', 'uptime', 'absolute']
         ]}
 }
 
 
+def copy_chart(name):
+    return {name: deepcopy(CHARTS[name])}
+
+
+RE = re.compile(r'\n([a-z_0-9 ]+):(?:keys=)?([^,\r]+)')
+
+
 class Service(SocketService):
     def __init__(self, configuration=None, name=None):
         SocketService.__init__(self, configuration=configuration, name=name)
-        self.order = ORDER
-        self.definitions = CHARTS
         self._keep_alive = True
-        self.chart_name = ""
+
+        self.order = list()
+        self.definitions = dict()
+
         self.host = self.configuration.get('host', 'localhost')
         self.port = self.configuration.get('port', 6379)
         self.unix_socket = self.configuration.get('socket')
-        password = self.configuration.get('pass', str())
+        p = self.configuration.get('pass')
+
+        self.auth_request = 'AUTH {0} \r\n'.format(p).encode() if p else None
+        self.request = 'INFO\r\n'.encode()
         self.bgsave_time = 0
-        self.requests = dict(request='INFO\r\n'.encode(),
-                             password=' '.join(['AUTH', password, '\r\n']).encode() if password else None)
-        self.request = self.requests['request']
 
-    def _get_data(self):
-        """
-        Get data from socket
-        :return: dict
-        """
-        if self.requests['password']:
-            self.request = self.requests['password']
-            raw = self._get_raw_data().strip()
-            if raw != "+OK":
-                self.error("invalid password")
-                return None
-            self.request = self.requests['request']
-        response = self._get_raw_data()
-        if response is None:
-            # error has already been logged
+    def do_auth(self):
+        resp = self._get_raw_data(request=self.auth_request)
+        if not resp:
+            return False
+        if not resp.strip() != '+OK':
+            self.error("invalid password")
+            return False
+        return True
+
+    def get_raw_and_parse(self):
+        if self.auth_request and not self.do_auth():
             return None
 
-        try:
-            parsed = response.split("\n")
-        except AttributeError:
-            self.error("response is invalid/empty")
+        resp = self._get_raw_data()
+
+        if not resp:
             return None
 
-        data = dict()
-        for line in parsed:
-            if len(line) < 5 or line[0] == '$' or line[0] == '#':
-                continue
+        parsed = RE.findall(resp)
+
+        if not parsed:
+            self.error("response is invalid/empty")
+            return None
 
-            if line.startswith('db'):
-                tmp = line.split(',')[0].replace('keys=', '')
-                record = tmp.split(':')
-                data[record[0]] = record[1]
-                continue
+        return dict((k.replace(' ', '_'), v) for k, v in parsed)
 
-            try:
-                t = line.split(':')
-                data[t[0]] = t[1]
-            except (IndexError, ValueError):
-                self.debug("invalid line received: " + str(line))
+    def get_data(self):
+        """
+        Get data from socket
+        :return: dict
+        """
+        data = self.get_raw_and_parse()
 
         if not data:
-            self.error("received data doesn't have any records")
             return None
 
         try:
-            data['hit_rate'] = (int(data['keyspace_hits']) * 100) / (int(data['keyspace_hits'])
-                                                                     + int(data['keyspace_misses']))
-        except (KeyError, ZeroDivisionError, TypeError):
+            data['hit_rate'] = (
+                    (int(data['keyspace_hits']) * 100) / (int(data['keyspace_hits']) + int(data['keyspace_misses']))
+            )
+        except (KeyError, ZeroDivisionError):
             data['hit_rate'] = 0
 
-        if data['rdb_bgsave_in_progress'] != '0\r':
+        if data.get('redis_version'):
+            self.get_data_redis_specific(data)
+
+        return data
+
+    def get_data_redis_specific(self, data):
+        if data['rdb_bgsave_in_progress'] != '0':
             self.bgsave_time += self.update_every
         else:
             self.bgsave_time = 0
 
-        data['rdb_last_bgsave_status'] = 0 if data['rdb_last_bgsave_status'] == 'ok\r' else 1
+        data['rdb_last_bgsave_status'] = 0 if data['rdb_last_bgsave_status'] == 'ok' else 1
         data['rdb_bgsave_in_progress'] = self.bgsave_time
 
-        return data
+    def check(self):
+        """
+        Parse configuration, check if redis is available, and dynamically create chart lines data
+        :return: boolean
+        """
+        data = self.get_raw_and_parse()
+
+        if not data:
+            return False
+
+        self.order = PIKA_ORDER if data.get("pika_version") else REDIS_ORDER
+
+        for n in self.order:
+            self.definitions.update(copy_chart(n))
+
+        if data.get("redis_version"):
+            for k in data:
+                if k.startswith('db'):
+                    self.definitions['keys_redis']['lines'].append([k, None, 'absolute'])
+
+        return True
 
     def _check_raw_data(self, data):
         """
@@ -185,17 +247,3 @@ class Service(SocketService):
 
         self.debug("waiting more data from redis")
         return False
-
-    def check(self):
-        """
-        Parse configuration, check if redis is available, and dynamically create chart lines data
-        :return: boolean
-        """
-        data = self._get_data()
-        if data is None:
-            return False
-
-        for name in data:
-            if name.startswith('db'):
-                self.definitions['keys']['lines'].append([name, None, 'absolute'])
-        return True