netdata/logsmanagement/parser.h

// SPDX-License-Identifier: GPL-3.0-or-later

/** @file parser.h
 *  @brief Header of parser.c 
 */

#ifndef PARSER_H_
#define PARSER_H_

#include <regex.h> 
#include "daemon/common.h"
#include "libnetdata/libnetdata.h"

// Forward decleration
typedef struct log_parser_metrics Log_parser_metrics_t;


/* -------------------------------------------------------------------------- */
/*                           Configuration-related                            */
/* -------------------------------------------------------------------------- */

typedef enum{

    CHART_COLLECTED_LOGS_TOTAL =    1 << 0,
    CHART_COLLECTED_LOGS_RATE =     1 << 1,

    /* FLB_WEB_LOG charts */
    CHART_VHOST =                   1 << 2,            
    CHART_PORT =                    1 << 3,             
    CHART_IP_VERSION =              1 << 4,
    CHART_REQ_CLIENT_CURRENT =      1 << 5,
    CHART_REQ_CLIENT_ALL_TIME =     1 << 6,
    CHART_REQ_METHODS =             1 << 7,
    CHART_REQ_PROTO =               1 << 8,
    CHART_BANDWIDTH =               1 << 9,
    CHART_REQ_PROC_TIME =           1 << 10,
    CHART_RESP_CODE_FAMILY =        1 << 11,
    CHART_RESP_CODE =               1 << 12,
    CHART_RESP_CODE_TYPE =          1 << 13,
    CHART_SSL_PROTO =               1 << 14,
    CHART_SSL_CIPHER =              1 << 15,

    /* FLB_SYSTEMD or FLB_SYSLOG charts */
    CHART_SYSLOG_PRIOR =            1 << 16,
    CHART_SYSLOG_SEVER =            1 << 17,
    CHART_SYSLOG_FACIL =            1 << 18,

    /* FLB_KMSG charts */
    CHART_KMSG_SUBSYSTEM =          1 << 19,
    CHART_KMSG_DEVICE =             1 << 20,

    /* FLB_DOCKER_EV charts */
    CHART_DOCKER_EV_TYPE =          1 << 21,
    CHART_DOCKER_EV_ACTION =        1 << 22,

    /* FLB_MQTT charts*/
    CHART_MQTT_TOPIC =              1 << 23

} chart_type_t;

typedef struct log_parser_config{
    void *gen_config;					/**< Pointer to (optional) generic configuration, as per use case. */
    unsigned long int chart_config;		/**< Configuration of which charts to enable according to chart_type_t **/
} Log_parser_config_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                        Web Log parsing and metrics                         */
/* -------------------------------------------------------------------------- */

#define VHOST_MAX_LEN 255               /**< Max vhost string length, inclding terminating \0 **/
#define PORT_MAX_LEN 6			        /**< Max port string length, inclding terminating \0 **/
#define REQ_SCHEME_MAX_LEN 6			/**< Max request scheme length, including terminating \0 **/
#define REQ_CLIENT_MAX_LEN 46           /**< https://superuser.com/questions/381022/how-many-characters-can-an-ip-address-be#comment2219013_381029 **/
#define REQ_METHOD_MAX_LEN 18           /**< Max request method length, including terminating \0 **/
#define REQ_URL_MAX_LEN 128             /**< Max request URL length, including terminating \0 **/
#define REQ_PROTO_PREF_SIZE (sizeof("HTTP/") - 1)
#define REQ_PROTO_MAX_LEN 4             /**< Max request protocol numerical part length, including terminating \0 **/
#define REQ_SIZE_MAX_LEN 11             /**< Max size of bytes received, including terminating \0 **/
#define REQ_PROC_TIME_MAX_LEN 11        /**< Max size of request processing time, including terminating \0 **/
#define REQ_RESP_CODE_MAX_LEN 4         /**< Max size of response code, including terminating \0 **/
#define REQ_RESP_SIZE_MAX_LEN 11        /**< Max size of request response size, including terminating \0 **/
#define UPS_RESP_TIME_MAX_LEN 10        /**< Max size of upstream response time, including terminating \0 **/ 
#define SSL_PROTO_MAX_LEN 8             /**< Max SSL protocol length, inclding terminating \0 **/
#define SSL_CIPHER_SUITE_MAX_LEN 256    /**< TODO: Check max len for ssl cipher suite string is indeed 256 **/

#define RESP_CODE_ARR_SIZE 501          /**< Size of resp_code array, assuming 500 valid resp codes + 1 for "other" **/

#define WEB_LOG_INVALID_HOST_STR "invalid"
#define WEB_LOG_INVALID_PORT -1
#define WEB_LOG_INVALID_PORT_STR "inv"
#define WEB_LOG_INVALID_CLIENT_IP_STR WEB_LOG_INVALID_PORT_STR

/* Web log configuration */
#define ENABLE_PARSE_WEB_LOG_LINE_DEBUG 0

#define VHOST_BUFFS_SCALE_FACTOR 1.5
#define PORT_BUFFS_SCALE_FACTOR 8       // Unlike Vhosts, ports are stored as integers, so scale factor can be bigger


typedef enum{
    VHOST_WITH_PORT,  // nginx: $host:$server_port      apache: %v:%p
    VHOST, 		      // nginx: $host ($http_host)      apache: %v
    PORT,             // nginx: $server_port            apache: %p
    REQ_SCHEME,       // nginx: $scheme                 apache: -
    REQ_CLIENT,       // nginx: $remote_addr            apache: %a (%h)
    REQ,			  // nginx: $request                apache: %r
    REQ_METHOD,       // nginx: $request_method         apache: %m
    REQ_URL,          // nginx: $request_uri            apache: %U
    REQ_PROTO,        // nginx: $server_protocol        apache: %H
    REQ_SIZE,         // nginx: $request_length         apache: %I
    REQ_PROC_TIME,    // nginx: $request_time           apache: %D  
    RESP_CODE,        // nginx: $status                 apache: %s, %>s
    RESP_SIZE,        // nginx: $bytes_sent, $body_bytes_sent apache: %b, %O, %B // TODO: Should separate %b from %O ?
    UPS_RESP_TIME,    // nginx: $upstream_response_time apache: -
    SSL_PROTO,        // nginx: $ssl_protocol           apache: -
    SSL_CIPHER_SUITE, // nginx: $ssl_cipher             apache: -
    TIME,             // nginx: $time_local             apache: %t
    CUSTOM
} web_log_line_field_t;

typedef struct web_log_parser_config{
    web_log_line_field_t *fields;  
    int num_fields;             		/**< Number of strings in the fields array. **/
    char delimiter;       				/**< Delimiter that separates the fields in the log format. **/
    int verify_parsed_logs;				/**< Boolean whether to try and verify parsed log fields or not **/
    int skip_timestamp_parsing;         /**< Boolean whether to skip parsing of timestamp fields **/
} Web_log_parser_config_t;

static const char *const req_method_str[] = {
    "ACL",
    "BASELINE-CONTROL",
    "BIND",
    "CHECKIN",
    "CHECKOUT",
    "CONNECT",
    "COPY",
    "DELETE",
    "GET",
    "HEAD",
    "LABEL",
    "LINK",
    "LOCK",
    "MERGE",
    "MKACTIVITY",
    "MKCALENDAR",
    "MKCOL",
    "MKREDIRECTREF",
    "MKWORKSPACE",
    "MOVE",
    "OPTIONS",
    "ORDERPATCH",
    "PATCH",
    "POST",
    "PRI",
    "PROPFIND",
    "PROPPATCH",
    "PUT",
    "REBIND",
    "REPORT",
    "SEARCH",
    "TRACE",
    "UNBIND",
    "UNCHECKOUT",
    "UNLINK",
    "UNLOCK",
    "UPDATE",
    "UPDATEREDIRECTREF",
    "-"
};

#define REQ_METHOD_ARR_SIZE (int)(sizeof(req_method_str) / sizeof(req_method_str[0]))

typedef struct web_log_metrics{
    /* Web log metrics */
    struct log_parser_metrics_vhosts_array{
        struct log_parser_metrics_vhost{
            char name[VHOST_MAX_LEN];   /**< Name of the vhost **/
            int count;					/**< Occurences of the vhost **/
        } *vhosts;
        int size;						/**< Size of vhosts array **/
        int size_max;
    } vhost_arr;
    struct log_parser_metrics_ports_array{
        struct log_parser_metrics_port{
            char name[PORT_MAX_LEN];    /**< Number of port in str */
            int port;   				/**< Number of port **/
            int count;					/**< Occurences of the port **/
        } *ports;
        int size;						/**< Size of ports array **/
        int size_max;
    } port_arr;
    struct log_parser_metrics_ip_ver{
        int v4, v6, invalid;
    } ip_ver;
    /**< req_clients_current_arr is used by parser.c to save unique client IPs 
     * extracted per circular buffer item and also in p_file_info to save unique 
     * client IPs per collection (poll) iteration of plugin_logsmanagement.c. 
     * req_clients_alltime_arr is used in p_file_info to save unique client IPs 
     * of all time (and so ipv4_size and ipv6_size can only grow and are never reset to 0). **/
    struct log_parser_metrics_req_clients_array{
        char (*ipv4_req_clients)[REQ_CLIENT_MAX_LEN];
        int ipv4_size;						   		 
        int ipv4_size_max;
        char (*ipv6_req_clients)[REQ_CLIENT_MAX_LEN];
        int ipv6_size;						   		 
        int ipv6_size_max;
    } req_clients_current_arr, req_clients_alltime_arr; 
    int req_method[REQ_METHOD_ARR_SIZE]; 
    struct log_parser_metrics_req_proto{
        int http_1, http_1_1, http_2, other;
    } req_proto;
    struct log_parser_metrics_bandwidth{
        long long req_size, resp_size;
    } bandwidth;
    struct log_parser_metrics_req_proc_time{
        int min, max, sum, count;
    } req_proc_time;
    struct log_parser_metrics_resp_code_family{
        int resp_1xx, resp_2xx, resp_3xx, resp_4xx, resp_5xx, other; // TODO: Can there be "other"?
    } resp_code_family; 
    /**< Array counting occurences of response codes. Each item represents the 
     * respective response code by adding 100 to its index, e.g. resp_code[102] 
     * counts how many 202 codes were detected. 501st item represents "other" */  
    unsigned int resp_code[RESP_CODE_ARR_SIZE]; 
    struct log_parser_metrics_resp_code_type{ /* Note: 304 and 401 should be treated as resp_success */
        int resp_success, resp_redirect, resp_bad, resp_error, other; // TODO: Can there be "other"?
    } resp_code_type;
    struct log_parser_metrics_ssl_proto{
        int tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, sslv2, sslv3, other;
    } ssl_proto;
    struct log_parser_metrics_ssl_cipher_array{
        struct log_parser_metrics_ssl_cipher{
            char name[SSL_CIPHER_SUITE_MAX_LEN];    /**< SSL cipher suite string **/
            int count;								/**< Occurences of the SSL cipher **/
        } *ssl_ciphers;
        int size;									/**< Size of SSL ciphers array **/
    } ssl_cipher_arr;
    int64_t timestamp;
} Web_log_metrics_t;

typedef struct log_line_parsed{
    char vhost[VHOST_MAX_LEN];
    int  port;
    char req_scheme[REQ_SCHEME_MAX_LEN];
    char req_client[REQ_CLIENT_MAX_LEN];
    char req_method[REQ_METHOD_MAX_LEN];
    char req_URL[REQ_URL_MAX_LEN];
    char req_proto[REQ_PROTO_MAX_LEN];
    int req_size;
    int req_proc_time;
    int resp_code;
    int resp_size;
    int ups_resp_time;
    char ssl_proto[SSL_PROTO_MAX_LEN];
    char ssl_cipher[SSL_CIPHER_SUITE_MAX_LEN];
    int64_t timestamp;
    int parsing_errors;
} Log_line_parsed_t;

Web_log_parser_config_t *read_web_log_parser_config(const char *log_format, const char delimiter);
#ifdef ENABLE_LOGSMANAGEMENT_TESTS
/* Used as public only for unit testing, normally defined as static */
int count_fields(const char *line, const char delimiter);
#endif // ENABLE_LOGSMANAGEMENT_TESTS
void parse_web_log_line(const Web_log_parser_config_t *wblp_config, 
                        char *line, const size_t line_len, 
                        Log_line_parsed_t *log_line_parsed);
void extract_web_log_metrics(Log_parser_config_t *parser_config, 
                            Log_line_parsed_t *line_parsed, 
                            Web_log_metrics_t *metrics);
Web_log_parser_config_t *auto_detect_web_log_parser_config(char *line, const char delimiter);

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                       Kernel logs (kmsg) metrics                           */
/* -------------------------------------------------------------------------- */

#define SYSLOG_SEVER_ARR_SIZE 9         /**< Number of severity levels plus 1 for 'unknown' **/

typedef struct metrics_dict_item{
    bool dim_initialized;
    int num;
    int num_new;
} metrics_dict_item_t;

typedef struct kernel_metrics{
    unsigned int sever[SYSLOG_SEVER_ARR_SIZE];      /**< Syslog severity, 0-7 plus 1 space for 'unknown' **/
    DICTIONARY *subsystem;
    DICTIONARY *device;
} Kernel_metrics_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                        Systemd and Syslog metrics                          */
/* -------------------------------------------------------------------------- */

#define SYSLOG_FACIL_ARR_SIZE 25        /**< Number of facility levels plus 1 for 'unknown' **/
#define SYSLOG_PRIOR_ARR_SIZE 193       /**< Number of priority values plus 1 for 'unknown' **/

typedef struct systemd_metrics{
    unsigned int sever[SYSLOG_SEVER_ARR_SIZE];      /**< Syslog severity, 0-7 plus 1 space for 'unknown' **/
    unsigned int facil[SYSLOG_FACIL_ARR_SIZE];      /**< Syslog facility, 0-23 plus 1 space for 'unknown' **/
    unsigned int prior[SYSLOG_PRIOR_ARR_SIZE];      /**< Syslog priority value, 0-191 plus 1 space for 'unknown' **/
} Systemd_metrics_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                          Docker Events metrics                             */
/* -------------------------------------------------------------------------- */

static const char *const docker_ev_type_string[] = {
    "container", "image", "plugin", "volume", "network", "daemon", "service", "node", "secret", "config", "unknown"
};

#define NUM_OF_DOCKER_EV_TYPES ((int) (sizeof docker_ev_type_string / sizeof docker_ev_type_string[0]))

#define NUM_OF_CONTAINER_ACTIONS 25 /**< == size of 'Containers actions' array, largest array in docker_ev_action_string **/

static const char *const docker_ev_action_string[NUM_OF_DOCKER_EV_TYPES][NUM_OF_CONTAINER_ACTIONS] = {
    /* Order of arrays is important, it must match the order of docker_ev_type_string[] strings. */

	/* Containers actions */
	{"attach", "commit", "copy", "create", "destroy", "detach", "die", "exec_create", "exec_detach", "exec_die", 
	"exec_start", "export", "health_status", "kill", "oom", "pause", "rename", "resize", "restart", "start", "stop", 
	"top", "unpause", "update", NULL},

	/* Images actions */
	{"delete", "import", "load", "pull", "push", "save", "tag", "untag", NULL},

	/* Plugins actions */
	{"enable", "disable", "install", "remove", NULL},

	/* Volumes actions */
	{"create", "destroy", "mount", "unmount", NULL},

	/* Networks actions */
	{"create", "connect", "destroy", "disconnect", "remove", NULL},

	/* Daemons actions */
	{"reload", NULL},

	/* Services actions */
	{"create", "remove", "update", NULL},

	/* Nodes actions */
	{"create", "remove", "update", NULL},

	/* Secrets actions */
	{"create", "remove", "update", NULL},

	/* Configs actions */
	{"create", "remove", "update", NULL},

    {"unknown", NULL}
};

typedef struct docker_ev_metrics{
    unsigned int ev_type[NUM_OF_DOCKER_EV_TYPES];
    unsigned int ev_action[NUM_OF_DOCKER_EV_TYPES][NUM_OF_CONTAINER_ACTIONS];
} Docker_ev_metrics_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                              MQTT metrics                                  */
/* -------------------------------------------------------------------------- */

typedef struct mqtt_metrics{
    DICTIONARY *topic;
} Mqtt_metrics_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                         Regex / Keyword search                             */
/* -------------------------------------------------------------------------- */

#define MAX_KEYWORD_LEN 100 /**< Max size of keyword used in keyword search, in bytes */
#define MAX_REGEX_SIZE MAX_KEYWORD_LEN + 7 /**< Max size of regular expression (used in keyword search) in bytes **/

int search_keyword(	char *src, size_t src_sz, 
                    char *dest, size_t *dest_sz, 
                    const char *keyword, regex_t *regex, 
                    const int ignore_case);

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                   Custom Charts configuration and metrics                  */
/* -------------------------------------------------------------------------- */

typedef struct log_parser_cus_config{
    char *chartname;					/**< Chart name where the regex metrics will appear in **/
    char *regex_str;					/**< String representation of the regex **/
    char *regex_name;					/**< If regex is named, this is where its name is stored **/
    regex_t regex;						/**< The compiled regex **/
} Log_parser_cus_config_t;

typedef struct log_parser_cus_metrics{
    unsigned long long count;
} Log_parser_cus_metrics_t;

/* -------------------------------------------------------------------------- */


/* -------------------------------------------------------------------------- */
/*                             General / Other                                */
/* -------------------------------------------------------------------------- */

struct log_parser_metrics{
    unsigned long long num_lines;
    // struct timeval tv;
    time_t last_update;
    union {
        Web_log_metrics_t *web_log;
        Kernel_metrics_t *kernel;
        Systemd_metrics_t *systemd;
        Docker_ev_metrics_t *docker_ev;
        Mqtt_metrics_t *mqtt;
    };	
    Log_parser_cus_metrics_t **parser_cus; /**< Array storing custom chart metrics structs **/
} ;

#endif  // PARSER_H_