HTTP proxy support + some cleanup (#8418)

* HTTP proxy support + some cleanup

* fix unrelated compiler warnings with -Wextra

* minor - log proxy setting

* run changed code trough .clang-format

* fix case when url ends by /

* update README

aclk/README.md

@@ -7,4 +7,19 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/aclk/README.md
 
 # Agent-cloud link (ACLK)
 
-This is the agent cloud link (ACLK) information file
+
+## Configuration Options
+
+In `netdata.conf`:
+
+```ini
+[agent_cloud_link]
+    proxy = none
+```
+
+Parameter proxy can take one of the following values:
+
+- `env` - the default (try to read environment variables `http_proxy` and `socks_proxy`)
+- `none` - do not use any proxy (even if system configured otherwise)
+- `socks5[h]://[user:pass@]host:ip` - will use specified socks proxy
+- `http://[user:pass@]host:ip` - will use specified http proxy

aclk/aclk_common.c

@@ -6,16 +6,31 @@ struct {
     ACLK_PROXY_TYPE type;
     const char *url_str;
 } supported_proxy_types[] = {
-    { .type = PROXY_TYPE_SOCKS5,   .url_str = "socks5" ACLK_PROXY_PROTO_ADDR_SEPARATOR  },
+    { .type = PROXY_TYPE_SOCKS5,   .url_str = "socks5"  ACLK_PROXY_PROTO_ADDR_SEPARATOR },
     { .type = PROXY_TYPE_SOCKS5,   .url_str = "socks5h" ACLK_PROXY_PROTO_ADDR_SEPARATOR },
+    { .type = PROXY_TYPE_HTTP,     .url_str = "http"    ACLK_PROXY_PROTO_ADDR_SEPARATOR },
     { .type = PROXY_TYPE_UNKNOWN,  .url_str = NULL                                      },
 };
 
+const char *aclk_proxy_type_to_s(ACLK_PROXY_TYPE *type)
+{
+    switch (*type) {
+        case PROXY_DISABLED:
+            return "disabled";
+        case PROXY_TYPE_HTTP:
+            return "HTTP";
+        case PROXY_TYPE_SOCKS5:
+            return "SOCKS";
+        default:
+            return "Unknown";
+    }
+}
+
 static inline ACLK_PROXY_TYPE aclk_find_proxy(const char *string)
 {
     int i = 0;
-    while( supported_proxy_types[i].url_str ) {
-        if(!strncmp(supported_proxy_types[i].url_str, string, strlen(supported_proxy_types[i].url_str)))
+    while (supported_proxy_types[i].url_str) {
+        if (!strncmp(supported_proxy_types[i].url_str, string, strlen(supported_proxy_types[i].url_str)))
             return supported_proxy_types[i].type;
         i++;
     }
@@ -24,13 +39,13 @@ static inline ACLK_PROXY_TYPE aclk_find_proxy(const char *string)
 
 ACLK_PROXY_TYPE aclk_verify_proxy(const char *string)
 {
-    if(!string)
+    if (!string)
         return PROXY_TYPE_UNKNOWN;
 
-    while(*string == 0x20)
+    while (*string == 0x20)
         string++;
 
-    if(!*string)
+    if (!*string)
         return PROXY_TYPE_UNKNOWN;
 
     return aclk_find_proxy(string);
@@ -38,112 +53,155 @@ ACLK_PROXY_TYPE aclk_verify_proxy(const char *string)
 
 // helper function to censor user&password
 // for logging purposes
-void safe_log_proxy_censor(char *proxy) {
+void safe_log_proxy_censor(char *proxy)
+{
     size_t length = strlen(proxy);
-    char *auth = proxy+length-1;
+    char *auth = proxy + length - 1;
     char *cur;
 
-    while( (auth >= proxy) && (*auth != '@') )
+    while ((auth >= proxy) && (*auth != '@'))
         auth--;
 
     //if not found or @ is first char do nothing
-    if(auth<=proxy)
+    if (auth <= proxy)
         return;
 
     cur = strstr(proxy, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
-    if(!cur)
+    if (!cur)
         cur = proxy;
     else
         cur += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
 
-    while(cur < auth) {
-        *cur='X';
+    while (cur < auth) {
+        *cur = 'X';
         cur++;
     }
 }
 
-static inline void safe_log_proxy_error(char *str, const char *proxy) {
+static inline void safe_log_proxy_error(char *str, const char *proxy)
+{
     char *log = strdupz(proxy);
     safe_log_proxy_censor(log);
     error("%s Provided Value:\"%s\"", str, log);
     freez(log);
 }
 
-static inline int check_socks_enviroment(const char **proxy) {
+static inline int check_socks_enviroment(const char **proxy)
+{
     char *tmp = getenv("socks_proxy");
 
-    if(!tmp)
+    if (!tmp)
+        return 1;
+
+    if (aclk_verify_proxy(tmp) == PROXY_TYPE_SOCKS5) {
+        *proxy = tmp;
+        return 0;
+    }
+
+    safe_log_proxy_error(
+        "Environment var \"socks_proxy\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".",
+        tmp);
+    return 1;
+}
+
+static inline int check_http_enviroment(const char **proxy)
+{
+    char *tmp = getenv("http_proxy");
+
+    if (!tmp)
         return 1;
 
-    if(aclk_verify_proxy(tmp) == PROXY_TYPE_SOCKS5) {
+    if (aclk_verify_proxy(tmp) == PROXY_TYPE_HTTP) {
         *proxy = tmp;
         return 0;
     }
 
-    safe_log_proxy_error("Environment var \"socks_proxy\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", tmp);
+    safe_log_proxy_error(
+        "Environment var \"http_proxy\" defined but of unknown format. Supported syntax: \"http[s]://[user:pass@]host:ip\".",
+        tmp);
     return 1;
 }
 
-const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type) {
+const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type)
+{
     const char *proxy = config_get(CONFIG_SECTION_ACLK, ACLK_PROXY_CONFIG_VAR, ACLK_PROXY_ENV);
     *type = PROXY_DISABLED;
 
-    if(strcmp(proxy, "none") == 0)
+    if (strcmp(proxy, "none") == 0)
         return proxy;
 
-    if(strcmp(proxy, ACLK_PROXY_ENV) == 0) {
-        if(check_socks_enviroment(&proxy) == 0)
+    if (strcmp(proxy, ACLK_PROXY_ENV) == 0) {
+        if (check_socks_enviroment(&proxy) == 0)
             *type = PROXY_TYPE_SOCKS5;
+        else if (check_http_enviroment(&proxy) == 0)
+            *type = PROXY_TYPE_HTTP;
         return proxy;
     }
 
     *type = aclk_verify_proxy(proxy);
-    if(*type == PROXY_TYPE_UNKNOWN) {
+    if (*type == PROXY_TYPE_UNKNOWN) {
         *type = PROXY_DISABLED;
-        safe_log_proxy_error("Config var \"" ACLK_PROXY_CONFIG_VAR "\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", proxy);
+        safe_log_proxy_error(
+            "Config var \"" ACLK_PROXY_CONFIG_VAR
+            "\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".",
+            proxy);
     }
 
     return proxy;
 }
 
+// helper function to read settings only once (static)
+// as claiming, challenge/response and ACLK
+// read the same thing, no need to parse again
+const char *aclk_get_proxy(ACLK_PROXY_TYPE *type)
+{
+    static const char *proxy = NULL;
+    static ACLK_PROXY_TYPE proxy_type = PROXY_NOT_SET;
+
+    if (proxy_type == PROXY_NOT_SET)
+        proxy = aclk_lws_wss_get_proxy_setting(&proxy_type);
+
+    *type = proxy_type;
+    return proxy;
+}
+
 int aclk_decode_base_url(char *url, char **aclk_hostname, char **aclk_port)
 {
-int pos = 0;
-    if (!strncmp("https://", url, 8))
-    {
+    int pos = 0;
+    if (!strncmp("https://", url, 8)) {
         pos = 8;
-    }
-    else if (!strncmp("http://", url, 7))
-    {
+    } else if (!strncmp("http://", url, 7)) {
         error("Cannot connect ACLK over %s -> unencrypted link is not supported", url);
         return 1;
     }
-int host_end = pos;
-    while( url[host_end] != 0 && url[host_end] != '/' && url[host_end] != ':' )
+    int host_end = pos;
+    while (url[host_end] != 0 && url[host_end] != '/' && url[host_end] != ':')
         host_end++;
-    if (url[host_end] == 0)
-    {
-        *aclk_hostname = strdupz(url+pos);
+    if (url[host_end] == 0) {
+        *aclk_hostname = strdupz(url + pos);
         *aclk_port = strdupz("443");
         info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
         return 0;
     }
-    if (url[host_end] == ':')
-    {
+    if (url[host_end] == ':') {
         *aclk_hostname = callocz(host_end - pos + 1, 1);
-        strncpy(*aclk_hostname, url+pos, host_end - pos);
+        strncpy(*aclk_hostname, url + pos, host_end - pos);
         int port_end = host_end + 1;
         while (url[port_end] >= '0' && url[port_end] <= '9')
             port_end++;
-        if (port_end - host_end > 6)
-        {
+        if (port_end - host_end > 6) {
             error("Port specified in %s is invalid", url);
             return 0;
         }
         *aclk_port = callocz(port_end - host_end + 1, 1);
-        for(int i=host_end + 1; i < port_end; i++)
+        for (int i = host_end + 1; i < port_end; i++)
             (*aclk_port)[i - host_end - 1] = url[i];
     }
+    if (url[host_end] == '/') {
+        *aclk_port = strdupz("443");
+        *aclk_hostname = callocz(1, host_end - pos + 1);
+        strncpy(*aclk_hostname, url+pos, host_end - pos);
+    }
     info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
     return 0;
 }

aclk/aclk_common.h

@@ -11,6 +11,8 @@ typedef enum aclk_proxy_type {
     PROXY_NOT_SET,
 } ACLK_PROXY_TYPE;
 
+const char *aclk_proxy_type_to_s(ACLK_PROXY_TYPE *type);
+
 #define ACLK_PROXY_PROTO_ADDR_SEPARATOR "://"
 #define ACLK_PROXY_ENV "env"
 #define ACLK_PROXY_CONFIG_VAR "proxy"
@@ -19,5 +21,6 @@ ACLK_PROXY_TYPE aclk_verify_proxy(const char *string);
 const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type);
 void safe_log_proxy_censor(char *proxy);
 int aclk_decode_base_url(char *url, char **aclk_hostname, char **aclk_port);
+const char *aclk_get_proxy(ACLK_PROXY_TYPE *type);
 
 #endif //ACLK_COMMON_H

aclk/aclk_lws_https_client.c

@@ -19,6 +19,7 @@ struct simple_hcc_data {
 
 static int simple_https_client_callback(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len)
 {
+    UNUSED(user);
     int n;
     char *ptr;
     char buffer[SMALL_BUFFER];
@@ -78,7 +79,7 @@ static int simple_https_client_callback(struct lws *wsi, enum lws_callback_reaso
         debug(D_ACLK, "LWS_CALLBACK_CLIENT_HTTP_WRITEABLE");
         if(perconn_data && perconn_data->payload) {
             n = strlen(perconn_data->payload);
-            if(perconn_data->data_size < LWS_PRE + n + 1) {
+            if(perconn_data->data_size < (size_t)LWS_PRE + n + 1) {
                 error("Buffer given is not big enough");
                 return 1;
             }
@@ -133,12 +134,16 @@ static const struct lws_protocols protocols[] = {
         simple_https_client_callback,
         0,
         0,
+        0,
+        0,
+        0
     },
-    { NULL, NULL, 0, 0 }
+    { NULL, NULL, 0, 0, 0, 0, 0 }
 };
 
 static void simple_hcc_log_divert(int level, const char *line)
 {
+    UNUSED(level);
     error("Libwebsockets: %s", line);
 }
 
@@ -159,15 +164,7 @@ int aclk_send_https_request(char *method, char *host, char *port, char *url, cha
     int n = 0;
     time_t timestamp;
 
-    //TODO -> deduplicate (aclk_lws_wss_connect)
-    static const char *proxy = NULL;
-    static ACLK_PROXY_TYPE proxy_type = PROXY_NOT_SET;
     struct lws_vhost *vhost;
-    char *log;
-
-    if(proxy_type == PROXY_NOT_SET)
-        proxy = aclk_lws_wss_get_proxy_setting(&proxy_type);
-
 
     memset(&info, 0, sizeof info);
 
@@ -212,17 +209,8 @@ int aclk_send_https_request(char *method, char *host, char *port, char *url, cha
     if(!vhost)
         fatal("Could not find the default LWS vhost.");
 
-    lws_set_socks(vhost, ":");
-    lws_set_proxy(vhost, ":");
-
-    if(proxy_type == PROXY_TYPE_SOCKS5) {
-        log = strdupz(proxy);
-        safe_log_proxy_censor(log);
-        info("Connecting using SOCKS5 proxy:\"%s\"", log);
-        freez(log);
-        if(aclk_wss_set_socks(vhost, proxy))
-            error("LWS failed to accept socks proxy.");
-    }
+    //set up proxy
+    aclk_wss_set_proxy(vhost);
 
     lws_client_connect_via_info(&i);
 

aclk/aclk_lws_wss_client.c

@@ -208,42 +208,81 @@ void aclk_lws_wss_client_destroy()
 #endif
 }
 
-int aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks) {
+static int aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks)
+{
     char *proxy = strstr(socks, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
 
-    if(!proxy)
+    if (!proxy)
         return -1;
 
     proxy += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
 
-    if(!*proxy)
+    if (!*proxy)
         return -1;
 
     return lws_set_socks(vhost, proxy);
 }
 
+void aclk_wss_set_proxy(struct lws_vhost *vhost)
+{
+    const char *proxy;
+    ACLK_PROXY_TYPE proxy_type;
+    char *log;
+
+    proxy = aclk_get_proxy(&proxy_type);
+
+    lws_set_socks(vhost, ":");
+    lws_set_proxy(vhost, ":");
+
+    if (proxy_type == PROXY_TYPE_UNKNOWN) {
+        error("Unknown proxy type");
+        return;
+    }
+
+    if (proxy_type == PROXY_TYPE_SOCKS5 || proxy_type == PROXY_TYPE_HTTP) {
+        log = strdupz(proxy);
+        safe_log_proxy_censor(log);
+        info("Connecting using %s proxy:\"%s\"", aclk_proxy_type_to_s(&proxy_type), log);
+        freez(log);
+    }
+    if (proxy_type == PROXY_TYPE_SOCKS5) {
+        if (aclk_wss_set_socks(vhost, proxy))
+            error("LWS failed to accept socks proxy.");
+        return;
+    }
+    if (proxy_type == PROXY_TYPE_HTTP) {
+        if (lws_set_proxy(vhost, proxy))
+            error("LWS failed to accept http proxy.");
+        return;
+    }
+    if (proxy_type != PROXY_DISABLED)
+        error("Unknown proxy type");
+}
+
 // Return code indicates if connection attempt has started async.
 int aclk_lws_wss_connect(char *host, int port)
 {
     struct lws_client_connect_info i;
     struct lws_vhost *vhost;
-    static const char *proxy = NULL;
-    static ACLK_PROXY_TYPE proxy_type = PROXY_NOT_SET;
-    char *log;
 
     if (!engine_instance) {
         return aclk_lws_wss_client_init(host, port);
         // PROTOCOL_INIT callback will call again.
     }
 
-    if(proxy_type == PROXY_NOT_SET)
-        proxy = aclk_lws_wss_get_proxy_setting(&proxy_type);
-
     if (engine_instance->lws_wsi) {
         error("Already Connected. Only one connection supported at a time.");
         return 0;
     }
 
+    memset(&i, 0, sizeof(i));
+    i.context = engine_instance->lws_context;
+    i.port = engine_instance->port;
+    i.address = engine_instance->host;
+    i.path = "/mqtt";
+    i.host = engine_instance->host;
+    i.protocol = "mqtt";
+
     // from LWS docu:
     // If option LWS_SERVER_OPTION_EXPLICIT_VHOSTS is given, no vhost is
     // created; you're expected to create your own vhosts afterwards using
@@ -253,30 +292,7 @@ int aclk_lws_wss_connect(char *host, int port)
     if(!vhost)
         fatal("Could not find the default LWS vhost.");
 
-    memset(&i, 0, sizeof(i));
-    i.context = engine_instance->lws_context;
-    i.port = engine_instance->port;
-    i.address = engine_instance->host;
-    i.path = "/mqtt";
-    i.host = engine_instance->host;
-    i.protocol = "mqtt";
-
-    switch (proxy_type) {
-    case PROXY_DISABLED:
-        lws_set_socks(vhost, ":");
-        lws_set_proxy(vhost, ":");
-        break;
-    case PROXY_TYPE_SOCKS5:
-        log = strdupz(proxy);
-        safe_log_proxy_censor(log);
-        info("Connecting using SOCKS5 proxy:\"%s\"", log);
-        freez(log);
-        if(aclk_wss_set_socks(vhost, proxy))
-            error("LWS failed to accept socks proxy.");
-        break;
-    default:
-        error("The proxy could not be set. Unknown proxy type.");
-    }
+    aclk_wss_set_proxy(vhost);
 
 #ifdef ACLK_SSL_ALLOW_SELF_SIGNED
     i.ssl_connection = LCCSCF_USE_SSL | LCCSCF_ALLOW_SELFSIGNED | LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK;

aclk/aclk_lws_wss_client.h

@@ -81,7 +81,7 @@ void aclk_lws_connection_data_received();
 void aclk_lws_connection_closed();
 void lws_wss_check_queues(size_t *write_len, size_t *write_len_bytes, size_t *read_len);
 
-int aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks);
+void aclk_wss_set_proxy(struct lws_vhost *vhost);
 
 #define FRAGMENT_SIZE 4096
 #endif

claim/claim.c

@@ -64,9 +64,9 @@ void claim_agent(char *claiming_arguments)
     ACLK_PROXY_TYPE proxy_type;
     char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy";
 
-    proxy_str = aclk_lws_wss_get_proxy_setting(&proxy_type);
+    proxy_str = aclk_get_proxy(&proxy_type);
 
-    if(proxy_type == PROXY_TYPE_SOCKS5)
+    if (proxy_type == PROXY_TYPE_SOCKS5 || proxy_type == PROXY_TYPE_HTTP)
         snprintf(proxy_flag, CLAIMING_PROXY_LENGTH, "-proxy=\"%s\"", proxy_str);
 
     snprintfz(command_buffer,

claim/netdata-claim.sh.in

@@ -129,7 +129,7 @@ do
                 -hostname=*) HOSTNAME=${arg:10} ;;
                 -verbose) VERBOSE=1 ;;
                 -insecure) INSECURE=1 ;;
-                -proxy=socks*) PROXY=${arg:7} ;;
+                -proxy=*) PROXY=${arg:7} ;;
                 -noproxy) NOPROXY=yes ;;
                 *)  echo >&2 "Unknown argument ${arg}"
                     exit 1 ;;
@@ -148,6 +148,7 @@ echo >&2 "Base URL: $URL_BASE"
 echo >&2 "Id: $ID"
 echo >&2 "Rooms: $ROOMS"
 echo >&2 "Hostname: $HOSTNAME"
+echo >&2 "Proxy: $PROXY"
 
 # create the claiming directory for this user
 if [ ! -d "${CLAIMING_DIR}" ] ; then
@@ -214,6 +215,8 @@ else
         --body-file=\"${CLAIMING_DIR}/tmpin.txt\""
         if [ "${NOPROXY}" = "yes" ] ; then
                 URLCOMMAND="${URLCOMMAND} --no-proxy"
+        elif [ "${PROXY:0:4}" = http ] ; then
+                URLCOMMAND="export http_proxy=${PROXY}; ${URLCOMMAND}"
         fi
 fi