synchronized exit for all threads

CMakeLists.txt

@@ -63,7 +63,8 @@ set(NETDATA_LINUX_FILES
         src/proc_uptime.c
         src/sys_kernel_mm_ksm.c
         src/sys_fs_cgroup.c
-        src/sys_fs_btrfs.c)
+        src/sys_fs_btrfs.c
+        )
 
 set(NETDATA_COMMON_FILES
         src/adaptive_resortable_list.c
@@ -94,6 +95,7 @@ set(NETDATA_COMMON_FILES
         src/health_json.c
         src/health_log.c
         src/inlined.h
+        src/locks.c
         src/locks.h
         src/log.c
         src/log.h
@@ -189,6 +191,8 @@ set(APPS_PLUGIN_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c
@@ -205,6 +209,8 @@ set(FREEIPMI_PLUGIN_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c
@@ -219,6 +225,8 @@ set(CGROUP_NETWORK_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c

src/Makefile.am

@@ -70,6 +70,7 @@ netdata_SOURCES = \
 	health_json.c \
 	health_log.c \
 	inlined.h \
+	locks.c \
 	locks.h \
 	log.c \
 	log.h \
@@ -227,6 +228,7 @@ apps_plugin_SOURCES = \
 	clocks.c clocks.h \
 	common.c common.h \
 	inlined.h \
+	locks.c locks.h \
 	log.c log.h \
 	procfile.c procfile.h \
 	web_buffer.c web_buffer.h \
@@ -248,6 +250,7 @@ freeipmi_plugin_SOURCES = \
 	clocks.c clocks.h \
 	common.c common.h \
 	inlined.h \
+	locks.c locks.h \
 	log.c log.h \
 	procfile.c procfile.h \
 	$(NULL)
@@ -261,6 +264,7 @@ cgroup_network_SOURCES = \
 	clocks.c clocks.h \
     common.c common.h \
     inlined.h \
+	locks.c locks.h \
     log.c log.h \
 	procfile.c procfile.h \
 	popen.c popen.h \

src/locks.c

@@ -0,0 +1,319 @@
+#include "common.h"
+
+// ----------------------------------------------------------------------------
+// automatic thread cancelability management, based on locks
+
+static __thread int netdata_thread_first_cancelability = 0;
+static __thread int netdata_thread_lock_cancelability = 0;
+
+static inline void netdata_thread_disable_cancelability(void) {
+    int old;
+    int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);
+    if(ret != 0)
+        error("THREAD_CANCELABILITY: pthread_setcancelstate() returned error %d", ret);
+    else {
+        if(!netdata_thread_lock_cancelability)
+            netdata_thread_first_cancelability = old;
+
+        netdata_thread_lock_cancelability++;
+    }
+}
+
+static inline void netdata_thread_enable_cancelability(void) {
+    if(netdata_thread_lock_cancelability < 1) {
+        error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): oops! invalid thread cancelability count %d - results will be undefined - please report this!", netdata_thread_lock_cancelability);
+    }
+    else if(netdata_thread_lock_cancelability == 1) {
+        int old = 1;
+        int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old);
+        if(ret != 0)
+            error("THREAD_CANCELABILITY: pthread_setcancelstate() returned error %d", ret);
+        else {
+            if(!old)
+                error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): oops! old thread cancelability was changed, expected ENABLED, found DISABLED - please report this!");
+        }
+
+        netdata_thread_lock_cancelability = 0;
+    }
+    else
+        netdata_thread_lock_cancelability--;
+}
+
+// ----------------------------------------------------------------------------
+// mutex
+
+int __netdata_mutex_init(netdata_mutex_t *mutex) {
+    int ret = pthread_mutex_init(mutex, NULL);
+    if(unlikely(ret != 0))
+        error("MUTEX_LOCK: failed to initialize (code %d).", ret);
+    return ret;
+}
+
+int __netdata_mutex_lock(netdata_mutex_t *mutex) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_mutex_lock(mutex);
+    if(unlikely(ret != 0)) {
+        netdata_thread_enable_cancelability();
+        error("MUTEX_LOCK: failed to get lock (code %d)", ret);
+    }
+    return ret;
+}
+
+int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_mutex_trylock(mutex);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_mutex_unlock(netdata_mutex_t *mutex) {
+    int ret = pthread_mutex_unlock(mutex);
+    if(unlikely(ret != 0))
+        error("MUTEX_LOCK: failed to unlock (code %d).", ret);
+    else
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_init(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_lock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_trylock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_unlock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+
+// ----------------------------------------------------------------------------
+// r/w lock
+
+int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_destroy(rwlock);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to destroy lock (code %d)", ret);
+    return ret;
+}
+
+int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_init(rwlock, NULL);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to initialize lock (code %d)", ret);
+    return ret;
+}
+
+int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_rdlock(rwlock);
+    if(unlikely(ret != 0)) {
+        netdata_thread_enable_cancelability();
+        error("RW_LOCK: failed to obtain read lock (code %d)", ret);
+    }
+
+    return ret;
+}
+
+int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_wrlock(rwlock);
+    if(unlikely(ret != 0)) {
+        error("RW_LOCK: failed to obtain write lock (code %d)", ret);
+        netdata_thread_enable_cancelability();
+    }
+
+    return ret;
+}
+
+int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_unlock(rwlock);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to release lock (code %d)", ret);
+    else
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_tryrdlock(rwlock);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_trywrlock(rwlock);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+
+int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_destroy(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_init(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_rdlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_wrlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_unlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_tryrdlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_trywrlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}

src/locks.h

@@ -1,276 +1,46 @@
 #ifndef NETDATA_LOCKS_H
 #define NETDATA_LOCKS_H
 
-// ----------------------------------------------------------------------------
-// mutex
-
 typedef pthread_mutex_t netdata_mutex_t;
-
 #define NETDATA_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
 
-static inline int __netdata_mutex_init(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_init(mutex, NULL);
-    if(unlikely(ret != 0))
-        error("MUTEX_LOCK: failed to initialize (code %d).", ret);
-    return ret;
-}
-
-static inline int __netdata_mutex_lock(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_lock(mutex);
-    if(unlikely(ret != 0))
-        error("MUTEX_LOCK: failed to get lock (code %d)", ret);
-    return ret;
-}
+typedef pthread_rwlock_t netdata_rwlock_t;
+#define NETDATA_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER
 
-static inline int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_trylock(mutex);
-    return ret;
-}
+extern int __netdata_mutex_init(netdata_mutex_t *mutex);
+extern int __netdata_mutex_lock(netdata_mutex_t *mutex);
+extern int __netdata_mutex_trylock(netdata_mutex_t *mutex);
+extern int __netdata_mutex_unlock(netdata_mutex_t *mutex);
+
+extern int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_init(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock);
+
+extern int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+
+extern int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
 
-static inline int __netdata_mutex_unlock(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_unlock(mutex);
-    if(unlikely(ret != 0))
-        error("MUTEX_LOCK: failed to unlock (code %d).", ret);
-    return ret;
-}
 
 #ifdef NETDATA_INTERNAL_CHECKS
 
-static inline int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) from %lu@%s, %s()", mutex, line, file, function);
-    }
-
-    int ret = __netdata_mutex_init(mutex);
-
-    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
-    }
-
-    int ret = __netdata_mutex_lock(mutex);
-
-    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
-    }
-
-    int ret = __netdata_mutex_trylock(mutex);
-
-    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
-    }
-
-    int ret = __netdata_mutex_unlock(mutex);
-
-    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
 #define netdata_mutex_init(mutex)    netdata_mutex_init_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
 #define netdata_mutex_lock(mutex)    netdata_mutex_lock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
 #define netdata_mutex_trylock(mutex) netdata_mutex_trylock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
 #define netdata_mutex_unlock(mutex)  netdata_mutex_unlock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
 
-#else // !NETDATA_INTERNAL_CHECKS
-
-#define netdata_mutex_init(mutex)    __netdata_mutex_init(mutex)
-#define netdata_mutex_lock(mutex)    __netdata_mutex_lock(mutex)
-#define netdata_mutex_trylock(mutex) __netdata_mutex_trylock(mutex)
-#define netdata_mutex_unlock(mutex)  __netdata_mutex_unlock(mutex)
-
-#endif // NETDATA_INTERNAL_CHECKS
-
-
-// ----------------------------------------------------------------------------
-// r/w lock
-
-typedef pthread_rwlock_t netdata_rwlock_t;
-
-#define NETDATA_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER
-
-static inline int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_destroy(rwlock);
-    if(unlikely(ret != 0))
-        error("RW_LOCK: failed to destroy lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_init(rwlock, NULL);
-    if(unlikely(ret != 0))
-        error("RW_LOCK: failed to initialize lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_rdlock(rwlock);
-    if(unlikely(ret != 0))
-        error("RW_LOCK: failed to obtain read lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_wrlock(rwlock);
-    if(unlikely(ret != 0))
-        error("RW_LOCK: failed to obtain write lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_unlock(rwlock);
-    if(unlikely(ret != 0))
-        error("RW_LOCK: failed to release lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_tryrdlock(rwlock);
-    return ret;
-}
-
-static inline int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {
-    int ret = pthread_rwlock_trywrlock(rwlock);
-    return ret;
-}
-
-
-#ifdef NETDATA_INTERNAL_CHECKS
-
-static inline int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_destroy(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_init(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_rdlock(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_wrlock(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_unlock(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_tryrdlock(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
-static inline int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
-    usec_t start = 0;
-
-    if(unlikely(debug_flags & D_LOCKS)) {
-        start = now_boottime_usec();
-        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
-    }
-
-    int ret = __netdata_rwlock_trywrlock(rwlock);
-
-    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
-
-    return ret;
-}
-
 #define netdata_rwlock_destroy(rwlock)   netdata_rwlock_destroy_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
 #define netdata_rwlock_init(rwlock)      netdata_rwlock_init_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
 #define netdata_rwlock_rdlock(rwlock)    netdata_rwlock_rdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
@@ -281,6 +51,11 @@ static inline int netdata_rwlock_trywrlock_debug( const char *file, const char *
 
 #else // !NETDATA_INTERNAL_CHECKS
 
+#define netdata_mutex_init(mutex)    __netdata_mutex_init(mutex)
+#define netdata_mutex_lock(mutex)    __netdata_mutex_lock(mutex)
+#define netdata_mutex_trylock(mutex) __netdata_mutex_trylock(mutex)
+#define netdata_mutex_unlock(mutex)  __netdata_mutex_unlock(mutex)
+
 #define netdata_rwlock_destroy(rwlock)    __netdata_rwlock_destroy(rwlock)
 #define netdata_rwlock_init(rwlock)       __netdata_rwlock_init(rwlock)
 #define netdata_rwlock_rdlock(rwlock)     __netdata_rwlock_rdlock(rwlock)

src/main.c

@@ -6,33 +6,28 @@ void netdata_cleanup_and_exit(int ret) {
     netdata_exit = 1;
 
     error_log_limit_unlimited();
-    info("netdata is preparing to exit...");
+    info("MAIN: netdata prepares to exit...");
 
-    // allow all the threads to cleanup by themselves
-    unsigned int w = (unsigned int)default_rrd_update_every + 2;
-    info("Giving %u secs to background threads to cleanup...", w);
-    sleep(w);
-
-    // kill all threads and childs
-    //info("Stopping all threads and child processes...");
-    //kill_childs();
+    // stop everything
+    info("MAIN: stopping all threads and child processes...");
+    cancel_main_threads();
 
     // cleanup the database (delete files not needed)
-    info("Cleaning up the database...");
+    info("MAIN: cleaning up the database...");
     rrdhost_cleanup_all();
 
     // free the database
-    //info("Freeing database memory...");
-    //rrdhost_free_all();
+    info("MAIN: freeing database memory...");
+    rrdhost_free_all();
 
     // unlink the pid
     if(pidfile[0]) {
-        info("Removing netdata PID file '%s'...", pidfile);
+        info("MAIN: removing netdata PID file '%s'...", pidfile);
         if(unlink(pidfile) != 0)
-            error("Cannot unlink pidfile '%s'.", pidfile);
+            error("MAIN: cannot unlink pidfile '%s'.", pidfile);
     }
 
-    info("All done - netdata is now exiting - bye bye...");
+    info("MAIN: all done - netdata is now exiting - bye bye...");
     exit(ret);
 }
 
@@ -182,50 +177,29 @@ int killpid(pid_t pid, int sig)
     return ret;
 }
 
-void kill_childs() {
+void cancel_main_threads() {
     error_log_limit_unlimited();
 
-    siginfo_t info;
-
-    struct web_client *w;
-    for(w = web_clients; w ; w = w->next) {
-        info("Stopping web client %s", w->client_ip);
-        int ret;
-        if((ret = pthread_cancel(w->thread)) != 0)
-            error("pthread_cancel() failed with code %d.", ret);
-
-        WEB_CLIENT_IS_OBSOLETE(w);
-    }
-
     int i;
     for (i = 0; static_threads[i].name != NULL ; i++) {
         if(static_threads[i].enabled) {
-            info("Stopping %s thread", static_threads[i].name);
+            info("MAIN: Calling pthread_cancel() on %s thread", static_threads[i].name);
             int ret;
             if((ret = pthread_cancel(*static_threads[i].thread)) != 0)
-                error("pthread_cancel() failed with code %d.", ret);
+                error("MAIN: pthread_cancel() failed with code %d.", ret);
+            else
+                info("MAIN: thread %s cancelled", static_threads[i].name);
 
             static_threads[i].enabled = 0;
         }
     }
 
-    if(tc_child_pid) {
-        info("Killing tc-qos-helper process %d", tc_child_pid);
-        if(killpid(tc_child_pid, SIGTERM) != -1)
-            waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED);
-
-        tc_child_pid = 0;
-    }
-
-    // stop all running plugins
-    pluginsd_stop_all_external_plugins();
-
     // if, for any reason there is any child exited
     // catch it here
-    info("Cleaning up an other children");
+    info("MAIN: waiting for any unfinished child processes");
+    siginfo_t info;
     waitid(P_PID, 0, &info, WEXITED|WNOHANG);
-
-    info("All threads/childs stopped.");
+    info("MAIN: all threads/childs stopped.");
 }
 
 struct option_def option_definitions[] = {

src/main.h

@@ -22,7 +22,7 @@ struct netdata_static_thread {
     char *config_section;
     char *config_name;
 
-    volatile int enabled;
+    volatile sig_atomic_t enabled;
 
     pthread_t *thread;
 
@@ -30,7 +30,7 @@ struct netdata_static_thread {
     void *(*start_routine) (void *);
 };
 
-extern void kill_childs(void);
+extern void cancel_main_threads(void);
 extern int killpid(pid_t pid, int signal);
 extern void netdata_cleanup_and_exit(int ret) NORETURN;
 

src/plugin_tc.c

@@ -829,9 +829,30 @@ static inline void tc_split_words(char *str, char **words, int max_words) {
 }
 
 volatile pid_t tc_child_pid = 0;
-void *tc_main(void *ptr) {
+
+static void tc_main_cleanup(void *ptr) {
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+    if(static_thread->enabled) {
+        static_thread->enabled = 0;
+
+        info("TC: cleaning up...");
+
+        if(tc_child_pid) {
+            info("TC: killing with SIGTERM tc-qos-helper process %d", tc_child_pid);
+            if(killpid(tc_child_pid, SIGTERM) != -1) {
+                siginfo_t info;
+
+                info("TC: waiting for tc plugin child process pid %d to exit...", tc_child_pid);
+                waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED);
+                // info("TC: finished tc plugin child process pid %d.", tc_child_pid);
+            }
 
+            tc_child_pid = 0;
+        }
+    }
+}
+
+void *tc_main(void *ptr) {
     info("TC thread created with task id %d", gettid());
 
     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
@@ -863,10 +884,10 @@ void *tc_main(void *ptr) {
 
     snprintfz(buffer, TC_LINE_MAX, "%s/tc-qos-helper.sh", netdata_configured_plugins_dir);
     char *tc_script = config_get("plugin:tc", "script to run to get tc values", buffer);
-    
-    for(;1;) {
-        if(unlikely(netdata_exit)) break;
 
+    pthread_cleanup_push(tc_main_cleanup, ptr);
+
+    while(!netdata_exit) {
         FILE *fp;
         struct tc_device *device = NULL;
         struct tc_class *class = NULL;
@@ -1151,8 +1172,8 @@ void *tc_main(void *ptr) {
 
 cleanup:
     info("TC thread exiting");
+    pthread_cleanup_pop(1);
 
-    static_thread->enabled = 0;
     pthread_exit(NULL);
     return NULL;
 }

src/plugins_d.c

@@ -476,15 +476,36 @@ cleanup:
     return count;
 }
 
+static void pluginsd_worker_thread_cleanup(void *arg) {
+    struct plugind *cd = (struct plugind *)arg;
+
+    info("PLUGINSD: '%s' thread exiting", cd->fullfilename);
+
+    if(cd->enabled && !cd->obsolete) {
+        cd->obsolete = 1;
+
+        if (cd->pid) {
+            siginfo_t info;
+            info("PLUGINSD: killing %s plugin child process pid %d", cd->id, cd->pid);
+            if (killpid(cd->pid, SIGTERM) != -1) {
+                info("PLUGINSD: waiting for %s plugin child process pid %d to exit...", cd->id, cd->pid);
+                waitid(P_PID, (id_t) cd->pid, &info, WEXITED);
+                info("PLUGINSD: finished %s plugin child process pid %d.", cd->id, cd->pid);
+            }
+            cd->pid = 0;
+        }
+    }
+}
+
 void *pluginsd_worker_thread(void *arg) {
     struct plugind *cd = (struct plugind *)arg;
     cd->obsolete = 0;
 
     size_t count = 0;
 
-    for(;;) {
-        if(unlikely(netdata_exit)) break;
+    pthread_cleanup_push(pluginsd_worker_thread_cleanup, arg);
 
+    while(!netdata_exit) {
         FILE *fp = mypopen(cd->cmd, &cd->pid);
         if(unlikely(!fp)) {
             error("Cannot popen(\"%s\", \"r\").", cd->cmd);
@@ -503,8 +524,7 @@ void *pluginsd_worker_thread(void *arg) {
         // get the return code
         int code = mypclose(fp, cd->pid);
 
-        if(unlikely(netdata_exit)) break;
-        else if(code != 0) {
+        if(code != 0) {
             // the plugin reports failure
 
             if(likely(!cd->successful_collections)) {
@@ -548,16 +568,33 @@ void *pluginsd_worker_thread(void *arg) {
         if(unlikely(!cd->enabled)) break;
     }
 
-    info("PLUGINSD: '%s' thread exiting", cd->fullfilename);
-
-    cd->obsolete = 1;
+    pthread_cleanup_pop(1);
     pthread_exit(NULL);
     return NULL;
 }
 
-void *pluginsd_main(void *ptr) {
-    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+static void pluginsd_main_cleanup(void *data) {
+    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)data;
+    if(static_thread->enabled) {
+        static_thread->enabled = 0;
+
+        info("PLUGINSD: cleaning up plugin threads...");
+        struct plugind *cd;
+
+        for (cd = pluginsd_root; cd; cd = cd->next) {
+            if (cd->enabled && !cd->obsolete) {
+                info("PLUGINSD: Calling pthread_cancel() on %s plugin thread", cd->id);
+                int ret;
+                if ((ret = pthread_cancel(cd->thread)) != 0)
+                    error("PLUGINSD: pthread_cancel() failed with code %d.", ret);
+            }
+        }
+
+        info("PLUGINSD: cleanup completed.");
+    }
+}
 
+void *pluginsd_main(void *ptr) {
     info("PLUGINS.D thread created with task id %d", gettid());
 
     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
@@ -574,9 +611,9 @@ void *pluginsd_main(void *ptr) {
     // so that we don't log broken directories on each loop
     int directory_errors[PLUGINSD_MAX_DIRECTORIES] =  { 0 };
 
-    for(;;) {
-        if(unlikely(netdata_exit)) break;
+    pthread_cleanup_push(pluginsd_main_cleanup, ptr);
 
+    while(!netdata_exit) {
         int idx;
         const char *directory_name;
 
@@ -668,31 +705,7 @@ void *pluginsd_main(void *ptr) {
         sleep((unsigned int) scan_frequency);
     }
 
-    info("PLUGINS.D thread exiting");
-
-    static_thread->enabled = 0;
+    pthread_cleanup_pop(1);
     pthread_exit(NULL);
     return NULL;
 }
-
-
-void pluginsd_stop_all_external_plugins() {
-    siginfo_t info;
-    struct plugind *cd;
-    for(cd = pluginsd_root ; cd ; cd = cd->next) {
-        if(cd->enabled && !cd->obsolete) {
-            info("Stopping %s plugin thread", cd->id);
-            pthread_cancel(cd->thread);
-
-            if(cd->pid) {
-                info("killing %s plugin child process pid %d", cd->id, cd->pid);
-                if(killpid(cd->pid, SIGTERM) != -1)
-                    waitid(P_PID, (id_t) cd->pid, &info, WEXITED);
-
-                cd->pid = 0;
-            }
-
-            cd->obsolete = 1;
-        }
-    }
-}

src/plugins_d.h

@@ -26,7 +26,7 @@ struct plugind {
     char fullfilename[FILENAME_MAX+1];  // with path
     char cmd[PLUGINSD_CMD_MAX+1];       // the command that it executes
 
-    pid_t pid;
+    volatile pid_t pid;
     pthread_t thread;
 
     size_t successful_collections;      // the number of times we have seen
@@ -36,8 +36,8 @@ struct plugind {
                                         // without collecting values
 
     int update_every;                   // the plugin default data collection frequency
-    volatile int obsolete;              // do not touch this structure after setting this to 1
-    volatile int enabled;               // if this is enabled or not
+    volatile sig_atomic_t obsolete;     // do not touch this structure after setting this to 1
+    volatile sig_atomic_t enabled;      // if this is enabled or not
 
     time_t started_t;
 
@@ -47,7 +47,6 @@ struct plugind {
 extern struct plugind *pluginsd_root;
 
 extern void *pluginsd_main(void *ptr);
-extern void pluginsd_stop_all_external_plugins(void);
 
 extern size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations);
 extern int pluginsd_split_words(char *str, char **words, int max_words);

src/popen.c

@@ -43,7 +43,7 @@ static void mypopen_del(FILE *fp) {
 #define PIPE_READ 0
 #define PIPE_WRITE 1
 
-FILE *mypopen(const char *command, pid_t *pidptr)
+FILE *mypopen(const char *command, volatile pid_t *pidptr)
 {
     int pipefd[2];