IMAPsync/X/cgi_memo

#!/bin/sh

# $Id: cgi_memo,v 1.99 2022/09/15 08:25:31 gilles Exp gilles $

if test -n "$1"; then 
        echoq() { echo "$@" ; } # not quiet mode
else
        echoq() { : ; } # quiet mode: nop
fi


run_test() {
        tests_count=`expr 1 + $tests_count`
        # do not run anything between the two following instructions
        "$@"; run_test_status=$?
        # now you can run something since $? is saved
        if test x"$run_test_status" = x"0"; then
                echo "ok $tests_count $@"
        else
                echo "not ok $tests_count $@"
                tests_failed_count=`expr 1 + $tests_failed_count`
                tests_failed_list="$tests_failed_list $tests_count"
        fi
        return $run_test_status
}


run_tests() {
        tests_count=0
        tests_failed_count=0
        tests_failed_list=
        for t in "$@"; do
                echo "### running $t"
                "$t"
        done
        echo 
        echo "#### ALL tests done"
        if test 0 -eq $tests_failed_count; then
                echo "ALL $tests_count TESTS SUCCESSFUL"
                return 0
        else
                # At least one failed
                echo "FAILED $tests_failed_count/$tests_count TESTS: $tests_failed_list"
                return 1
        fi
}


tests()
{
        :
        # All tests
        run_tests \
                tests_pattern_filename \
                tests_grep_last_modified \
                tests_grep_date \
                tests_date_to_epoch \
                tests_diff_current_vs_last_modified
}


tests_all_verbose_if_failure()
{
        # Run tests silent but if failure then rerun them verbose.
        # return 0 if all tests passed
        # return 1 if some tests failed
        
        if ! tests > /dev/null 3>&1 ; then
                tests
                return 1
        fi
        return 0
}

#### Variable definitions

tests_count=0
tests_failed_count=0


here_is_freebsd()
{
        test FreeBSD = `uname -s`
}

here_is_linux()
{
        test Linux = `uname -s`
}


ram_total_in_bytes()
{
        here_is_linux   && grep MemTotal /proc/meminfo | awk '{print $2 " * 1024"}' | bc && return  
        here_is_freebsd && sysctl -n hw.physmem && return
        echo 1
}

ram_total_in_KiB()
{
        here_is_linux   && grep MemTotal /proc/meminfo | awk '{print $2}' && return  
        here_is_freebsd && sysctl -n hw.physmem | awk '{print $1 " / 1024"}' | bc  && return  
        echo 1
}

ram_total_var()
{
        ram_total_in_bytes=`ram_total_in_bytes`
        ram_total_in_KiB=`ram_total_in_KiB`
        ram_total_human=`bytestohuman $ram_total_in_bytes`
}

ram_total_var

echoq ram_total
ram_total()
{
        echo ram_total_in_KiB=$ram_total_in_KiB
        echo ram_total_in_bytes=$ram_total_in_bytes
        echo ram_total_human=$ram_total_human
}



echoq list_all_logs
list_all_logs()
{
    cat list_all_logs_auto.txt | grep -a -v 385d7a4d8d428d7aa2b57c8982629e2bd67698ed/ | grep -a "$1" 
}

echoq list_all_logs_generate
list_all_logs_generate()
{
    echo Result in list_all_logs.txt
    sortmtimef . | grep -a -v perl.core.txt | grep -a -v 385d7a4d8d428d7aa2b57c8982629e2bd67698ed/ | grep -a \./......................................../ | grep -a \.txt > list_all_logs.txt.tmp
    mv list_all_logs.txt.tmp list_all_logs.txt
}

list_log_matching()
{
        pattern="$1"
        # Ignore no args runs going to 385d7a4d8d428d7aa2b57c8982629e2bd67698ed
        cat list_all_logs_auto.txt | grep -a -v 385d7a4d8d428d7aa2b57c8982629e2bd67698ed | egrep -a -- "$pattern"
}


statsfile()
{
        base=`pattern_filename G $1 $2`
        statsfile=$base.txt
        echo $statsfile
}



echoq biggest_transfer
biggest_transfer()
{
    statsfile=`statsfile Total_bytes_transferred $1`
    bytestohuman `datamash_file_op_index "$statsfile" max 5`
}

echoq total_bytes_transferred
total_bytes_transferred()
{
    statsfile=`statsfile Total_bytes_transferred $1`
    datamash_file_op_index "$statsfile" sum 5
}

# Total volume transferred
echoq total_volume_transferred
total_volume_transferred()
{
    #echo -n 'numfmt --to=iec-i '
    bytestohuman `total_bytes_transferred $1`
}


echoq total_messages_transferred
total_messages_transferred()
{
    statsfile=`statsfile Messages_transferred $1`
    datamash_file_op_index "$statsfile" sum  4 %16.0f | tr -d ' '
}


longest_transfer()
{
        statsfile=`statsfile Transfer_time $1`
        LC_ALL=C printf "%.0f\n" `datamash_file_op_index "$statsfile" max 4`
}


echoq pids_of_imapsync_running
pids_of_imapsync_running() {
    pgrep -d ' ' -f cgi-bin/imapsync
    : # always return true
}

echoq number_of_imapsync_running
number_of_imapsync_running()
{
    pids_of_imapsync_running | wc -w | tr -d ' '
    : # always return true
}

echoq pids_of_proximapsync_running
pids_of_proximapsync_running() {
    pgrep -d ' ' -f cgi-bin/proximapsync
    : # always return true
}

echoq number_of_proximapsync_running
number_of_proximapsync_running()
{
    pids_of_proximapsync_running | wc -w | tr -d ' '
    : # always return true
}

echoq memory_used_by_all_proximapsync_KiB
memory_used_by_all_proximapsync_KiB()
{
        # Sum up all memory taken by imapsync runs, in KiB.
        pids_of_proximapsync_running=`pids_of_proximapsync_running`
        if test -n "$pids_of_proximapsync_running" ; then
                ps -o rss -p $pids_of_proximapsync_running | sed 1,1d | datamash sum  1
        else
                echo 0
        fi
}


echoq number_and_pids_of_imapsync_running
number_and_pids_of_imapsync_running()
{
    echo "`number_of_imapsync_running` : `pids_of_imapsync_running`"
    : # always return true
}

echoq memory_used_by_all_imapsync_KiB
memory_used_by_all_imapsync_KiB()
{
        # Sum up all memory taken by imapsync runs, in KiB.
        pids_of_imapsync_running=`pids_of_imapsync_running`
        if test -n "$pids_of_imapsync_running" ; then
                ps -o rss -p $pids_of_imapsync_running | sed 1,1d | datamash sum  1
        else
                echo 0
        fi
}

echoq memory_used_by_all_processes_KiB
memory_used_by_all_processes_KiB()
{
        # Sum up all memory taken by all processes, in KiB.
        ps -o rss -p `pgrep -f '.*'` | sed 1,1d | datamash sum  1
}

memory_used_by_all_processes_var()
{
        memory_used_by_all_processes_KiB=`memory_used_by_all_processes_KiB`
        memory_used_by_all_processes_bytes=`KiBytes_to_Bytes $memory_used_by_all_processes_KiB`
        # $ram_total_in_bytes is fixed so done at the beginning
        memory_used_by_all_processes_percent=`ratio_percent $memory_used_by_all_processes_bytes $ram_total_in_bytes`
        memory_used_by_all_processes_human=`bytestohuman $memory_used_by_all_processes_bytes`
        
}

echoq memory_used_by_all_processes
memory_used_by_all_processes()
{
        memory_used_by_all_processes_var
        echo memory_used_by_all_processes_KiB=$memory_used_by_all_processes_KiB
        echo memory_used_by_all_processes_bytes=$memory_used_by_all_processes_bytes
        echo memory_used_by_all_processes_percent=$memory_used_by_all_processes_percent
        echo memory_used_by_all_processes_human=$memory_used_by_all_processes_human
}

memory_available_var()
{
        memory_used_by_all_processes_KiB=`memory_used_by_all_processes_KiB`
        memory_used_by_all_processes_bytes=`KiBytes_to_Bytes $memory_used_by_all_processes_KiB`
        memory_available_in_bytes=`expr $ram_total_in_bytes - $memory_used_by_all_processes_bytes`
        memory_available_human=`bytestohuman $memory_available_in_bytes`
        memory_available_in_slots=`expr $memory_available_in_bytes / 250000000`
}

memory_available()
{
        memory_available_var
        echo memory_available_in_bytes=$memory_available_in_bytes
        echo memory_available_human=$memory_available_human
        echo memory_available_in_slots=$memory_available_in_slots
}

memory_available_in_slots()
{
        memory_available_var
        echo $memory_available_in_slots
}

memory_used_by_all_imapsync_var()
{
        memory_used_by_all_imapsync_KiB=`memory_used_by_all_imapsync_KiB`
        memory_used_by_all_imapsync_bytes=`KiBytes_to_Bytes $memory_used_by_all_imapsync_KiB`
        # $ram_total_in_bytes is fixed so done at the beginning
        memory_used_by_all_imapsync_percent=`ratio_percent $memory_used_by_all_imapsync_bytes $ram_total_in_bytes`
        memory_used_by_all_imapsync_human=`bytestohuman $memory_used_by_all_imapsync_bytes`
        
}

echoq memory_used_by_all_imapsync
memory_used_by_all_imapsync()
{
        memory_used_by_all_imapsync_var
        echo memory_used_by_all_imapsync_KiB=$memory_used_by_all_imapsync_KiB
        echo memory_used_by_all_imapsync_bytes=$memory_used_by_all_imapsync_bytes
        echo memory_used_by_all_imapsync_percent=$memory_used_by_all_imapsync_percent
        echo memory_used_by_all_imapsync_human=$memory_used_by_all_imapsync_human
}




oom_immune_pid()
{
        pidtoimmune=${1:-$$}
        # $2 is the immume value or -12
        test -f /proc/$pidtoimmune/oom_adj || return
        echo -n "$pidtoimmune "
        cat /proc/$pidtoimmune/oom_* | tr '\n' ' '
        { test -f /proc/$pidtoimmune/oom_adj && echo ${2:-"-12"} > /proc/$pidtoimmune/oom_adj && echo -n ">>> " && cat /proc/$pidtoimmune/oom_adj ; }
        
}


echoq oom_immune_imapsync_running
oom_immune_imapsync_running() {
    for pidsimapsync in  `pids_of_imapsync_running` 
    do
        oom_immune_pid $pidsimapsync $1
    done
}

pidapache()
{
        test -f /var/run/apache2/apache2.pid && cat /var/run/apache2/apache2.pid && return
        test -f /var/run/httpd/httpd.pid     && cat /var/run/httpd/httpd.pid     && return
}


echoq oom_immune_apache
oom_immune_apache() {
        pidapache=`pidapache`
        echo "[$pidapache]"
        test -n "$pidapache" && oom_immune_pid "$pidapache" "$1" || echo no apache running
}

apache_status_restart()
{
echo 'systemctl --no-pager status apache2 ; systemctl --no-pager restart  apache2 ; systemctl --no-pager status apache2'
echo 'systemctl --no-pager status httpd   ; systemctl --no-pager restart  httpd   ; systemctl --no-pager status httpd'
}

echoq nb_migrations_launched
nb_migrations_launched() {
    list_all_logs | egrep "$1" | egrep -o [a-f0-9]{40} | sort | uniq -c | wc -l
}

nb_migrations_launched_old() {
    /bin/ls . | egrep [a-f0-9]{40} | sort | uniq | wc -l
}



echoq current_stats
current_stats() {
    echo -n "Nb accounts: "; nb_migrations_launched
    echo -n "Nb imapsync running: "; number_and_pids_of_imapsync_running
    # dstat, Linux 
    dstat --version > /dev/null 2>&1 && dstat -l -n -cdgyms 60 1 && return
    # no dstat, FreeBSD
    dstat --version > /dev/null 2>&1 || vmstat 2 15 && return
    #clear
}

echoq watch_current_stats
watch_current_stats() {
    export -f current_stats
    # watch -n 120 current_stats
    while : ; do 
        clear
        oom_immune_imapsync_running
        current_stats
		sleep 6
    done
}


echoq 'grep_in_all_logs str1 str2 ... # up to str5. Results in mtime order of logfiles'
grep_in_all_logs() {
    grep_file=grep_`echo "$1 $2 $3 $4 $5" | tr ' ' '_' | tr -cd '0-9a-zA-Z_.\n'`.txt
    echo results in "${grep_file}"
    list_all_logs | tr '\n' '\000'| xargs -0 egrep -E -i "$1" | egrep -i "$2" | egrep -i "$3" | egrep -i "$4" | egrep -i "$5" | tee "${grep_file}.tmp"
    mv "${grep_file}.tmp" "${grep_file}"
}

echoq grep_in_logs_manual
grep_in_logs_manual()
{
        cat << EOF
list_all_logs /2021_11 | tail -666 | tr '\n' '\000'| xargs -0 egrep -i LALALA | tee grep_LALALA.txt
EOF
}

echoq 'grep_stats_from_list_all_logs # long'
grep_stats_from_list_all_logs() {
    echo results in grep_stats.txt
    # remove empty lines because it would grep all lines of all logs
    sed -i".bak" '/^[[:space:]]*$/d' stat_patterns.txt
    list_all_logs | tr '\n' '\000'| xargs -0 egrep -i -f stat_patterns.txt > grep_stats.txt.tmp 
    mv grep_stats.txt.tmp grep_stats.txt
}


tests_pattern_filename()
{
        run_test test ""     = "`pattern_filename`" 
        run_test test "abcd" = "`pattern_filename abcd`" 

        run_test test "ab_0123__4567_cd" = "`pattern_filename ab[0123][4567]cd`" 


        run_test test "ab_cd" = "`pattern_filename ab cd`" 
        run_test test "ab_cd" = "`pattern_filename   ab   cd  `" 

        run_test test "abcd"      = "`pattern_filename ab\&cd`" 

        run_test test "abcd"      = "`pattern_filename ab""cd`" 
        run_test test "abcd"      = "`pattern_filename ab""cd`"
        run_test test "ab_cd"     = "`pattern_filename ab" "cd`"
        run_test test "ab__cd"    = "`pattern_filename ab"  "cd`"
        run_test test "ab__cd"    = "`pattern_filename ab "" cd`"
        run_test test "ab___cd"   = "`pattern_filename ab " " cd`"
        run_test test "ab____cd"  = "`pattern_filename ab "  " cd`"
        run_test test "ab____cd"  = "`pattern_filename ab  "  "  cd`"
        run_test test "ab_____cd" = "`pattern_filename ab  "   "  cd`"

        run_test test "a_b_c_d" = "`pattern_filename a b c d`" 

}

pattern_filename()
{
        echo "$@" | tr ' .[]' '____' | tr -cd '0-9a-zA-Z_.'
}



echoq 'grep_stats_from_list_log_matching lognamepattern # time depending on lognamepattern'
grep_stats_from_list_log_matching() {
    pattern="$1"
    pattern_filename=`pattern_filename "$pattern"`
    results_filename=grep_stats_"$pattern_filename".txt
    echo results in "$results_filename"
    # remove empty lines because it would grep all lines of all logs
    sed -i".bak" '/^[[:space:]]*$/d' stat_patterns.txt
    list_log_matching "$pattern" | tr '\n' '\000'| xargs -0 egrep -i -f stat_patterns.txt > "$results_filename".tmp 
    mv "$results_filename".tmp  "$results_filename"
}



grep_any() {
        file=`statsfile "$1" "$2"`
        pattern_filtered=`pattern_filename "$2"`
        echo $file
        if test -f grep_stats_"$pattern_filtered".txt ; then
                egrep -i "$1" grep_stats_"$pattern_filtered".txt > $file.tmp
                mv $file.tmp $file
        else
                echo File not found: grep_stats_"$pattern_filtered".txt
        fi
}

grep_load() {
        file=`statsfile "Load" "$1"`
        pattern_filtered=`pattern_filename "$1"`
        echo $file
        if test -f grep_stats_"$pattern_filtered".txt ; then
                egrep -o 'Load is ..?\... ..?\... ..?\... .*' grep_stats_"$pattern_filtered".txt > $file.tmp
                mv $file.tmp $file
        else
                echo File not found: grep_stats_"$pattern_filtered".txt
        fi
}


stat_patterns_list() {
   cat stat_patterns.txt | sed '/^[[:space:]]*$/d' | tr -d '^'
}


echoq 'grep_all_stat_from_patterns_list lognamepattern # long'
grep_all_stat_from_patterns_list() {
    grep_load "$1"
    stat_patterns_list | while read k; do grep_any "$k" "$1"; done
}


sum_first_column_G_HTTP_USER_AGENT_sorted()
{
        awk '{sum += $1} END {print sum}'  G_HTTP_USER_AGENT_${1}_sorted.txt
}

stat_useragent_X()
{
         grep -o 'HTTP_USER_AGENT.*' G_HTTP_USER_AGENT_$1.txt \
         | tail -10000000 | sort | egrep -o -w 'Mozilla/5.0 \([^;]+' \
         | sort | egrep -o '\([a-zA-Z]+' | sort  | uniq -c | sort -g \
         | grep -v KHTML | tr -d '(' > G_HTTP_USER_AGENT_${1}_sorted.txt
}

echoq 'percent_stat_useragent_X'
percent_stat_useragent_X()
{
        stat_useragent_X "$1"
        sum_first_column_G_HTTP_USER_AGENT=`sum_first_column_G_HTTP_USER_AGENT_sorted $1`
        { while read num_useragent useragent ; do
                #echo KK $num_useragent $useragent
                PerCent=`echo "scale=2; 100*$num_useragent/$sum_first_column_G_HTTP_USER_AGENT" | bc -l`
                echo "$useragent $PerCent %    ( $num_useragent / $sum_first_column_G_HTTP_USER_AGENT )"
        done 
        } < G_HTTP_USER_AGENT_${1}_sorted.txt

}


stat_load()
{
	echo G_Load_$1.txt
    echo -n 'Load        1 min      5 min   15 min ' ; grep -o 'on.*cores' G_Load_$1.txt |sort| uniq
    echo -n 'Load min:    ' ; LC_NUMERIC=C datamash --format=%3.1f -W min    3 min    4 min    5 < G_Load_$1.txt
    echo -n 'Load q1:     ' ; LC_NUMERIC=C datamash --format=%3.1f -W q1     3 q1     4 q1     5 < G_Load_$1.txt
    echo -n 'Load median: ' ; LC_NUMERIC=C datamash --format=%3.1f -W median 3 median 4 median 5 < G_Load_$1.txt
    echo -n 'Load mean:   ' ; LC_NUMERIC=C datamash --format=%3.1f -W mean   3 mean   4 mean   5 < G_Load_$1.txt
    echo -n 'Load q3:     ' ; LC_NUMERIC=C datamash --format=%3.1f -W q3     3 q3     4 q3     5 < G_Load_$1.txt
    echo -n 'Load max:    ' ; LC_NUMERIC=C datamash --format=%3.1f -W max    3 max    4 max    5 < G_Load_$1.txt
}

echoq stat_exit_value
stat_exit_value()
{
        statsfile=`statsfile Exiting_with_return_value "$1"`
        good_lines_nb=`grep '(EX' $statsfile | wc -l | tr -d ' '`
        grep '(EX' "$statsfile" \
        | datamash  --sort groupby 6 -W count 5 \
        | awk -v good_lines_nb=$good_lines_nb  \
                '{ printf "%.2g%%  %s\n",  100*$2/good_lines_nb,  $1 }' \
        | sort -n
}

echoq stat_exit_value_by_value
stat_exit_value_by_value()
{
        statsfile=`statsfile Exiting_with_return_value "$1"`
        datamash  --sort groupby 5 -W count 5 < "$statsfile"
}



datamash_file_op_index() { 
    file="$1"
    op="${2:-mean}"
    index="${3:-4}" # the four field by default
    format="${4:-%16.1f}" # --format=%16.1f by default
    func="${5:-}"
    val_datamash_file_op_index=`LC_ALL=C datamash --format="$format" -W "$op" "$index" < "$file"`
    func_return=
    test -n "$func" && func_return=`eval $func $val_datamash_file_op_index`
    echo "$val_datamash_file_op_index" $func_return
} 

stat_any() {
    echo stat_any "$@"
    test -f "$1" || { echo "usage: stat_any file index" ; return 1 ; }
    file="$1"
    index=${2:-4} # the four field by default
    func="${3:-}"
    for op in \
        "min     " \
        "perc:10 " \
        "q1      " \
        "median  " \
        "mean    " \
        "q3      " \
        "perc:90 " \
        "max     " \
        
    do
        echo -n "$file $index $op " ; datamash_file_op_index $file $op $index %16.1f $func
    done
    echo
}



echoq stat_all
stat_all()
{
    stat_load "$1" ; echo
    
    echo G_REMOTE_ADDR_$1.txt
    egrep -o 'REMOTE_ADDR is .*' G_REMOTE_ADDR_$1.txt | sort -g |  uniq -c | sort -g | tail -5

    echo
    echo G_REMOTE_HOST_$1.txt
    egrep -o 'REMOTE_HOST is .*' G_REMOTE_HOST_$1.txt | sort -g |  uniq -c | sort -g | tail -5
    
    echo
    echo G_HTTP_COOKIE_$1.txt 
    egrep -o 'imapsync_runs=[0-9]+'  G_HTTP_COOKIE_$1.txt  | egrep -o '[0-9]+' | sort -n | tail -1
    
    # stat_any  G_HTTP_REFERER.txt
    echo
    echo G_HTTP_REFERER_$1.txt 
    egrep -o 'HTTP_REFERER is .*' G_HTTP_REFERER_$1.txt | sort -g |  uniq -c | sort -g
    
    echo
    echo G_Host1_IMAP_server_$1.txt
    cat  G_Host1_IMAP_server_$1.txt | datamash -s -W -g 4 count 4 | awk '{ print $2 " " $1 }' | sort -g | tail -5

    echo
    echo G_Host2_IMAP_server_$1.txt
    cat  G_Host2_IMAP_server_$1.txt | datamash -s -W -g 4 count 4 | awk '{ print $2 " " $1 }' | sort -g | tail -5


    echo
    stat_any  G_Host1_Nb_messages_$1.txt
    stat_any  G_Host2_Nb_messages_$1.txt
    stat_any  G_Messages_transferred_$1.txt
    stat_any  G_Messages_skipped_$1.txt
    stat_any  G_Messages_found_in_host1_not_in_host2_$1.txt 9
    stat_any  G_Messages_found_in_host2_not_in_host1_$1.txt 9
    
    # stat_any  G_Folders_synced.txt
    egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced_$1.txt | egrep -o '^[0-9]+' > G_Folders_synced__$1.txt
    egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced_$1.txt | egrep -o '[0-9]+/[0-9]+' | egrep -o '[0-9]+$' > G_Folders_total_seen_$1.txt
    stat_any  G_Folders_synced__$1.txt 1
    stat_any  G_Folders_total_seen_$1.txt 1

    #
    stat_any  G_Transfer_time_$1.txt
    stat_any  G_Host1_Total_size_$1.txt
    stat_any  G_Host2_Total_size_$1.txt
    stat_any  G_Total_bytes_transferred_$1.txt 5
    stat_any  G_Message_rate_$1.txt
    stat_any  G_Average_bandwidth_rate_$1.txt 5
    stat_any  G_Biggest_message_$1.txt
    stat_any  G_Detected_errors_$1.txt 2 
    #stat_any  G_Exiting_with_return_value.txt 5 # GROUP
    stat_any  G_Memory_consumption_at_the_end_$1.txt 7
    stat_any  G_Memory_consumption_at_the_end_$1.txt 10
    #stat_any  G_failure_Error_login.txt
    
    echo cpu time
    stat_any  G_CPU_time_and_cpu_$1.txt  6
    echo '%allcpus'
    stat_any  G_CPU_time_and_cpu_$1.txt 10
    
    echo G_Host1_banner_$1.txt
    server_survey_percent G_Host1_banner_$1.txt  | tail -6
    
    echo
    echo G_Host2_banner_$1.txt
    server_survey_percent G_Host2_banner_$1.txt  | tail -6
    
    echo
    echo USER_AGENT
    percent_stat_useragent_X $1 ; 
    
    echo
    echo EXIT values
    stat_exit_value $1
    
    echo
    echo "Data made at" `date -r grep_stats_$1.txt`
}

stat_transfer_time_mean()
{
        statsfile=`statsfile Transfer_time "$1"`
        datamash_file_op_index "$statsfile" mean
}

stat_throuput_since_day_one_in_days()
{
        number_of_syncs=`number_of_syncs "$1"`
        days_since_first_use=`days_since_first_use "$1"`
        c "$number_of_syncs / $days_since_first_use"
}

stat_queue_mean_old()
{
        stat_throuput_since_day_one_in_days=`stat_throuput_since_day_one_in_days`
        stat_transfer_time_mean=`stat_transfer_time_mean "$1"`
        stat_queue_mean_raw=`c "$stat_throuput_since_day_one_in_days * $stat_transfer_time_mean / 3600 / 24"`
        LC_ALL=C printf "%2.2f\n" $stat_queue_mean_raw
}


stat_queue_mean()
{
        first_log=`first_log "$1"`
        last_log=`last_log "$1"`
        number_of_syncs=`number_of_syncs "$1"`
        seconds_between_files=`seconds_between_files $first_log $last_log`
        stat_transfer_time_mean=`stat_transfer_time_mean "$1"`
        stat_queue_mean_raw=`c "$number_of_syncs / $seconds_between_files * $stat_transfer_time_mean"`
        LC_ALL=C printf "%2.2f\n" $stat_queue_mean_raw
}


first_log()
{
        list_all_logs "$1"_ | grep -a /"$1" | head -1
}

last_log()
{
        list_all_logs "$1"_ | grep -a /"$1" | tail -1
}

start_date()
{
        first_log=`first_log "$1"`
        date -r "$first_log"
}

end_date()
{
        last_log=`last_log "$1"`
        date -r "$last_log"
}



echoq dirs_of_syncs_finished_recently
dirs_of_syncs_finished_recently() {
    find .  -maxdepth 1 -mtime "${1:--1}" | grep -v "385d7a4d8d428d7aa2b57c8982629e2bd67698ed" | egrep [a-f0-9]{40} | while read d; do
        test -f "$d" && continue
        test -f  $d/imapsync.pid  && continue
        echo $d
    done
}

echoq 'logfiles_finished_recently -3 # less than 3 days, default is like -1'
logfiles_finished_recently()
{
    {
    # +2 more than 2 days ago
    # -3 less than 3 days ago
    #  7 exactly   7 days ago
    #set -x
    find .  -maxdepth 1 -mtime "${1:--1}" | grep -v "385d7a4d8d428d7aa2b57c8982629e2bd67698ed" |  egrep [a-f0-9]{40} | while read d; do
        test -f "$d" && continue
        test -f  $d/imapsync.pid  && continue
        test -d $d/ || continue
        ls -trb `find $d/ -type f -mtime "${1:--1}" | grep \.txt`
    done
    } 
}


last_dirs_written()
{
        ls -tr | tail -1800
}

last_file_written_in_dir()
{
        ls -trd $1/*.txt |tail -1
}

is_dir_running_imapsync()
{
        test -d "$1" || return 1
        test -f  "$1/imapsync.pid"  && PID=`head -1 "$1/imapsync.pid"` &&
        ps -p $PID -o comm= > /dev/null
}

echoq logfiles_running
logfiles_running()
{
        last_dirs_written | while read d
        do
                is_dir_running_imapsync "$d" && 
                        last_file_written_in_dir "$d"
        done
}

epoch_of_file()
{
        date -r "$1" +%s 
}

epoch_of_now()
{
        date +%s 
}

is_file_older_than()
{
        # return 1 if not exist or recent
        # return 0 if older than "$2" seconds or 15 minutes (900 secondes)
        test -f "$1" || return 1
        epoch_file=`epoch_of_file "$1"`
        epoch_now=`epoch_of_now`
        epoch_diff=`expr $epoch_now - $epoch_file`
        #echo "$epoch_now - $epoch_file = $epoch_diff"
        if test "${2:-900}" -lt "$epoch_diff" 
        then
                #echo older than $2
                return 0
        else
                #echo newer than $2
                return 1
        fi
}


newer()
{
	test -f "$2" || return 0
	test "$1" -nt "$2"
}

pids_of_imapsync_not_writing_since_x_secondes()
{
        x_secondes=${1:-900} # 15 minutes by default
        last_dirs_written | while read d
        do
                is_dir_running_imapsync "$d" && 
                is_file_older_than `last_file_written_in_dir "$d"` "$x_secondes" &&
                head -1 "$d/imapsync.pid" | tr  '\n' ' ' 
                        
        done

}

kill_HUP_pids_of_imapsync_not_writing_since_x_secondes()
{
        pids_not_writing=`pids_of_imapsync_not_writing_since_x_secondes ${1:-900}` 
        test -n "$pids_not_writing" && echo kill -HUP "$pids_not_writing" # && kill -HUP "$pids_not_writing"
}


watch_logfiles_running_old() {
    # the "tail --pid=" option does not exist on FreeBSD, it's GNU/Linux
    while date; do
        inotifywait /var/tmp/imapsync_cgi -e create 2>/dev/null &
        PID_inotifywait=$!
        logfiles_running | xargs -d'\n' tail --pid=$PID_inotifywait -f -v
        echo "NEW SYNC IS RUNNING"
        echo "Syncs running: "; number_and_pids_of_imapsync_running
        sleep 3
    done
}

watch_logfiles_running_old2() {
    while date; do
        kill $PID_inotifywait
        inotifywait /var/tmp/imapsync_cgi -e create 2>/dev/null &
        PID_inotifywait=$!
        kill_tail_logfiles_running
        tail_logfiles_running
        wait $PID_inotifywait 
        kill_tail_logfiles_running 
        echo "NEW SYNC IS RUNNING"
        echo "Syncs running: "; number_and_pids_of_imapsync_running
        sleep 3
    done
}




tail_logfiles_running() {
        logfiles_running=`logfiles_running`
        test -n "$logfiles_running" && tail -f $logfiles_running
        #PID_tail_logfiles_running=$!
        #fg
}

echoq watch_logfiles_running
watch_logfiles_running() {
        tail_logfiles_running
}

kill_tail_logfiles_running() {
        kill $PID_tail_logfiles_running
}



echoq watch_new_runs
watch_new_runs() {
    while { date; echo -n "Nb syncs currently: " ;  number_and_pids_of_imapsync_running ; } do
        inotifywait  . -e create 2>/dev/null | { read path action f 
        echo $f
        sleep 2
        test -f  $f/imapsync.pid  && PID=`head -1 $f/imapsync.pid` && echo PID $PID
        echo -e '\a'  
        }
    done
}

echoq pidfiles_running_and_not_running
pidfiles_running_and_not_running() {
    ls -tr | while read f; do
        test -f  $f/imapsync.pid  && PID=`head -1 $f/imapsync.pid` && echo -n "$PID " &&
        { ps -p $PID -o comm= | tr  '\n' ' '  && { test -f /proc/$PID/oom_score &&  
            { echo -12 > /proc/$PID/oom_adj ; } && echo -n "oom_score " && cat /proc/$PID/oom_score | tr  '\n' ' ' ; : ; }  
        } &&
        { ls -tr $f/*.txt |tail -1 ;  } 
     done
}

pidfile_dandling() {
        pidfile_dandling_DIR=$1
        test -d $pidfile_dandling_DIR || return 2
        test -f  $pidfile_dandling_DIR/imapsync.pid || return 3
        pidfile_dandling_PID=`head -1 $pidfile_dandling_DIR/imapsync.pid` 
        #echo "$pidfile_dandling_PID"
        test -n "$pidfile_dandling_PID" || return 4
        test "$pidfile_dandling_PID" -ge 1 || return 5
        if ! ps -p "$pidfile_dandling_PID" -o comm= > /dev/null ; then
            #echo -n "DANDLING $pidfile_dandling_DIR/imapsync.pid "
            #echo "# PID $pidfile_dandling_PID"
            return 0
	fi
        return 99

}

echoq pidfiles_not_running
pidfiles_not_running() {
    ls -tr | while read f; do
        if  pidfile_dandling "$f" ; then
            pidfiles_not_running_PID=`head -1 $f/imapsync.pid`
            echo -n "rm $f/imapsync.pid # "
            { ls -tr $f/*.txt 2>/dev/null |tail -1 ;  } | tr  '\n' ' ' 
            echo "# PID $pidfiles_not_running_PID"
            #head -2 $f/imapsync.pid
	fi
    done
}


first_use() {
    test -f first_use && cat first_use && return
    echo "${1:-2017} ${2:-01} ${3:-09}"
}

filedate()
{
        test FreeBSD = `uname -s` && gdate -r "$1" '+%Y %m %d'
        test Linux   = `uname -s` &&  date -r "$1" '+%Y %m %d'
}


days_between_files()
{
        epoch1=`epoch_of_file "$1"`
        epoch2=`epoch_of_file "$2"`
        echo epoch1 $epoch1 epoch2 $epoch2
        expr \( $epoch2 - $epoch1 \) / 3600 / 24
}


seconds_between_files()
{
        epoch1=`epoch_of_file "$1"`
        epoch2=`epoch_of_file "$2"`
        expr \( $epoch2 - $epoch1 \)
}



days_since_first_use() {
    first_use=`first_use "$@"`
    #echo $[$[$(date +%s)-$(epoch_of_y_m_d_h_m_s 2017 01 09 00 00 00)]/60/60/24]
    echo $[$[$(date +%s)-$(epoch_of_y_m_d_h_m_s $first_use 00 00 00)]/60/60/24]
}


epoch_of_y_m_d_h_m_s() {
        date -v -1d > /dev/null 2>&1 && date -u -v ${1:-1970}y -v ${2:-1}m -v ${3:-1}d -v ${4:-0}H -v ${5:-0}M -v ${6:-0}S +%s && return
        date --date="1 day ago" > /dev/null && date -u -d "${1:-1970}-${2:-1}-${3:-1} ${4:-0}:${5:-0}:${6:-0}" +%s && return
}

date_x_days_ago() {
        date -v -1d > /dev/null 2>&1 && date -u -v -${1:-0}d "+%Y-%m-%d %a" && return
        date --date="1 day ago" > /dev/null && date -u --date="${1:-0} day ago" "+%Y-%m-%d %a" && return
}

seconds_to_days_hours() {
    #eval "echo $(date -ud "@${1:-0}" +'$((%s/3600/24)) days %_H hours %_M min %_S sec')"
    date -v -1d > /dev/null 2>&1        && eval "echo $(date -ur "${1:-0}" +'$((%s/3600/24)) days %_H hours %_M min %_S sec')" && return
    date --date="1 day ago" > /dev/null && eval "echo $(date -ud "@${1:-0}" +'$((%s/3600/24)) days %_H hours %_M min %_S sec')" && return
}

seconds_to_days_hours_echo() {
    date -v -1d > /dev/null 2>&1        && echo "echo $(date -ur "${1:-0}" +'$((%s/3600/24)) days %_H hours %_M min %_S sec')" && return
    date --date="1 day ago" > /dev/null && echo "echo $(date -ud "@${1:-0}" +'$((%s/3600/24)) days %_H hours %_M min %_S sec')" && return
}

printf_this_one_div10()
{ 
        num=$1
        printf "% $((num/10))s\n" $1
}

echoq 'runs_per_day 7 # last 7 days'
runs_per_day() {
    historic_start=`days_since_first_use`
    start=${1:-$historic_start}
    for cc in `count 0 $start`; do
        DATE=`date_x_days_ago $cc`
        # find on FreeBSD finds nothing with -mtime 0
        test FreeBSD = `uname -s` && cc=`expr 1 + $cc`
        runs_this_day=`find . -maxdepth 1 -mtime $cc   -ls |wc -l`
        echo -n "$DATE $cc days ago: " ; printf_this_one_div10 $runs_this_day
    done
}

echoq summary_run
summary_run() {
    for summary_run_DIR in "$@"; do
        echo Analysing $summary_run_DIR
        echo -n "Nb logs: "; ls $summary_run_DIR/*.txt | wc -l 
        summary_run_LOGS_LIST=`ls $summary_run_DIR/*.txt`
        echo -n "List logs: "; echo $summary_run_LOGS_LIST
        #echo connect failure
        summary_run_CONNECT_FAIL=`grep -i 'failure: can not open imap connection on' $summary_run_DIR/*.txt|wc -l`
	echo CONN $summary_run_CONNECT_FAIL
        #echo login failure
        grep -i 'failure: Error login on' $summary_run_DIR/*.txt
        #echo Differences 
        grep -i "difference host2 - host1" $summary_run_DIR/*.txt
    done
}



logs_nb() {
    logs_nb_DIR="$1"
    logs_nb_LOGS_LIST="$logs_nb_DIR"/*.txt
}



vnstat_init() {
        test FreeBSD = `uname -s` && VNSTATI_DIR=/usr/local/www/apache24/data/vnstat
        test Linux = `uname -s` && VNSTATI_DIR=/var/www/html/vnstat
        test -d $VNSTATI_DIR || mkdir -p $VNSTATI_DIR
}

echoq vnstat_gen
vnstat_gen() {
        vnstat_init || return 
        for opt in s h hg hs d m y t vs 5 5g ; do
                test "$1" && echo vnstati -$opt -o $VNSTATI_DIR/vnstat_${opt}.png
                vnstati -$opt -o $VNSTATI_DIR/vnstat_${opt}.png
        done
}


echoq vnstat_index_hs
vnstat_index_hs()
{
        (
        vnstat_init || return 
        cd $VNSTATI_DIR/ || return 
        for f in `ls -r ./*/vnstat_hs.png`
        do 
                echo '<img src="'$f'" border="0" alt="hourly"><br>'
        done > index_hs.html
        )
}

echoq vnstat_archive
vnstat_archive() {
        (
        vnstat_gen "$1" || return 
        
        now_ymdhms=`date +%Y_%m_%d_%H_%M_%S` || return 
        mkdir $VNSTATI_DIR/$now_ymdhms/ || return 
        cd $VNSTATI_DIR/$now_ymdhms/ || return 
        test "$1" && pwd
        cp -a ../*.png ../*.html .
        
        )
        test "$1" && pwd
}



echoq dstat_csv
dstat_csv() {
    #dstat   -l -n -cdgyms 60 1
    dstat -t -l -n -cdgyms --output dstat.csv 60
}

echoq 'ratio_killed_by_TERM -3 # last 3 days'
ratio_killed_by_TERM() {
	logfiles_finished_recently=`logfiles_finished_recently $1`
        nb_logfiles_finished_recently=`echo $logfiles_finished_recently | wc -w`
	echo -n "Got a signal TERM: " && echo $logfiles_finished_recently | xargs grep -i 'Got a signal TERM' | wc -l
	echo -n "Got a signal     : " && echo $logfiles_finished_recently | xargs grep -i 'Got a signal'      | wc -l
	echo -n "Among finished   : " && echo $nb_logfiles_finished_recently
        echo "logfiles_finished_recently $1 | xargs grep -i 'Got a signal TERM' "
}

echoq 'nb_syncs_badly_finished -1 # last 1 day'
nb_syncs_badly_finished()
{
        logfiles_finished_recently=`logfiles_finished_recently $1`
        nb_logfiles_finished_recently=`echo $logfiles_finished_recently | wc -w | tr -d ' '`
        nb_syncs_badly_finished=`echo $logfiles_finished_recently | xargs grep -i 'Exiting with return value' | grep -v 'return value 0' | wc -l `
        echo $nb_syncs_badly_finished / $nb_logfiles_finished_recently \
        | awk '{ printf "%s %.2g%%  %s\n", "Total:", 100*$1/$3,  $0 }'
        
        echo $logfiles_finished_recently | xargs grep -i 'Exiting with return value' \
        | grep -v 'return value 0' | grep -o 'Exiting with return value.*)' \
        | sort | uniq -c | sort -n \
        | awk -v nb_logfiles_finished_recently=$nb_logfiles_finished_recently \
        '{ printf "%.2g%%  %s\n",  100*$1/nb_logfiles_finished_recently,  $0 }'
cat <<EOF
logfiles_finished_recently $1 | xargs grep -i 'Exiting with return value' | grep -v 'return value 0 ' | cut -d: -f1 | xargs  tail -11
EOF
}

echoq 'referrer_of_x  /var/log/apache/access.log_2022*.gz  | sort | uniq -c | sort -n'
referrer_of_x() {
        zegrep -h -s -o 'GET /X/? .*http[^"]+'  "${@:-/var/log/apache/access.log}" | grep -o 'http.*'
}


biggest_message_seen() {
    statsfile=`statsfile Biggest_message $1`
    cat "$statsfile" | grep -v Memory | datamash -W max 4  | xargs bytestohuman
}

biggest_message_transferred() {
# With this, the "Biggest message" may be not be transferred by imapsync itself.
    statsfile=`statsfile Biggest_message $1`
    grep 'Host2 Biggest message' < "$statsfile" | datamash -W max 4  | xargs bytestohuman
}


biggest_bandwidth_rate() {
    statsfile=`statsfile Average_bandwidth_rate $1`
    datamash_file_op_index "$statsfile" max 5 | tr -d ' ' | tr '\n' ' '
    echo KiB/s
}

average_bandwidth_rate() {
    statsfile=`statsfile Average_bandwidth_rate $1`
    datamash_file_op_index "$statsfile" mean 5 | tr -d ' ' | tr '\n' ' '
    echo KiB/s
}



max_number_of_messages_transferred() {
        statsfile=`statsfile Messages_transferred $1`
        datamash_file_op_index "$statsfile" max 4 "%.0f"
}

max_number_of_messages_skipped() {
        statsfile=`statsfile Messages_skipped $1`
        datamash_file_op_index "$statsfile" max 4 "%.0f"
}


echoq number_of_X_users
number_of_X_users()
{
        statsfile=`statsfile REMOTE_ADDR $1`
        test -f $statsfile || { echo No exists $statsfile ; return ; }
        datamash_file_op_index $statsfile unique 3 | tr , '\n'  | wc -l
}



summary_compute_old() {
        list_all_logs_generate \
        && grep_stats_from_list_all_logs \
        && grep_all_stat_from_patterns_list \
        && summary_display
}


echoq "summary_compute `date +%Y_%m_%d` # time depending on lognamepattern"
summary_compute() {
        grep_stats_from_list_log_matching "$1" \
        && grep_all_stat_from_patterns_list "$1" \
        && stat_all "$1" && summary_display "$1"
}


number_of_syncs()
{
        list_all_logs | egrep -a "$1"_ | wc -l
}


count_expression()
{
        #echo count_expression "[$1]" "[$2]"
        egrep -- "$1" "$2" | wc -l
}


patterns_alone_file_generate()
{
        patterns_file="$1"
        patterns_alone_file="$2"
        
        > $patterns_alone_file
        cat $patterns_file | 
                while read imap_server pattern
                do
                        echo "$pattern" >> $patterns_alone_file
                done

}

count_imap_server_all()
{
        count_imap_server_all=0
        cat $patterns_file | 
                while read imap_server pattern
                do
                        #echo count_imap_server "$pattern" "$banners_files"
                        count_imap_server=`count_expression "$pattern" "$banners_files"`
                        count_imap_server_all=`expr $count_imap_server_all + $count_imap_server`
                        echo $count_imap_server_all
                done

}

#echoq server_survey_percent
server_survey_percent()
{
        
        banners_files=${1:-G_Host1_banner.txt}
        patterns_file=${2:-server_survey_patterns.txt}
        
        patterns_alone_file=${patterns_file}.alone.txt
        
        patterns_alone_file_generate $patterns_file $patterns_alone_file
                
        banners_counted=`egrep        -f $patterns_alone_file  $banners_files | wc -l | tr -d ' \n'`
        banners_not_counted=`egrep -v -f $patterns_alone_file  $banners_files | wc -l | tr -d ' \n'`
        banners_all=`cat $banners_files | wc -l | tr -d ' \n'`
        banners_all_verif=`expr $banners_not_counted + $banners_counted`
        cat $patterns_file | 
                while read imap_server pattern
                do
                        #echo count_imap_server "$pattern" "$banners_files"
                        count_imap_server=`count_expression "$pattern" "$banners_files"`
                        percent_imap_server=`echo "scale=2; 100 * $count_imap_server/$banners_all" | bc -l`
                        echo $percent_imap_server% : $count_imap_server " : $imap_server : " "[$pattern]"
                done | sort -n

}

#echoq server_survey
server_survey()
{
        banners_files=${1:-G_Host1_banner.txt}
        patterns_file=${2:-server_survey_patterns.txt}

        server_survey_percent $banners_files $patterns_file
        
        count_imap_server_all=`count_imap_server_all | tail -1`

        echo  $banners_files
        echo  "Banners counted sum  $count_imap_server_all"
        echo  "Banners counted      $banners_counted"
        echo  "Banners not counted  $banners_not_counted"
        echo  "Banners all          $banners_all"
        echo  "Banners all verif    $banners_all_verif = $banners_not_counted + $banners_counted"
        if test $count_imap_server_all != $banners_counted
                then echo WARNING count_imap_server_all $count_imap_server_all != $banners_counted banners_counted \
                        diff `expr $count_imap_server_all - $banners_counted`
        fi
        echo "server_survey $banners_files # finished"
}

echoq server_survey_next_pattern
server_survey_next_pattern()
{
        patterns_alone_file_generate  server_survey_patterns.txt  server_survey_patterns.txt.alone.txt
        grep -h -o 'banner:.*' G_Host?_banner.txt |sort | uniq -c | sort -g > banner_counted_sorted.txt
        egrep -v -f server_survey_patterns.txt.alone.txt banner_counted_sorted.txt

}

echoq server_survey_last_pattern
server_survey_last_pattern()
{
        banners_files1=${1:-G_Host1_banner.txt}
        banners_files2=${2:-G_Host2_banner.txt}
        
        tail -1 server_survey_patterns.txt > pattern_alone_file.txt
        server_survey $banners_files1 pattern_alone_file.txt
        server_survey $banners_files2 pattern_alone_file.txt
}

echoq server_survey_host1
server_survey_host1()
{
        server_survey G_Host1_banner.txt
}

echoq server_survey_host2
server_survey_host2()
{
        server_survey G_Host2_banner.txt
}


date_space() 
{
        date | tr -d '\n'
        echo -n " "
 
}

#echoq date_if_new_hour
date_if_new_hour() 
{
        min=`date +%M`
        sec=`date +%S`
        #echo $min $sec
        if test "00" = "$min" && test 6 -ge $sec 
        then 
                echo
                date_space
                sleep 1
        fi
}

echoq watch_number_of_imapsync_running
watch_number_of_imapsync_running()
{
        date_space
        while number_of_imapsync_running | tr -d ' \n'
        do 
                sleep 6
                date_if_new_hour
        done
}



#echoq number_of_bytes_sent_received_per_second_during
number_of_bytes_sent_received_per_second_during()
{
        # $1 : number of seconds to watch
        here_is_freebsd && number_of_bytes_sent_received_per_second_during_freebsd ${1:-1}
        here_is_linux   && number_of_bytes_sent_received_per_second_during_linux   ${1:-1}
}

number_of_bytes_sent_received_per_second_during_freebsd()
{
	netstat_result=`netstat -I em0 -w ${1:-1} -q 1 | tail -1`
	echo $netstat_result | awk -v sec="${1:-1}" '{ printf "%.0f", ($4+$7)/sec }'
}

tx_file_linux()
{
        test -r /sys/class/net/eth0/statistics/tx_bytes && echo /sys/class/net/eth0/statistics/tx_bytes
        test -r /sys/class/net/eno0/statistics/tx_bytes && echo /sys/class/net/eno0/statistics/tx_bytes
        test -r /sys/class/net/ens3/statistics/tx_bytes && echo /sys/class/net/ens3/statistics/tx_bytes
}

rx_file_linux()
{
        test -r /sys/class/net/eth0/statistics/rx_bytes && echo /sys/class/net/eth0/statistics/rx_bytes
        test -r /sys/class/net/eno0/statistics/rx_bytes && echo /sys/class/net/eno0/statistics/rx_bytes
        test -r /sys/class/net/ens3/statistics/rx_bytes && echo /sys/class/net/ens3/statistics/rx_bytes
}

number_of_bytes_sent_received_per_second_during_linux()
{
        tx_file_linux=`tx_file_linux`
        rx_file_linux=`rx_file_linux`
        tx_1=`cat $tx_file_linux`
        rx_1=`cat $rx_file_linux`
        sleep ${1:-1}
        tx_2=`cat $tx_file_linux`
        rx_2=`cat $rx_file_linux`
        echo "( $tx_2 - $tx_1 + $rx_2 - $rx_1 ) / ${1:-1}" | bc
}

div_1_by_2_or_zero()
{
        if test X"$2" = X"0"; then
                echo "0"
        else
                echo "$1 $2" | awk '{ printf "%.0f\n", $1/$2 }'
        fi
}


load_1_minute_linux()
{
        cat /proc/loadavg | cut -d' ' -f1
}

load_1_minute_freebsd()
{
        /sbin/sysctl vm.loadavg | egrep -o '[0-9]+\.[0-9]++' | head -1
}

load_1_minute()
{
        here_is_linux && load_1_minute_linux
        here_is_freebsd && load_1_minute_freebsd
}

KiBytes_to_Bytes()
{
        expr 1024 \* $1 
}

number_of_cores_live()
{
        here_is_linux && cat /proc/cpuinfo | grep ^processor | wc -l && return
        here_is_freebsd && sysctl -n kern.smp.cpus && return
        echo 1
}

number_of_cores()
{
        test -n "$number_of_cores" && echo "$number_of_cores" && return
        number_of_cores=`number_of_cores_live`
        echo "$number_of_cores"
}

cpu_slots_available()
{
        # rule of thumb: 4 imapsync processes per cpu
        echo "scale=0; ( `number_of_cores` - `load_1_minute` ) * 4 / 1 " | bc
}



maxint()
{
        test "$1" -gt "$2" && echo "$1" || echo "$2"
}

minint()
{
        test "$1" -gt "$2" && echo "$2" || echo "$1"
}

slots_available()
{
        minint "`cpu_slots_available`" "`memory_available_in_slots`"
}



bandwidth_nominal_bytes_per_second() 
{
	test "$hostname" = vp3 && echo 25000000 # 100 mbps + rx 100 mbps tx
	test "$hostname" = vp4 && echo 25000000 # 100 mbps + rx 100 mbps tx
	test "$hostname" = ks6 && echo 25000000 # 100 mbps + rx 100 mbps tx
	echo 25000000 # default 100 mbps + rx 100 mbps tx  
}

ratio_percent()
{
        expr 100 \* $1 / $2
}

echoq number_of_imapsync_running_bandwidth
number_of_imapsync_running_bandwidth()
{
        # Maybe I could do two number_of_imapsync_running one before
        # one after and average the two.
        number_of_imapsync_running=`number_of_imapsync_running`
        load_1_minute=`load_1_minute`
        bandwidth_bytes_per_sec=`number_of_bytes_sent_received_per_second_during ${1:-1}`
        bandwidth_bytes_per_sec_max=`bandwidth_nominal_bytes_per_second`
        bandwidth_percent=`ratio_percent $bandwidth_bytes_per_sec $bandwidth_bytes_per_sec_max`
        
        memory_used_by_all_processes_var # sets memory_used_by_all_processes_KiB memory_used_by_all_processes_bytes memory_used_by_all_processes_percent memory_used_by_all_processes_human
        memory_available_var # sets memory_available_in_slots
        
        ratio=`div_1_by_2_or_zero $bandwidth_bytes_per_sec $number_of_imapsync_running`
        date=`date_ymdhms`
        date_u=`LANG= date -u`
        bandwidth_bytes_per_sec_human=`bytestohuman $bandwidth_bytes_per_sec`
        ratio_human=`bytestohuman $ratio`
        
        slots_available=`slots_available`
        
        sleep 0.1
        echo "$date $number_of_imapsync_running $bandwidth_bytes_per_sec $ratio $bandwidth_bytes_per_sec_human $ratio_human $load_1_minute $memory_used_by_all_processes_KiB $memory_used_by_all_processes_percent $bandwidth_percent $slots_available"
        
        #echo "$hostname Load: $load_1_minute; Syncs: $number_of_imapsync_running; Mem: $memory_used_by_all_processes_human ($memory_used_by_all_processes_percent%); Bandwidth: $bandwidth_bytes_per_sec_human/s ($bandwidth_percent%);  Bandwidth per sync: $ratio_human/s; Date/time: $date_u;" >/var/tmp/imapsync_current_old_${1:-1}.txt

        printf "%8s Load: %4s; Syncs: %2s (%2s); Mem: %11s (%3s%%); Bandwidth: %11s/s (%3s%%); Bandwidth per sync: %11s/s; Date/time: %s;\n" \
		       "$hostname" "$load_1_minute" "$number_of_imapsync_running" "$slots_available" "$memory_used_by_all_processes_human" "$memory_used_by_all_processes_percent" "$bandwidth_bytes_per_sec_human" "$bandwidth_percent" "$ratio_human" "$date_u" > /var/tmp/imapsync_current_${1:-1}.txt
			  
        echo "$date $hostname $number_of_imapsync_running mem $memory_used_by_all_processes_KiB $memory_used_by_all_processes_percent cpu $load_1_minute bandwidth $bandwidth_bytes_per_sec $bandwidth_percent slots $slots_available" > /var/tmp/imapsync_mem_cpu_bandwidth_${1:-1}.txt
}


echoq loop_number_of_imapsync_running_bandwidth
loop_number_of_imapsync_running_bandwidth()
{
	while :
	do 
		:
		nirbd=`number_of_imapsync_running_bandwidth ${1:-1}`
		echo $nirbd
		echo $nirbd >> /var/tmp/number_of_imapsync_running_every_${1:-1}s.txt
	done

}

grep_header()
{
        echo "$2" | grep "$1" | cut -d: -f2-
}

get_header()
{
        curl -s --head --max-time 4 "$1"
}


tests_grep_last_modified()
{
        run_test test ""     = "`grep_last_modified`" 
        run_test test "" = "`grep_last_modified abcd`" 
        run_test test " Fri, 01 Jul 2022 09:05:38 GMT" = "`grep_last_modified 'Last-Modified: Fri, 01 Jul 2022 09:05:38 GMT'`" 
        run_test test " Fri, 01 Jul 2022 09:05:38 GMT" = "`grep_last_modified 'HTTP/1.1 200 OK
Date: Fri, 01 Jul 2022 09:05:44 GMT
Server: Apache/2.4.53 (Debian)
Last-Modified: Fri, 01 Jul 2022 09:05:38 GMT
ETag: "a6-5e2bab09b5bc2"
Accept-Ranges: bytes
Content-Length: 166
Vary: Accept-Encoding
Content-Type: text/plain
'`" 
}

grep_last_modified()
{
        grep_header Last-Modified: "$1"
}



tests_grep_date()
{
        run_test test ""     = "`grep_date`" 
        run_test test "" = "`grep_date abcd`" 
        run_test test " Fri, 01 Jul 2022 09:05:44 GMT" = "`grep_date 'Date: Fri, 01 Jul 2022 09:05:44 GMT'`" 
        run_test test " Fri, 01 Jul 2022 09:05:44 GMT" = "`grep_date 'HTTP/1.1 200 OK
Date: Fri, 01 Jul 2022 09:05:44 GMT
Server: Apache/2.4.53 (Debian)
Last-Modified: Fri, 01 Jul 2022 09:05:38 GMT
ETag: "a6-5e2bab09b5bc2"
Accept-Ranges: bytes
Content-Length: 166
Vary: Accept-Encoding
Content-Type: text/plain
'`" 
}



grep_date()
{
        grep_header Date: "$1"
}


tests_date_to_epoch()
{
        run_test test "1234567890" = "`date_to_epoch 'Fri, 13 Feb 2009 23:31:30 GMT'`" 
        run_test test "1234567890" = "`date_to_epoch 'Sat Feb 14 00:31:30 CET 2009'`" 
}

date_to_epoch()
{
        here_is_freebsd && { gdate --date "$1" +%s && return ; }
        here_is_linux   && {  date --date "$1" +%s && return ; }
}

tests_diff_current_vs_last_modified()
{
        run_test test ""     = "`diff_current_vs_last_modified`"
        run_test test ""     = "`diff_current_vs_last_modified blabla`" 
        run_test test ""     = "`diff_current_vs_last_modified blabla blibli`"
        
        # No "Last-Modified:"
        run_test test "" = "`diff_current_vs_last_modified 'HTTP/1.1 200 OK
Date: Fri, 01 Jul 2022 09:05:44 GMT
Server: Apache/2.4.53 (Debian)
'`"

        # No "Date:"
        run_test test "" = "`diff_current_vs_last_modified 'HTTP/1.1 200 OK
Last-Modified: Fri, 01 Jul 2022 09:05:38 GMT
'`"

        # Both "Date:" and "Last-Modified:"
        run_test test "6" = "`diff_current_vs_last_modified 'HTTP/1.1 200 OK
Date: Fri, 01 Jul 2022 09:05:44 GMT
Server: Apache/2.4.53 (Debian)
Last-Modified: Fri, 01 Jul 2022 09:05:38 GMT
ETag: "a6-5e2bab09b5bc2"
Accept-Ranges: bytes
Content-Length: 166
Vary: Accept-Encoding
Content-Type: text/plain
'`"
}


diff_current_vs_last_modified()
{
        test -n "$1" || { echo "" && return ; }
        last_modified=`grep_last_modified "$1"`
        test -n "$last_modified" || { echo "" && return ; }
        current_date=`grep_date "$1"`
        test -n "$current_date" || { echo "" && return ; }
        last_modified_epoch=`date_to_epoch "$last_modified"`
        current_date_epoch=`date_to_epoch "$current_date"`
        echo $current_date_epoch - $last_modified_epoch | bc
}

freshness()
{
        header_to_check=`get_header "$1"`
        diff_current_vs_last_modified "$header_to_check"
}



imapsync_spots_single()
{
        :
        freshness=`freshness "$1"`
        test -n "$freshness" || return
        if test 30 -gt "$freshness"
        then
                get_url "$1"
        fi
}


imapsync_spots_available()
{
        imapsync_hosts=`curl --silent https://i005.lamiral.info/imapsync_remote.txt`
        #echo $imapsync_hosts
        for imapsync_host in $imapsync_hosts 
        do
                imapsync_spots_link=`https_and_host_part "$imapsync_host"`/imapsync_mem_cpu_bandwidth_6.txt
                #echo $imapsync_spots_link
                imapsync_spots_single "$imapsync_spots_link"
        done | egrep -o 'slots [0-9]+' | awk '{ sum+=$2  } END { print sum }'
}

add_spare_if_needed()
{
        :
        imapsync_spots_available=`imapsync_spots_available`
        echo "imapsync_spots_available=$imapsync_spots_available"
        if test 0 -gt "$imapsync_spots_available"
        then
                add_spare
        fi
}

imapsync_spare_running()
{
        imapsync_spare_host=`curl --silent https://i005.lamiral.info/imapsync_remote_spare.txt`
        #echo $imapsync_spare_host
        test -n "$imapsync_spare_host" || return 
        imapsync_spare_link=`https_and_host_part "$imapsync_spare_host"`/imapsync_mem_cpu_bandwidth_6.txt
        #echo $imapsync_spare_link
        imapsync_spots_single "$imapsync_spare_link" | cut -d' ' -f3
}


remove_spare_if_not_needed()
{
        imapsync_spare_running=`imapsync_spare_running`
        imapsync_spots_available=`imapsync_spots_available`
        echo imapsync_spare_running   = $imapsync_spare_running
        echo imapsync_spots_available = $imapsync_spots_available
        
        if test "0" = "$imapsync_spare_running" && test "1" -lt "$imapsync_spots_available"
        then
                echo remove_spare
                remove_spare
        fi
        
}




remove_spare()
{
        imapsync_remote_spare_be_void
        echo wait_runs_to_be_finished to be coded
        shelve_spare
}


add_spare()
{
        unshelve_spare
        imapsync_remote_spare_be_i050
}

imapsync_remote_spare_be_i050()
{
        ssh root@i005.lamiral.info  'ln -sfv /var/tmp/imapsync_remote_i050.txt /var/tmp/imapsync_remote_spare.txt'
}

imapsync_remote_spare_be_void()
{
        ssh root@i005.lamiral.info  'ln -sfv /var/tmp/imapsync_remote_void.txt /var/tmp/imapsync_remote_spare.txt'
}


unshelve_spare()
{
        openstack server list
        check_i050 && echo "i050 is already up and running" && return 0
        openstack server unshelve i050
        openstack server list
        openstack server show     i050
        while ! ping -c 1 i050.lamiral.info ; do : ; done
        sleep 10
        check_i050 && echo "i050 is up and running" || { echo "i050 failed waking up" ; return 1 ; }
}

shelve_spare()
{
        ssh root@i050.lamiral.info halt
        while ping -c 1 i050.lamiral.info ; do sleep 1 ; done
        echo i050 is down
        openstack server shelve i050
        openstack server show   i050
        openstack server list
        echo openstack server list
}

check_i050()
{
        curl -v --max-time 7 --data 'testslive=1;exitonload=0' https://i050.lamiral.info/cgi-bin/imapsync 2>/dev/null | grep 'Exiting with return value 0'
}




loop_add_or_remove_spare()
{
        while :
        do
                :
                imapsync_spare_running=`imapsync_spare_running`
                imapsync_spots_available=`imapsync_spots_available`
                echo imapsync_spare_running   = $imapsync_spare_running
                echo imapsync_spots_available = $imapsync_spots_available
                add_or_remove_spare "$imapsync_spots_available" "$imapsync_spare_running" | tee -a /var/tmp/add_or_remove_spare_$$.txt
                sleep 60
        done
}


tests_add_or_remove_spare()
{
        dothis=echo
        run_test test "add_spare"      = "`add_or_remove_spare -2 ''`" #
        run_test test "add_spare"      = "`add_or_remove_spare -1 ''`" #
        run_test test "add_spare"      = "`add_or_remove_spare  0 ''`" #
        run_test test "echo can sleep" = "`add_or_remove_spare  1 ''`" #
        run_test test "echo can sleep" = "`add_or_remove_spare  2 ''`" #
        run_test test "remove_spare"   = "`add_or_remove_spare  5 0`"  #
        run_test test "remove_spare"   = "`add_or_remove_spare  2 0`"  #
        run_test test "echo can sleep" = "`add_or_remove_spare  0 0`"  #
        run_test test "echo can sleep" = "`add_or_remove_spare  1 0`"  #
        run_test test "echo can sleep" = "`add_or_remove_spare  5 5`"  #
}

add_or_remove_spare()
{
        :
        spots_available="$1"
        spare_running="$2"
        #echo spots_available="$1" spare_running="$2"
        # No spot available, no spare running 
        if   test 0 -ge "$spots_available" && test "" = "$spare_running"
        then
                $dothis add_spare
        # At leat 2 spots available, no spare running
        elif test 2 -le "$spots_available" && test 0 = "$spare_running"
        then
                $dothis remove_spare
        else
                $dothis echo can sleep
        fi
}


openstack_memo()
{
        :
        cat << EOF

openstack server list
openstack server show      eaf08fae-afd5-4354-ab2f-345357fea83b
openstack server unshelve  eaf08fae-afd5-4354-ab2f-345357fea83b
openstack server shelve    eaf08fae-afd5-4354-ab2f-345357fea83b

# need to be unshelved
openstack console log show eaf08fae-afd5-4354-ab2f-345357fea83b

EOF
}


echoq imapsync_current
imapsync_current()
{
        imapsync_hosts=`curl --silent https://i005.lamiral.info/imapsync_remote.txt https://i005.lamiral.info/imapsync_remote_spare.txt https://i005.lamiral.info/imapsync_remote_extra.txt`
        #echo $imapsync_hosts
        for imapsync_host in $imapsync_hosts 
        do
                imapsync_current_link=`https_and_host_part "$imapsync_host"`/imapsync_current_6.txt
                #echo $imapsync_current_link
                imapsync_current_single "$imapsync_current_link"
        done
}

imapsync_current_save()
{
        :
        imapsync_current | tee /var/tmp/imapsync_current_6s_all.txt.tmp
        mv /var/tmp/imapsync_current_6s_all.txt.tmp /var/tmp/imapsync_current_6s_all.txt
}


https_and_host_part()
{
        echo "$1" | egrep -o 'https://[^/]+'
}

get_url()
{
        curl --silent --max-time 2 "$1"
}

imapsync_current_single()
{
        :
        freshness=`freshness "$1"`
        test -n "$freshness" || return
        if test 30 -gt "$freshness"
        then
                get_url "$1"
        else
                :
                echo "$freshness"
        fi
}


echoq loop_imapsync_current
loop_imapsync_current()
{
	while :
	do 
                
		imapsync_current_save
                echo
		sleep 30
	done
}


echoq generate_proximapsync_stuff
generate_proximapsync_stuff()
{
	newer imapsync_form.js proximapsync_form.js &&
	{
		echo generating proximapsync_form.js
		sed 's/cgi-bin\/imapsync/cgi-bin\/proximapsync/'  imapsync_form.js > proximapsync_form.js
		echo | ci -f -l -m"from imapsync_form.js" proximapsync_form.js
	}
	
	newer imapsync_form_extra.html proximapsync_form_extra.html &&
	{
		echo generating proximapsync_form_extra.html
		sed 's/cgi-bin\/imapsync/cgi-bin\/proximapsync/'  imapsync_form_extra.html > proximapsync_form_extra.html
		sed -i 's/imapsync_form.js/proximapsync_form.js/' proximapsync_form_extra.html
		echo | ci -f -l -m"from imapsync_form_extra.html" proximapsync_form_extra.html
	}
	
	newer imapsync_form_extra_free.html proximapsync_form_extra_free.html &&
	{
		echo generating proximapsync_form_extra_free.html
		sed 's/cgi-bin\/imapsync/cgi-bin\/proximapsync/'  imapsync_form_extra_free.html > proximapsync_form_extra_free.html
		sed -i 's/imapsync_form.js/proximapsync_form.js/' proximapsync_form_extra_free.html
		echo | ci -f -l -m"from imapsync_form_extra_free.html" proximapsync_form_extra_free.html
	}
}

echoq summary_display lognamepattern
summary_display() {
    echo    "Start date : " `start_date "$1"`
    echo    "End   date : " `end_date   "$1"`
    echo -n "Number of /X users:            " ; number_of_X_users "$1"
    echo -n "Number of /X accounts synced:  " ; nb_migrations_launched "$1"
    echo -n "Number of /X syncs:            " ; number_of_syncs "$1"
    echo -n "Total volume /X transferred:   " ; total_volume_transferred "$1"
    echo -n "Total messages /X transferred: " ; total_messages_transferred "$1"
    echo -n "Biggest transfer:              " ; biggest_transfer "$1"
    echo -n "Biggest message seen:          " ; biggest_message_seen "$1"
    echo -n "Biggest message transferred:   " ; biggest_message_transferred "$1"
    echo -n "Biggest bandwidth rate:        " ; biggest_bandwidth_rate "$1"
    echo -n "Average bandwidth rate:        " ; average_bandwidth_rate "$1"
    echo -n "Max messages transferred:      " ; max_number_of_messages_transferred "$1"
    echo -n "Max messages skipped:          " ; max_number_of_messages_skipped "$1"
    echo -n "Longest transfer:              " ; seconds_to_days_hours `longest_transfer "$1"`
    echo -n "Queue length mean is:          " ; stat_queue_mean "$1"
    echo "Data made at" `date -r grep_stats_"$1".txt`
}




echoq various_usefull
various_usefull() {
cat <<'EOF'
strace -e trace=signal -f `pgrep /usr/sbin/apach | xargs -n1 echo -n  " -p "` 2>&1
egrep -o '* ID .*' G_Read___ID.txt | sort | uniq -c | sort -n
egrep -o '* ID .*' G_Read___ID.txt | sort |  awk '{ print $1 " " $2 " "  $3 " NIL" }' | datamash -s -W -g 3 count 3 | awk '{ print $2 " " $1 }' | sort -g

locate perl.core | xargs -n 1 gdb -q -x /tmp/gdb_quit.txt -c

zcat /var/log/apache/httpd-access.log.*.gz|egrep -o -w 'Mozilla/5.0 \([^;]+' | sort | egrep -o '\([a-zA-Z]+' | sort  | uniq -c | sort -g | grep -v KHTML
zcat /var/log/apache/httpd-access.log.*.gz|grep 'POST /cgi-bin/imapsync' | egrep -o -w 'Mozilla/5.0 \([^;]+' | sort | egrep -o '\([a-zA-Z]+' | sort  | uniq -c | sort -g | grep -v KHTML

egrep -o '\[.+@[^]]+]' G_success_login.txt |head -222222 | sort | uniq -c | sort -g
cat  G_success_login_on.txt | ./domains_servers_map | sort | uniq -c | sort -g

list_all_logs |tail -9999 | xargs grep -i 'Exiting with return value 112' | tee  Error_112_last_9999_syncs.txt
cut -d: -f1 Error_112_last_30_days.txt | xargs grep -oih 'Invalid system flag.*'  | sort | uniq -c

list_all_logs | xargs grep -i 'Exiting with return value 112' | tee  Error_112_all_syncs.txt
cut -d: -f1 Error_112_all_syncs.txt  | tail -100  | xargs egrep -oih 'Invalid system flag [^(]+'  | sort | uniq -c

logfiles_finished_recently -300| xargs grep -i 'Exiting with return value 10 ' | grep -v 'return value 0 ' | cut -d: -f1 | xargs  tail -11 | grep 'failure: can not open imap connection on' | uniq -c | sort -g | grep http | tee ../http_host_failures.txt

# Searching big messages copied over 500 MB
list_all_logs|tail -50000 | xargs egrep '{.?[56789]........}    copied'

# online processes stats
cat /var/tmp/number_of_imapsync_running_every_60s.txt | datamash -W min 2 max 2 mean 2 median 2 q1 2 q3 2
for v in 2 3 4; do cat  /var/tmp/number_of_imapsync_running_every_6s.txt | datamash --format=%10.0f -W min  $v max $v mean $v median $v q1 $v q3 $v ; done
netstat -I em0 -b -n -w 6 -q 1
while :; do ssh root@ks5 'cd /var/tmp/imapsync_cgi/ ; . cgi_memo ; loop_number_of_imapsync_running_bandwidth 6' ; echo $?; done

# Search memory eater
cat G_Memory_consumption_at_the_end.txt | sort -g -k7 | grep 202[01] |tail -100  | cut -f1 -d: | while read f; do echo $f ; grep 'Memory consumption at the end' $f; grep 'Host. Nb messages' $f ; grep 'Biggest message' $f ; grep 'Memory/biggest message ratio' $f ; done
cat G_Host2_Nb_messages.txt  | sort -g -k4 | grep 202[01] |tail -100 | cut -f1 -d: | while read f; do echo $f ; grep 'Memory consumption at the end' $f; grep 'Host. Nb messages' $f ; grep 'Biggest message' $f ; grep 'Memory/biggest message ratio' $f ; done
cat G_Host1_Nb_messages.txt  | sort -g -k4 | grep 202[01] |tail -100 | cut -f1 -d: | while read f; do echo $f ; grep 'Memory consumption at the end' $f; grep 'Host. Nb messages' $f ; grep 'Biggest message' $f ; grep 'Memory/biggest message ratio' $f ; done

# Best bandwidth moments
cat /var/tmp/number_of_imapsync_running_every_6s.txt | sort -k3 -g| tail -66

# Worst load momemts
cat /var/tmp/number_of_imapsync_running_every_6s.txt | sort -k9 -g| tail -66

# Sort by number of parallel runs and by load in case of equality
cat /var/tmp/number_of_imapsync_running_every_6s.txt | grep ^2022_ | sort -k2 -k9 -g | tail -666

# Sort by number of parallel runs and by bandwidth in case of equality
cat /var/tmp/number_of_imapsync_running_every_6s.txt | grep ^2022_ | sort -k2 -k3 -g | tail -666


# getrusage on FreeBSD, espacially disk i/o
procstat -r `pgrep -f cgi-bin/imapsync`

# Sum up all memory taken by imapsync runs, in KiB.
ps -o rss -p `pgrep -f cgi-bin/imapsync` | sed 1,1d | datamash sum  1
memory_used_by_all_imapsync_KiB

#
systemctl --no-pager status apache2 ; systemctl --no-pager restart  apache2 ; systemctl --no-pager status apache2
systemctl --no-pager status httpd   ; systemctl --no-pager restart  httpd   ; systemctl --no-pager status httpd

# Search for 'Client-Aborted: die X-Died: EOF when chunk header expected' and for what hosts
tail -66666 list_all_logs_auto.txt | grep -v 385d7a4d8d428d7aa2b57c8982629e2bd67698ed|egrep -o '.*/'|sort | uniq | while read d; do echo $d/*_proxy.txt ; done | grep -v '*' | xargs egrep -l 'Client-Aborted: die X-Died: EOF when chunk header expected' | xargs grep 'Gonna delegate the imap sync to'

EOF
}


echoq perf_help
perf_help() {
        test FreeBSD = `uname -s` && { 
                echo FreeBSD here
                echo "nload -t 6000 em0  -u K -i 100000 -o 100000"
                echo "iftop -i em0  -f 'port imap or port imaps' -B #  t p >"
        }
        
        test Linux = `uname -s`  && { 
                echo Linux here
                echo "nload -t 6000 eth0 -u K  -i 100000 -o 100000 # Linux"
                echo "iftop -i eth0 -f 'port imap or port imaps' -B #  t p >"
        }
}

tests_all_verbose_if_failure

init()
{ 
	hostname=`hostname`
}

init