#!/bin/sh # $Id: imapsync_csv_wrapper,v 1.13 2021/11/01 13:24:47 gilles Exp gilles $ exec 3>&1 debug=true # comment the following to have debug data debug=false PATH=$PATH:/home/www/apache24/cgi-bin/ echo3() { echo "$@" >&3 } echodebug() { $debug && echo3 "$@" } run_test() { tests_count=`expr 1 + $tests_count` # do not run anything between the two following instructions "$@"; run_test_status=$? # now you can run something since $? is saved if test x"$run_test_status" = x"0"; then echo "ok $tests_count $@" else echo "not ok $tests_count $@" tests_failed_count=`expr 1 + $tests_failed_count` tests_failed_list="$tests_failed_list $tests_count" fi return $run_test_status } run_tests() { tests_count=0 tests_failed_count=0 tests_failed_list= for t in "$@"; do echo "### running $t" "$t" done echo echo "#### ALL tests done" if test 0 -eq $tests_failed_count; then echo "ALL $tests_count TESTS SUCCESSFUL" return 0 else # At least one failed echo "FAILED $tests_failed_count/$tests_count TESTS: $tests_failed_list" return 1 fi } uri_unescape() { #echo "$@" | perl -MURI::Escape -e 'print uri_unescape(<>)' echo "$1" | perl -pe 's/\%(\w\w)/chr hex $1/ge' } tests_uri_unescape() { run_test test '' = '' run_test test "" = "" run_test test '1' = '1' run_test test 'A B' = 'A B' run_test test "'A B' = 'A B'" run_test test '' = "`uri_unescape`" run_test test "" = "`uri_unescape`" run_test test 'A' = `uri_unescape A` run_test test 'The cat' = "`uri_unescape The%20cat`" run_test test "\r" = "\r" run_test test '"\r" = "`uri_unescape %0D`"' run_test test "\n" = "\n" nl=`uri_unescape %0A` nl="\n" run_test test "_\n_" = "_${nl}_" run_test test '!"#$%&'\''()' = "`uri_unescape %21%22%23%24%25%26%27%28%29`" run_test test '*+,/:;=?@[]' = "`uri_unescape %2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D`" run_test test '"_a\n\n\nz_" = "_`uri_unescape a%0A%0A%0Az`_"' run_test test '"_`uri_unescape a%0A%0A%0Az`_" = "_`uri_unescape a%0A%0A%0Az`_"' run_test test 'A B' = 'A B' uri_unescape 'a%0Az' > /tmp/$$_zzz1.txt /bin/echo "a z" > /tmp/$$_zzz2.txt run_test diff /tmp/$$_zzz1.txt /tmp/$$_zzz2.txt rm /tmp/$$_zzz1.txt /tmp/$$_zzz2.txt z3=`uri_unescape a%0A%0A%0Az` z4="a\n\n\nz" run_test test '"$z3" = "$z4"' run_test test '"\n" = "`uri_unescape %0A`"' #run_test test '\n' = "`uri_unescape %0A`" run_test test '"a\r\nz" = "a\r\nz"' run_test test '"_a\r\nz_" = "`uri_unescape _a%0D%0Az_`"' run_test test 'A' = `uri_unescape A` } value_of_key() { key="$1" test 1 -le "$#" && shift keyval="$1" #echo "key:$key" #echo "keyval:$keyval" #set -x echo "$keyval" | perl -ne "s/^.*$key"'=([^&]+).*$/$1/ and print' #set +x } tests_value_of_key() { run_test test "" = "`value_of_key`" run_test test "" = "`value_of_key badkey`" run_test test "" = "`value_of_key badkey mykey=myvalue`" run_test test "myvalue" = "`value_of_key mykey mykey=myvalue`" run_test test "myvalue" = `value_of_key mykey "mykey=myvalue"` run_test test "myvalue" = `value_of_key mykey "mykey=myvalue&other=rrr"` run_test test "myvalue" = `value_of_key mykey "otra=zzz&mykey=myvalue"` run_test test "myvalue" = `value_of_key mykey "otra=zzz&mykey=myvalue&other=rrr"` } remove_blanks_and_comments() { /bin/echo -n '' echo "$1" | perl -ne '!/^ *#.*$|^[\r ]*$/ && print' } tests_remove_blanks_and_comments() { #set -x run_test test "" = "`remove_blanks_and_comments`" run_test test "a" = "`remove_blanks_and_comments a`" myval=" " run_test test "" = "`remove_blanks_and_comments $myval`" myval="a " run_test test "a" = "`remove_blanks_and_comments $myval`" myval=" a " run_test test "a" = "`remove_blanks_and_comments $myval`" myval=" a " run_test test "a" = "`remove_blanks_and_comments $myval`" myval="a b" run_test test '"$myval" = `remove_blanks_and_comments $myval`' myval="# caca a " run_test test '"a" = "`remove_blanks_and_comments $myval`"' myval="# caca a b # ooo c " myval_filtered="a b c " run_test test '"$myval_filtered" = "`remove_blanks_and_comments $myval`"' run_test remove_blanks_and_comments run_test remove_blanks_and_comments ' ' run_test remove_blanks_and_comments ' # blabla ' run_test test "" = "`remove_blanks_and_comments ' ' `" run_test test "" = "`remove_blanks_and_comments ' # blabla ' `" } is_blank_or_comment() { test "" = "`remove_blanks_and_comments $1 `" } tests_is_blank_or_comment() { run_test is_blank_or_comment run_test is_blank_or_comment '' run_test is_blank_or_comment ' ' run_test is_blank_or_comment '# blabla 1 ' run_test is_blank_or_comment ' # blabla 2 ' is_blank_or_comment 'blabla 1 ' ; run_test test 1 = $? is_blank_or_comment ' blabla 2 ' ; run_test test 1 = $? is_blank_or_comment ' blabla 3 # blibli ' ; run_test test 1 = $? } logfailures() { run_on_each_line_failures="$run_on_each_line_failures $@" } tests_logfailures() { #reset run_on_each_line_failures= # add Blabla run_test logfailures Blabla run_test test " Blabla" = "$run_on_each_line_failures" # add Kiki run_test logfailures Kiki run_test test " Blabla Kiki" = "$run_on_each_line_failures" #reset again run_on_each_line_failures= } run_on_each_line() { echodebug '### Entering run_on_each_line ###' test -f abort_$$.txt && { echo abort demanded so not doing normal stuff ; return ; } /bin/echo -n '' test -z "$1" && { echo3 'First parameter is empty. Leaving run_on_each_line().' ; return ; } csv_data_multilines="$1" echodebug "csv_data_multilines=[$csv_data_multilines]" test 1 -le "$#" && shift command=${1:-echo} echodebug command="$command" "$@" test 1 -le "$#" && shift line_number=0 csv_number=0 failures_number=0 while IFS=';' read h1 u1 p1 h2 u2 p2 extra fake; do test -f abort_$$.txt && { echo abort demanded so not doing normal stuff ; return ; } line_number=`expr 1 + $line_number` if is_blank_or_comment "$h1$u1$p1$h2$u2$p2$extra" ; then echo3 "Ignoring line " $line_number "$h1$u1$p1$h2$u2$p2$extra" else csv_number=`expr 1 + $csv_number` echo3 "Processing line " $line_number csv run $csv_number "[$h1] [$u1] [xxx] [$h2] [$u2] [xxx] [$extra] $$" $command "$@" --host1 "$h1" --user1 "$u1" --password1 "$p1" --host2 "$h2" --user2 "$u2" --password2 "$p2" $extra command_status=$? test "0" != "$command_status" && failures_number=`expr 1 + $failures_number` echodebug command_status=$command_status failures_number=$failures_number test "0" != "$command_status" && logfailures "line $line_number csv run $csv_number returned with status $command_status ($h1;$u1;$p1;$h2;$u2;$p2;$extra;)" fi done << EOF $csv_data_multilines EOF echodebug Leaving run_on_each_line, failures_number=$failures_number test "0" != "$failures_number" && echo "$run_on_each_line_failures" test "0" != "$failures_number" && return 1 return 0 } tests_run_on_each_line() { # 1-4 run_test test '' = "`run_on_each_line`" run_test test '' = "`run_on_each_line ''`" run_test test '' = "`run_on_each_line '' echo imapsync`" run_test test '' = "`run_on_each_line '' echo imapsync blabla blibli`" # 5-7 blank data run_test test '' = "`run_on_each_line ' '`" run_test test '' = "`run_on_each_line ' ' `" run_test test '' = "`run_on_each_line ' ' echo imapsync blabla blibli`" # 8-10 comment data run_test test '' = "`run_on_each_line '# '`" run_test test '' = "`run_on_each_line ' # '`" run_test test '' = "`run_on_each_line '# ' echo imapsync blabla blibli`" # 11-13 empty line myval=' ' myret=`run_on_each_line "$myval"` run_test test "" = "$myret" myret=`run_on_each_line "$myval"` run_test test "" = "$myret" myret=`run_on_each_line "$myval" echo imapsync blabla blibli` run_test test "" = "$myret" # 14-16 blanks or comments myval=' # blabla 1 # blabla 2 ' myret=`run_on_each_line "$myval"` run_test test "" = "$myret" myret=`run_on_each_line "$myval" ` run_test test "" = "$myret" myret=`run_on_each_line "$myval" echo imapsync blabla blibli` run_test test "" = "$myret" # 17-18 # csv data no blank no comment lines='a1;b1;c1;d1;e1;f1 a2;b2;c2;d2;e2;f2 a3;b3;c3;d3;e3;f3' # no command (so echo) lines_out='--host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 --host1 a2 --user1 b2 --password1 c2 --host2 d2 --user2 e2 --password2 f2 --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3' run_test test "$lines_out" = "`run_on_each_line "$lines"`" # command = echo imapsync lines_out='imapsync --host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 imapsync --host1 a2 --user1 b2 --password1 c2 --host2 d2 --user2 e2 --password2 f2 imapsync --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3' run_test test "$lines_out" = "`run_on_each_line "$lines" echo imapsync`" # 19-20 # csv data + blanks + comments lines='a1;b1;c1;d1;e1;f1 # blabla 2 a2;b2;c2;d2;e2;f2 # bloblo 3 a3;b3;c3;d3;e3;f3 # end' # no command (so echo) lines_out='--host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 --host1 a2 --user1 b2 --password1 c2 --host2 d2 --user2 e2 --password2 f2 --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3' run_test test "$lines_out" = "`run_on_each_line "$lines"`" # command = echo imapsync lines_out='imapsync --host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 imapsync --host1 a2 --user1 b2 --password1 c2 --host2 d2 --user2 e2 --password2 f2 imapsync --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3' run_test test "$lines_out" = "`run_on_each_line "$lines" echo imapsync`" # 21-22 # csv data 3/3;/6/6;/7/7;/8/8; parameters + blanks + comments # runs 1 2 3 4 5 6 7 8 lines='a1;b1;c1 # blabla 2 a2;b2;c2; # bloblo 3 a3;b3;c3;d3;e3;f3 a4;b4;c4;d4;e4;f4; a5;b5;c5;d5;e5;f5;--extra blurp a6;b6;c6;d6;e6;f6;--extra blurp; a7;b7;c7;d7;e7;f7;--extra blurp; ignored a8;b8;c8;d8;e8;f8;--extra blurp; ignored; # end' # no command (so echo) lines_out='--host1 a1 --user1 b1 --password1 c1 --host2 --user2 --password2 --host1 a2 --user1 b2 --password1 c2 --host2 --user2 --password2 --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3 --host1 a4 --user1 b4 --password1 c4 --host2 d4 --user2 e4 --password2 f4 --host1 a5 --user1 b5 --password1 c5 --host2 d5 --user2 e5 --password2 f5 --extra blurp --host1 a6 --user1 b6 --password1 c6 --host2 d6 --user2 e6 --password2 f6 --extra blurp --host1 a7 --user1 b7 --password1 c7 --host2 d7 --user2 e7 --password2 f7 --extra blurp --host1 a8 --user1 b8 --password1 c8 --host2 d8 --user2 e8 --password2 f8 --extra blurp' run_test test "$lines_out" = "`run_on_each_line "$lines"`" # command = echo imapsync lines_out='imapsync --host1 a1 --user1 b1 --password1 c1 --host2 --user2 --password2 imapsync --host1 a2 --user1 b2 --password1 c2 --host2 --user2 --password2 imapsync --host1 a3 --user1 b3 --password1 c3 --host2 d3 --user2 e3 --password2 f3 imapsync --host1 a4 --user1 b4 --password1 c4 --host2 d4 --user2 e4 --password2 f4 imapsync --host1 a5 --user1 b5 --password1 c5 --host2 d5 --user2 e5 --password2 f5 --extra blurp imapsync --host1 a6 --user1 b6 --password1 c6 --host2 d6 --user2 e6 --password2 f6 --extra blurp imapsync --host1 a7 --user1 b7 --password1 c7 --host2 d7 --user2 e7 --password2 f7 --extra blurp imapsync --host1 a8 --user1 b8 --password1 c8 --host2 d8 --user2 e8 --password2 f8 --extra blurp' run_test test "$lines_out" = "`run_on_each_line "$lines" echo imapsync`" # 23-2x # "" input lines='a1;b1;c1;d1;e1;f1;--caca "KK"' # no command (so echo) lines_out='--host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 --caca "KK"' run_test test "$lines_out" = "`run_on_each_line "$lines"`" # command = echo imapsync lines_out='imapsync --host1 a1 --user1 b1 --password1 c1 --host2 d1 --user2 e1 --password2 f1 --caca "KK"' run_test test "$lines_out" = "`run_on_each_line "$lines" echo imapsync`" } remove_double_quotes() { echo "$1" | tr -d '"' } tests_remove_double_quotes() { echo 'Entering tests_remove_double_quotes()' run_test test -z `remove_double_quotes` run_test test 'abc' = `remove_double_quotes 'abc'` run_test test -z `remove_double_quotes '"'` run_test test -z `remove_double_quotes '""'` run_test test 'abc' = `remove_double_quotes '"ab"c'` echo 'Leaving tests_remove_double_quotes()' } hashsync() { #set -x mystring=`remove_double_quotes ${1:-''}` mykey=${2:-''} # impressive! is not it? quoting shit! perl -MDigest::HMAC_SHA1 -e 'print Digest::HMAC_SHA1::hmac_sha1_hex( "'"$mystring"'", "'"$mykey"'" )' #set +x } tests_hashsync() { echo 'Entering tests_hashsync()' run_test test 'fbdb1d1b18aa6c08324b7d64b71fb76370690e1d' = `hashsync` run_test test 'fbdb1d1b18aa6c08324b7d64b71fb76370690e1d' = `hashsync ''` run_test test 'fbdb1d1b18aa6c08324b7d64b71fb76370690e1d' = `hashsync '' ''` run_test test 'e86a28a3611c1e7bbaf8057cd00ae122781a11fe' = `hashsync 'zzz' ''` run_test test '6a7b451ac99eab1531ad8e6cd544b32420c552ac' = `hashsync 'zzz' 'A'` run_test test 'fbdb1d1b18aa6c08324b7d64b71fb76370690e1d' = `hashsync '""'` run_test test 'e86a28a3611c1e7bbaf8057cd00ae122781a11fe' = `hashsync '"zzz"'` echo 'Leaving tests_hashsync()' } tests_caca() { run_test test 'z' = '' run_test test '1' = '1' } change_working_directory_if_under_cgi() { test -n "$SERVER_SOFTWARE" && { pwd #env ! test -d "$1" && { echo creating directory "$1" mkdir -p "$1" } echo changing current directory to "$1" cd "$1" pwd } } write_my_pid() { echo Writing my pid $$ in imapsync_csv_wrapper.pid echo $$ > imapsync_csv_wrapper.pid } kill_pid_in_files() { echo Will kill -TERM pid found in "$@" for pidfile in "$@"; do # FIXME add test -f $pidfile at least if test -f $pidfile; then pidlist="$pidlist `head -1 $pidfile`" test -f "$pidfile" && echo Doing kill -TERM `head -1 "$pidfile"` "$pidfile" test -f "$pidfile" && kill -TERM `head -1 "$pidfile"` else echo "No $pidfile here" fi done } abort_other_process() { echodebug 'Entering abort_other_process()' $debug && ls -ltr # No active imapsync_csv_wrapper => nothing to abort if ! test -f imapsync_csv_wrapper.pid ; then echo No imapsync_csv_wrapper.pid here. Nothing to abort. echodebug 'Leaving abort_other_process()' return 0 else # a message to the active imapsync_csv_wrapper main_pid_to_abort=`head -1 imapsync_csv_wrapper.pid` touch abort_${main_pid_to_abort}.txt echodebug 'sleep 3' && sleep 3 kill_pid_in_files imapsync.pid echodebug 'sleep 3 again then nother try to kill' && sleep 3 kill_pid_in_files imapsync.pid ls -ltr echodebug 'Leaving abort_other_process()' return 0 fi } quit() { echo "Leaving. I am $0 pid $$" for pidfile in "$@"; do test -f "$pidfile" && echo rm "$pidfile" && rm "$pidfile" done } tests() { : # All tests run_tests \ tests_uri_unescape \ tests_value_of_key \ tests_remove_blanks_and_comments \ tests_run_on_each_line \ tests_hashsync \ tests_is_blank_or_comment \ tests_logfailures \ tests_remove_double_quotes \ } header() { # HTTP header, may be empty echo } getcsvdata() { read imapsync_csv_wrapper_from_ui echodebug '######################################################################' echodebug Here are the UI parameters, raw: echodebug "var:" echodebug "$imapsync_csv_wrapper_from_ui" echodebug;echodebug csv_data=`value_of_key csv_data "$imapsync_csv_wrapper_from_ui"` echodebug '######################################################################' echodebug Here csv_data: echodebug "$csv_data" csv_data_unescaped=`uri_unescape "$csv_data"` echodebug '######################################################################' echodebug Here csv_data, unescaped: echodebug "$csv_data_unescaped" csv_data_pure=`remove_blanks_and_comments "$csv_data_unescaped"` echodebug '######################################################################' echodebug Here csv_data no blanks nor comments: echodebug "$csv_data_pure" } tests_all_verbose_if_failure() { # Run tests silent but if failure then rerun them verbose. # return 0 if all tests passed # return 1 if some tests failed if ! tests > /dev/null 3>&1 ; then tests return 1 fi return 0 } abort_other_if_asked_to() { echodebug Looking for abort abort=`value_of_key abort "$imapsync_csv_wrapper_from_ui"` echodebug abort=$abort if test 'on' = "$abort" ; then abort_other_process echo "Abort done. Leaving" return 0 else echodebug 'Not asked to abort other process' echodebug 'Leaving abort_other_if_asked_to()' return 1 fi } is_another_running() { # I should check that the pid is actually a running one # but that's ok for now if test -f imapsync_csv_wrapper.pid; then echo Looks like another imapsync_csv_wrapper is running. Leaving return 0 else echodebug Looks like I am alone. Good. Let us go on. return 1 fi } signal_handling() { # 2=INT 3=QUIT 15=TERM trap "quit imapsync_csv_wrapper.pid abort_$$.txt" 15 2 3 0 $debug && trap return 0 } banner_round() { banner_line=' ################################################################################' banner_title=${1:-$banner_line} echo3 "$banner_line$banner_title$banner_line" } justlogin() { banner_round ' ######################## just login check round ################################' date test -f abort_$$.txt && { echo Abort demanded so not doing normal stuff ; return ; } echo Now run imapsync --justlogin to check all credentials are ok if run_on_each_line "$csv_data_unescaped" \ imapsync --no-modulesversion --justlogin --tmpdir . then echo success of the just login check round return 0 else echo failure of the just login check round return 1 fi } justfoldersizes() { banner_round ' ######################## just folders sizes counting round #####################' date test -f abort_$$.txt && { echo Abort demanded so not doing normal stuff ; return ; } echo Now run imapsync --justfoldersizes to show the volume to be synced run_on_each_line "$csv_data_unescaped" \ imapsync --no-modulesversion --justfoldersizes --tmpdir . \ || return 1 } sync_all() { banner_round ' ######################## now sync them all round ###############################' date test -f abort_$$.txt && { echo Abort demanded so not doing normal stuff ; return ; } echo Now run imapsync on the data given run_on_each_line "$csv_data_unescaped" \ imapsync --no-modulesversion --tmpdir . \ || return 1 } tests_initialisation() { #### Variables initialisation tests_count=0 tests_failed_count=0 tests_failed_list= } main() { header # All stderr to stdin, yeah I am like that exec 2>&1 # Some tests, uncomment which one you work with and also the return one #run_tests tests_is_blank_or_comment #run_tests tests_run_on_each_line #run_tests tests_remove_blanks_and_comments #tests_hashsync #run_tests tests_logfailures #tests_remove_double_quotes #return tests_all_verbose_if_failure || return 1 echodebug my pid is $$ #return getcsvdata hashsync=`hashsync "$csv_data_pure"` test -z $hashsync && return 1 change_working_directory_if_under_cgi /var/tmp/imapsync_cgi/$hashsync/ abort_other_if_asked_to && return 0 is_another_running && return 0 write_my_pid signal_handling SERVER_SOFTWARE_BAK="$SERVER_SOFTWARE" unset SERVER_SOFTWARE # Now the real stuff #justlogin || return 1 #justfoldersizes sync_all || return 1 $debug && ls -ltr quit imapsync_csv_wrapper.pid abort_$$.txt echo $0 ended at the end } tests_initialisation main return 0