diff --git a/shim_copy_baseDir.sh b/shim_copy_baseDir.sh index def0906..ca1b31a 100755 --- a/shim_copy_baseDir.sh +++ b/shim_copy_baseDir.sh @@ -129,17 +129,5 @@ sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -if [ "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees the same" -else - echo "RESULT: BAD trees have $diffs differences" -fi +./test_util/compare_trees.sh +echo diff --git a/shim_copy_flatten.sh b/shim_copy_flatten.sh index c90bd4a..5227421 100755 --- a/shim_copy_flatten.sh +++ b/shim_copy_flatten.sh @@ -125,17 +125,5 @@ sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -if [ "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees the same" -else - echo "RESULT: BAD trees have $diffs differences" -fi +./test_util/compare_trees.sh +echo diff --git a/shim_copy_mirror.sh b/shim_copy_mirror.sh index b0a7844..5e41f68 100755 --- a/shim_copy_mirror.sh +++ b/shim_copy_mirror.sh @@ -128,27 +128,5 @@ sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -cd shim_dirA -find . -type l | xargs ls -al | cut --bytes=42- >../dirA.links -cd ../shim_dirB -find . -type l | xargs ls -al | cut --bytes=42- >../dirB.links -cd .. - -sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links - -linkdiffs="`diff dirA.links dirC.links|wc -l`" - -if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored" -else - echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links" -fi +./test_util/compare_trees.sh --check-links +echo diff --git a/shim_copy_mirror_sftp.sh b/shim_copy_mirror_sftp.sh index 92afd32..a3099d1 100755 --- a/shim_copy_mirror_sftp.sh +++ b/shim_copy_mirror_sftp.sh @@ -124,27 +124,5 @@ sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -cd shim_dirA -find . -type l | xargs ls -al | cut --bytes=42- >../dirA.links -cd ../shim_dirB -find . -type l | xargs ls -al | cut --bytes=42- >../dirB.links -cd .. - -sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links - -linkdiffs="`diff dirA.links dirC.links|wc -l`" - -if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored" -else - echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links" -fi +./test_util/compare_trees.sh --check-links +echo diff --git a/shim_copy_post.sh b/shim_copy_post.sh index 315ca11..def01d5 100755 --- a/shim_copy_post.sh +++ b/shim_copy_post.sh @@ -83,6 +83,8 @@ cp dirone/dirtwo/filetwo "dirone/filetwo copy with spaces in the name" echo "#test 1 sha512 160 copy a file with spaces in the name (both source and dest)" cp "dirone/filetwo copy with spaces in the name" "dirone/second spaced file" +# sleep to allow filetwo to be downloaded before renaming it to filefour +sleep 10 echo "#test 1 rename 145 rename in a sub-dir" mv dirone/dirtwo/filetwo dirone/dirtwo/filefour @@ -118,10 +120,16 @@ echo "#test 1 rename move test_file into dirthree subdir (new name)" mv test_file dirthree/new_test_file # issue #208 rm directory with trailing slash -echo "#test 1 directory create with trailing slash" -mkdir dir_test/ +echo "#test 1 directory create without trailing slash" +mkdir dir_test echo "#test 1 remove directory with trailing slash" rm -r dir_test/ +echo "#test 1 directory create with trailing slash" +mkdir dir_test2/ + +echo "#test 1 remove directory with trailing slash" +rm -r dir_test2/ + echo "#test 0 comment 160 shim copy posting end" diff --git a/shim_copy_post2.sh b/shim_copy_post2.sh index e23d4e1..9e29f09 100755 --- a/shim_copy_post2.sh +++ b/shim_copy_post2.sh @@ -96,7 +96,8 @@ cp dirone/dirtwo/filetwo "dirone/filetwo copy with spaces in the name" echo "#test 1 sha512 160 copy a file with spaces in the name (both source and dest)" cp "dirone/filetwo copy with spaces in the name" "dirone/second spaced file" - +# sleep to allow filetwo to be downloaded before renaming it to filefour +sleep 10 echo "#test 1 rename 145 rename in a sub-dir" mv dirone/dirtwo/filetwo dirone/dirtwo/filefour diff --git a/shim_copy_strip.sh b/shim_copy_strip.sh index bff64fe..97a4d2c 100755 --- a/shim_copy_strip.sh +++ b/shim_copy_strip.sh @@ -138,18 +138,5 @@ sr3 stop subscribe/local_copy.conf sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -if [ "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees the same" -else - echo "RESULT: BAD trees have $diffs differences" -fi +./test_util/compare_trees.sh +echo diff --git a/shim_copy_strip_slash.sh b/shim_copy_strip_slash.sh index 01dd6da..82dc5f8 100755 --- a/shim_copy_strip_slash.sh +++ b/shim_copy_strip_slash.sh @@ -130,18 +130,6 @@ sr3 stop subscribe/local_copy.conf sr3 remove subscribe/local_copy.conf echo "#test 0 comment comparing trees" - - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -if [ "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees the same" -else - echo "RESULT: BAD trees have $diffs differences" -fi + +./test_util/compare_trees.sh +echo diff --git a/shim_test_race_conditions.sh b/shim_test_race_conditions.sh index 5215f2e..4cf46d2 100755 --- a/shim_test_race_conditions.sh +++ b/shim_test_race_conditions.sh @@ -208,27 +208,7 @@ sleep ${sleepytime} sr3 remove cpost/local_post.conf # wait for retries to finish before stopping subscriber - -echo "Waiting for retries to finish..." -stalled=0 -stalled_value=-1 -retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)" -while [ $retry_msgcnt -gt 0 ]; do - printf "${flow_test_name} Still %4s messages to retry, waiting...\n" "$retry_msgcnt" - sleep 15 - retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)" - - if [ "${stalled_value}" == "${retry_msgcnt}" ]; then - stalled=$((stalled+1)); - if [ "${stalled}" == 5 ]; then - printf "\n Warning some retries stalled, skipping..., might want to check the logs\n\n" - retry_msgcnt=0 - fi - else - stalled_value=$retry_msgcnt - stalled=0 - fi -done +./test_util/wait_for_retries.sh # wait a bit longer sleep 60 @@ -241,33 +221,6 @@ rm ~/.config/sr3/plugins/screw_up_order.py sed -i '/callback screw_up_order/d' ~/.config/sr3/default.conf echo "#test 0 comment comparing trees" - - -cd shim_dirA -find -H . -type f | xargs -d '\n' md5sum >../dirA.sums -cd ../shim_dirB -find -H . -type f | xargs -d '\n' md5sum >../dirB.sums -cd .. - -diffs="`diff dirA.sums dirB.sums| wc -l`" - -cd shim_dirA -find . -type l | xargs ls -al | cut --bytes=42- >../dirA.links -cd ../shim_dirB -find . -type l | xargs ls -al | cut --bytes=42- >../dirB.links -cd .. - -sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links - -linkdiffs="`diff dirA.links dirC.links|wc -l`" - -if [ "${linkdiffs}" -eq 0 -a "${diffs}" -eq 0 ]; then - echo "RESULT: Good! trees links the same: `wc -l dirA.sums` files and `wc -l dirA.links` links mirrored" -else - echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links" - echo "shim_dirA:" - ls -hal shim_dirA - echo - echo "shim_dirB:" - ls -hal shim_dirB -fi + +./test_util/compare_trees.sh --check-links +echo diff --git a/test_util/compare_trees.sh b/test_util/compare_trees.sh new file mode 100755 index 0000000..64688c7 --- /dev/null +++ b/test_util/compare_trees.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +check_links=false +if [[ " $* " == *"--check-links"* ]]; then + check_links=true +fi + + +# check for file differences +cd shim_dirA +find -H . -type f | xargs -d '\n' md5sum >../dirA.sums +cd ../shim_dirB +find -H . -type f | xargs -d '\n' md5sum >../dirB.sums +cd .. + +diffs="`diff dirA.sums dirB.sums| wc -l`" + + +# check for link differences +if [[ $check_links == true ]]; then + cd shim_dirA + find . -type l | xargs ls -al | cut --bytes=42- >../dirA.links + cd ../shim_dirB + find . -type l | xargs ls -al | cut --bytes=42- >../dirB.links + cd .. + sed 's+shim_dirB+shim_dirA+' dirB.links >dirC.links + + linkdiffs="`diff dirA.links dirC.links|wc -l`" +fi + + +# check for directory differences (missing dirs, extra dirs, etc.) +dir_diffs="$(diff -q --no-dereference -r shim_dirA shim_dirB)" +dir_diffs_result=$? +if [ "${dir_diffs_result}" -ne 0 ]; then + dir_diffs_msg="and these other differences: ${dir_diffs/$'\n'/, }" +fi + + +# report result with links +if [[ $check_links == true ]]; then + + if [ "${diffs}" -eq 0 ] && [ "${dir_diffs_result}" -eq 0 ] && [ "${linkdiffs}" -eq 0 ]; then + echo "RESULT: Good! trees links the same: $(wc -l dirA.sums) files and $(wc -l dirA.links) links mirrored" + else + echo "RESULT: BAD tree differences in $diffs files, and $linkdiffs links ${dir_diffs_msg}" + fi + +# report result without links +else + + if [ "${diffs}" -eq 0 ] && [ "${dir_diffs_result}" -eq 0 ]; then + echo "RESULT: Good! trees the same: $(wc -l dirA.sums) files mirrored" + else + echo "RESULT: BAD trees have $diffs file differences ${dir_diffs_msg}" + fi +fi diff --git a/test_util/wait_for_retries.sh b/test_util/wait_for_retries.sh new file mode 100755 index 0000000..3a8b164 --- /dev/null +++ b/test_util/wait_for_retries.sh @@ -0,0 +1,21 @@ +#!/bin/bash +echo "Waiting for retries to finish..." +stalled=0 +stalled_value=-1 +retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)" +while [ $retry_msgcnt -gt 0 ]; do + printf "${flow_test_name} Still %4s messages to retry, waiting...\n" "$retry_msgcnt" + sleep 15 + retry_msgcnt="$(cat ~/.cache/sr3/subscribe/local_copy/*retry* 2>/dev/null | sort -u | wc -l)" + + if [ "${stalled_value}" == "${retry_msgcnt}" ]; then + stalled=$((stalled+1)); + if [ "${stalled}" == 5 ]; then + printf "\n Warning some retries stalled, skipping..., might want to check the logs\n\n" + retry_msgcnt=0 + fi + else + stalled_value=$retry_msgcnt + stalled=0 + fi +done