Detect port leaks in tests (#414)

run_tests.sh now checks that all the ports were released.

Updated tests:
 - TestTestgresCommon::test_node_constructor__default
 - TestTestgresCommon::test_kill__ok
 - TestTestgresRemote::test_init__unk_LANG_and_LC_CTYPE

Author: dmitry-lipetsk

run_tests.sh

@@ -37,45 +37,105 @@ if [ -n "${TEST_CFG__REMOTE_HOST:-}" ] && [ -n "${TEST_CFG__REMOTE_USERNAME:-}"
     [ -n "${TEST_CFG__REMOTE_SSH_KEY:-}" ] && cmd_str="$cmd_str -i \"$TEST_CFG__REMOTE_SSH_KEY\""
     [ -n "${TEST_CFG__REMOTE_PORT:-}" ]    && cmd_str="$cmd_str -p \"$TEST_CFG__REMOTE_PORT\""
 
-    cmd_str="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\" 'df -T'"
-    REMOTE_FS_STATE_QUERY="$cmd_str"
+    REMOTE_SSH_PREFIX="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\""
 else
-    REMOTE_FS_STATE_QUERY=""
+    REMOTE_SSH_PREFIX=""
 fi
 
-show_fs_state() {
-    df -T
-    if [ -n "$REMOTE_FS_STATE_QUERY" ]; then
-        eval "$REMOTE_FS_STATE_QUERY"
+exec_command() {
+    local cmd="$1"
+    local prefix="$2"
+
+    eval "$prefix $cmd"
+}
+
+show_fs_state__impl() {
+    local prefix="$1"
+    local host_label="$2"
+
+    set +x
+    echo "------------- ${host_label} FS STATE"
+    set -x
+    exec_command "df -T" "$prefix"
+}
+
+check_leftover_ports__impl() {
+    local prefix="$1"
+    local host_label="$2"
+    local ports_dir="/tmp/testgres/ports"
+
+    set +x
+    echo "------------- Checking ${host_label} ports lock directory"
+    set -x
+
+    # Check command: will print FOUND if the directory exists and is not empty
+    local check_cmd="if [ -d '${ports_dir}' ] && [ \"\$(ls -A '${ports_dir}' 2>/dev/null)\" ]; then echo 'FOUND'; fi"
+
+    # Temporarily disable bash's instant drop (set +e) to safely intercept the result
+    set +e
+    local result
+    result=$(exec_command "$check_cmd" "$prefix")
+    set -e
+
+    set +x
+    if [ "$result" = "FOUND" ]; then
+        echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+        echo "ERROR: Leftover ports detected in $ports_dir on $host_label machine!"
+        echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+        set -x
+
+        # We display a list of frozen ports so that the culprits can be identified
+        exec_command "ls -la '$ports_dir'" "$prefix"
+        
+        # We hard-drop the entire control script
+        # sleep 3600
+        exit 1
+    else
+        echo "Clear. No leftover port locks."
+    fi
+    set -x
+}
+
+fs_verification__impl() {
+    show_fs_state__impl "$1" "$2"
+
+    check_leftover_ports__impl "$1" "$2"
+}
+
+fs_verification() {
+    fs_verification__impl "" "LOCAL"
+    
+    if [ -n "$REMOTE_SSH_PREFIX" ]; then
+        fs_verification__impl "$REMOTE_SSH_PREFIX" "REMOTE"
     fi
 }
 
 # ---------------------------------------- PATH
 
-show_fs_state
+fs_verification
 
 # run tests (PATH)
 time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # ---------------------------------------- PG_BIN
 
-show_fs_state
+fs_verification
 
 # run tests (PG_BIN)
 PG_BIN=$(pg_config --bindir) \
 time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # ---------------------------------------- PG_CONFIG
 
-show_fs_state
+fs_verification
 
 # run tests (PG_CONFIG)
 PG_CONFIG=$(pg_config --bindir)/pg_config \
 time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # ---------------------------------------- pg8000
 
-show_fs_state
+fs_verification
 
 # test pg8000
 pip uninstall -y psycopg2
@@ -85,7 +145,7 @@ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # ---------------------------------------- finish
 
-show_fs_state
+fs_verification
 
 # ---------------------------------------- coverage
 

tests/test_testgres_common.py

@@ -190,15 +190,15 @@ def test_default_username2(
         return
 
     def test_node_constructor__default(self):
-        node = PostgresNode()
-        assert node._os_ops is not None
-        assert isinstance(node._os_ops, OsOperations)
-        assert node._port_manager is not None
-        assert isinstance(node._port_manager, PortManager)
-        assert node._name is not None
-        assert type(node._name) is str
-        assert node._name != ""
-        assert node._base_dir is None
+        with PostgresNode() as node:
+            assert node._os_ops is not None
+            assert isinstance(node._os_ops, OsOperations)
+            assert node._port_manager is not None
+            assert isinstance(node._port_manager, PortManager)
+            assert node._name is not None
+            assert type(node._name) is str
+            assert node._name != ""
+            assert node._base_dir is None
         return
 
     def test_node_constructor__host(self):
@@ -771,6 +771,8 @@ def test_kill__ok(
         finally:
             if node.is_started:
                 node.stop()
+
+        node.cleanup(release_resources=True)
         return
 
     def test_kill_backgroud_writer__ok(

tests/test_testgres_remote.py

@@ -82,6 +82,7 @@ def test_init__unk_LANG_and_LC_CTYPE(self):
         assert isinstance(node_version, Version)
 
         if node.version < Version("11"):
+            node.cleanup(release_resources=True)
             pytest.skip("This test does not work on old PG10-.")
 
         try:
@@ -152,6 +153,8 @@ def test_init__unk_LANG_and_LC_CTYPE(self):
                 assert expectedMsg2 in exc.error
                 continue
 
+            node.cleanup(release_resources=True)
+
             if not errorIsDetected:
                 pytest.xfail("All the bad data are processed without errors!")