Improved integration tests

.github/workflows/test_on_self_hosted.yml

@@ -1,3 +1,7 @@
+# Local run (self-hosted): from repo root export UE_ROOT and set DISPLAY 
+# (for example `export UE_ROOT=/path/to/UnrealEngine; export DISPLAY=:0`), 
+# then execute the same sequence used below: 
+# `git submodule sync; git submodule update --init --recursive; ./setup_linux_dev_tools.sh; rm -rf build; ./build.sh simlibs_release; "$UE_ROOT/Engine/Build/BatchFiles/Linux/Build.sh" BlocksEditor Linux Development "$(pwd)/unreal/Blocks/Blocks.uproject" -waitmutex; python3 -m venv airsimenv; source airsimenv/bin/activate; python -m pip install --upgrade pip; python -m pip install "setuptools<81" wheel; python -m pip install -e "client/python/projectairsim[datacollection]"; python -m pip install pytest pytest-cov pytest-asyncio; launch Unreal headless on DISPLAY=:0; wait for ports 8989/8990; cd client/python/projectairsim/tests; python -m pytest -v --junitxml=pytest-results.xml`.
 name: ProjectAirSim CI (Self-Hosted)
 
 on:
@@ -26,6 +30,7 @@ jobs:
 
     env:
       UE_ROOT: ${{ secrets.UE_ROOT }}
+      PX4_ROOT: ${{ secrets.PX4_ROOT }}
       AIRSIM_PY_ENV: ${{ github.workspace }}/airsimenv
       DEBIAN_FRONTEND: noninteractive
 
@@ -164,7 +169,61 @@ jobs:
         run: |
           source "$AIRSIM_PY_ENV/bin/activate"
           cd client/python/projectairsim/tests
-          python -m pytest -v --junitxml=pytest-results.xml
+          python -m pytest -v --ignore=test_px4_sitl.py --junitxml=pytest-results.xml
+
+      - name: Resolve PX4 folder path
+        run: |
+          set -euo pipefail
+          PX4_DIR="${PX4_ROOT:-$GITHUB_WORKSPACE/PX4-Autopilot}"
+          if [ ! -d "$PX4_DIR" ]; then
+            echo "PX4 folder not found at: $PX4_DIR"
+            echo "Set secrets.PX4_ROOT or place PX4-Autopilot at $GITHUB_WORKSPACE/PX4-Autopilot"
+            exit 1
+          fi
+          echo "Using PX4 folder: $PX4_DIR"
+          echo "PX4_DIR=$PX4_DIR" >> "$GITHUB_ENV"
+
+      - name: Launch PX4 SITL
+        run: |
+          set -euo pipefail
+          rm -f px4.pid px4.log
+          (
+            cd "$PX4_DIR"
+            make px4_sitl_default none_iris
+          ) > px4.log 2>&1 &
+          echo $! > px4.pid
+          echo "PX4 launch PID: $(cat px4.pid)"
+          for i in {1..60}; do
+            if ps -p "$(cat px4.pid)" > /dev/null 2>&1; then
+              echo "PX4 SITL is running"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "PX4 SITL failed to start"
+          tail -n 300 px4.log || true
+          exit 1
+
+      - name: Run PX4 SITL tests
+        run: |
+          source "$AIRSIM_PY_ENV/bin/activate"
+          cd client/python/projectairsim/tests
+          python -m pytest -v test_px4_sitl.py --junitxml=pytest-results-px4_sitl.xml
+
+      - name: Stop PX4
+        if: always()
+        run: |
+          if [ -f px4.pid ]; then
+            kill "$(cat px4.pid)" || true
+          fi
+          pkill -f px4 || true
+          pkill -f none_iris || true
+
+      - name: Dump PX4 logs on failure
+        if: failure()
+        run: |
+          echo "==== PX4 Log Output ===="
+          cat px4.log || true
 
       - name: Stop Unreal
         if: always()

client/python/projectairsim/requirements.txt

@@ -2,6 +2,7 @@
 wheel
 pytest
 pytest-cov
+pytest-asyncio>=0.23
 
 # Install projectairsim package in editable development mode
 -e .

client/python/projectairsim/tests/conftest.py

@@ -0,0 +1,178 @@
+"""
+Copyright (C) Microsoft Corporation.
+Copyright (C) 2025 IAMAI CONSULTING CORP
+MIT License.
+
+Global pytest configuration for ProjectAirSim tests.
+Provides test isolation and stability improvements.
+"""
+
+import pytest
+import time
+import gc
+
+
+def _socket_is_closed(socket_obj) -> bool:
+    if socket_obj is None:
+        return True
+    return bool(getattr(socket_obj, "closed", False))
+
+
+def _client_needs_reconnect(client) -> bool:
+    if client is None:
+        return False
+    if not getattr(client, "state", False):
+        return True
+    if _socket_is_closed(getattr(client, "socket_topics", None)):
+        return True
+    if _socket_is_closed(getattr(client, "socket_services", None)):
+        return True
+    return False
+
+
+def _safe_disconnect(client) -> None:
+    if client is None:
+        return
+    try:
+        client.disconnect()
+    except Exception:
+        pass
+
+
+def _reload_world_if_possible(world) -> None:
+    if world is None:
+        return
+    sim_config = getattr(world, "sim_config", None)
+    if sim_config is None:
+        return
+    world.load_scene(sim_config, delay_after_load_sec=1.0)
+
+
+def _extract_client_and_world(request):
+    client = None
+    world = None
+
+    for fixture_name in ("client", "world", "drone", "multirotor", "robo_fixture", "robo"):
+        if fixture_name not in request.fixturenames:
+            continue
+
+        try:
+            fixture_obj = request.getfixturevalue(fixture_name)
+        except Exception:
+            continue
+
+        if fixture_name == "client":
+            client = fixture_obj
+        elif fixture_name == "world":
+            world = fixture_obj
+            client = getattr(fixture_obj, "client", client)
+        else:
+            client = getattr(fixture_obj, "client", client)
+            world = getattr(fixture_obj, "world", world)
+
+    return client, world
+
+
+@pytest.fixture(scope="function", autouse=True)
+def ensure_client_connected(request):
+    client, world = _extract_client_and_world(request)
+
+    if _client_needs_reconnect(client):
+        _safe_disconnect(client)
+        client.connect()
+        try:
+            client.get_topic_info()
+        except Exception:
+            pass
+        _reload_world_if_possible(world)
+
+    yield
+
+
+@pytest.fixture(scope="function", autouse=True)
+def test_isolation(request):
+    """
+    Automatic fixture to improve test isolation.
+
+    Adds a delay after each test to allow the simulator to settle,
+    preventing race conditions and state pollution between sequential tests.
+    This is especially important for tests that:
+    - Load/unload scenes
+    - Modify textures/materials
+    - Control drone movements
+    - Capture images
+
+    The delay can be skipped by marking a test with @pytest.mark.no_isolation
+    """
+    # Before test: force garbage collection to clean up any lingering objects
+    gc.collect()
+
+    yield
+
+    # Check if test is marked to skip isolation delay
+    if "no_isolation" in request.keywords:
+        return
+
+    # After test: add delay to allow simulator to settle
+    # This prevents issues with:
+    # - Scene loading timeouts
+    # - Texture/material update propagation
+    # - Image capture timing
+    time.sleep(1.5)
+
+    # Force garbage collection after test
+    gc.collect()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def session_setup_teardown():
+    """
+    Session-level setup and teardown.
+    Runs once before all tests and once after all tests complete.
+    """
+    print("\n" + "="*80)
+    print("Starting ProjectAirSim test session")
+    print("Tests will run sequentially with isolation delays")
+    print("="*80 + "\n")
+
+    yield
+
+    print("\n" + "="*80)
+    print("ProjectAirSim test session complete")
+    print("="*80 + "\n")
+
+
+@pytest.fixture(scope="module")
+def module_isolation():
+    """
+    Module-level isolation fixture.
+    Add longer delay between test modules to allow major state resets.
+    """
+    yield
+    # Longer delay between modules
+    print("\n--- Module complete, allowing simulator to reset ---")
+    time.sleep(3.0)
+
+
+# Custom pytest markers for test categorization
+def pytest_configure(config):
+    """Register custom markers for test categorization."""
+    config.addinivalue_line(
+        "markers", "no_isolation: skip automatic test isolation delay"
+    )
+    config.addinivalue_line(
+        "markers", "slow: marks tests as slow (deselect with '-m \"not slow\"')"
+    )
+    config.addinivalue_line(
+        "markers", "requires_scene(name): marks tests that require specific scene loaded"
+    )
+
+
+def pytest_runtest_makereport(item, call):
+    """
+    Hook to track test failures and potentially adjust isolation behavior.
+    """
+    if call.when == "call":
+        # Could add custom logic here to increase delays after failures
+        pass
+