chore: Added agent and MCP template tests workflow

.github/workflows/pr-checks.yaml

@@ -0,0 +1,121 @@
+name: PR checks
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    env:
+      PIP_ROOT_USER_ACTION: ignore
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ vars.PYTHON_VERSION }}
+
+    - name: Install dependencies
+      run: |
+        pip install pre-commit
+
+    - name: Run pre-commit
+      run: pre-commit run --all-files
+      continue-on-error: false
+
+  run-tests:
+    runs-on: ubuntu-latest
+    env:
+      PIP_ROOT_USER_ACTION: ignore
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect changed templates
+        id: changed
+        run: |
+          # Extract the unique template folder name from every changed path under agents/
+          # Works uniformly for base agents, community agents, and MCP templates alike.
+          # pytest -k matches on substrings of test IDs, so the folder name is sufficient:
+          #   agents/base/langgraph-react-agent   → matches test_base_agent[base/langgraph-react-agent]
+          #   agents/community/mcp-autoai-template → matches test_mcp_autoai_template
+          AGENT_TEMPLATE_PARAMETER_NAMES=$(git diff --name-only origin/main...HEAD \
+            | grep -E '^agents/(base|community)/[^/]+/' \
+            | awk -F'/' '{print $2"/"$3}' \
+            | uniq)
+
+          MCP_TEMPLATE_TEST_NAMES=$(git diff --name-only origin/main...HEAD \
+            | grep -E '^agents/(mcp/[^/]+|community/mcp-autoai-template)/' \
+            | awk -F'/' '{print $3}' \
+            | uniq \
+            | tr '-' '_' \
+            | sed 's/^/test_/')
+
+          echo "Agent tests to run:"
+          echo "$AGENT_TEMPLATE_PARAMETER_NAMES"
+          echo "MCP tests to run:"
+          echo "$MCP_TEMPLATE_TEST_NAMES"
+
+          TEMPLATE_NAMES=$(printf '%s\n' $AGENT_TEMPLATE_PARAMETER_NAMES $MCP_TEMPLATE_TEST_NAMES | sort -u)
+
+          if [[ -z "$TEMPLATE_NAMES" ]]; then
+            echo "No agent or MCP templates changed — skipping tests."
+            echo "k_expr=" >> "$GITHUB_OUTPUT"
+          else
+            K_EXPR=$(echo "$TEMPLATE_NAMES" | tr '\n' ' ' | sed 's/ *$//' | sed 's/ / or /g')
+            echo "k_expr=$K_EXPR" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Python
+        if: steps.changed.outputs.k_expr != ''
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ vars.PYTHON_VERSION }}
+
+      - name: Install uv
+        if: steps.changed.outputs.k_expr != ''
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install Poetry
+        if: steps.changed.outputs.k_expr != ''
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Install test dependencies
+        if: steps.changed.outputs.k_expr != ''
+        run: |
+          pip install -r tests/requirements.txt
+
+      - name: Run tests
+        if: steps.changed.outputs.k_expr != ''
+        env:
+          WATSONX_URL: ${{ secrets.WATSONX_URL }}
+          WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}
+          WATSONX_COMPUTE_NAME: ${{ secrets.WATSONX_COMPUTE_NAME }}
+          WATSONX_IAM_SERVICE_ID_CRN: ${{ secrets.WATSONX_IAM_SERVICE_ID_CRN }}
+          COS_API_KEY: ${{ secrets.COS_API_KEY }}
+          COS_HMAC_ACCESS_KEY_ID: ${{ secrets.COS_HMAC_ACCESS_KEY_ID }}
+          COS_HMAC_SECRET_ACCESS_KEY: ${{ secrets.COS_HMAC_SECRET_ACCESS_KEY }}
+          COS_IAM_API_KEY_DESCRIPTION: ${{ secrets.COS_IAM_API_KEY_DESCRIPTION }}
+          COS_IAM_API_KEY_NAME: ${{ secrets.COS_IAM_API_KEY_NAME }}
+          COS_IAM_ROLE_CRN: ${{ secrets.COS_IAM_ROLE_CRN }}
+          COS_IAM_SERVICE_ID_CRN: ${{ secrets.COS_IAM_SERVICE_ID_CRN }}
+          COS_RESOURCE_INSTANCE_ID: ${{ secrets.COS_RESOURCE_INSTANCE_ID }}
+          PSQL_HOST: ${{ secrets.PSQL_HOST }}
+          PSQL_PORT: ${{ secrets.PSQL_PORT }}
+          PSQL_DATABASE: ${{ secrets.PSQL_DATABASE }}
+          PSQL_USERNAME: ${{ secrets.PSQL_USERNAME }}
+          PSQL_PASSWORD: ${{ secrets.PSQL_PASSWORD }}
+          MILVUS_HOST: ${{ secrets.MILVUS_HOST }}
+          MILVUS_PORT: ${{ secrets.MILVUS_PORT }}
+          MILVUS_USERNAME: ${{ secrets.MILVUS_USERNAME }}
+          MILVUS_PASSWORD: ${{ secrets.MILVUS_PASSWORD }}
+        run: pytest -vv tests -k "${{ steps.changed.outputs.k_expr }}"

.github/workflows/run-tests.yaml

@@ -0,0 +1,59 @@
+name: Run tests defined in the repository
+
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "0 7 * * 0"
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    env:
+      PIP_ROOT_USER_ACTION: ignore
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ vars.PYTHON_VERSION }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Install test dependencies
+        run: |
+          pip install -r tests/requirements.txt
+
+      - name: Run tests
+        env:
+          WATSONX_URL: ${{ secrets.WATSONX_URL }}
+          WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}
+          WATSONX_COMPUTE_NAME: ${{ secrets.WATSONX_COMPUTE_NAME }}
+          WATSONX_IAM_SERVICE_ID_CRN: ${{ secrets.WATSONX_IAM_SERVICE_ID_CRN }}
+          COS_API_KEY: ${{ secrets.COS_API_KEY }}
+          COS_HMAC_ACCESS_KEY_ID: ${{ secrets.COS_HMAC_ACCESS_KEY_ID }}
+          COS_HMAC_SECRET_ACCESS_KEY: ${{ secrets.COS_HMAC_SECRET_ACCESS_KEY }}
+          COS_IAM_API_KEY_DESCRIPTION: ${{ secrets.COS_IAM_API_KEY_DESCRIPTION }}
+          COS_IAM_API_KEY_NAME: ${{ secrets.COS_IAM_API_KEY_NAME }}
+          COS_IAM_ROLE_CRN: ${{ secrets.COS_IAM_ROLE_CRN }}
+          COS_IAM_SERVICE_ID_CRN: ${{ secrets.COS_IAM_SERVICE_ID_CRN }}
+          COS_RESOURCE_INSTANCE_ID: ${{ secrets.COS_RESOURCE_INSTANCE_ID }}
+          PSQL_HOST: ${{ secrets.PSQL_HOST }}
+          PSQL_PORT: ${{ secrets.PSQL_PORT }}
+          PSQL_DATABASE: ${{ secrets.PSQL_DATABASE }}
+          PSQL_USERNAME: ${{ secrets.PSQL_USERNAME }}
+          PSQL_PASSWORD: ${{ secrets.PSQL_PASSWORD }}
+          MILVUS_HOST: ${{ secrets.MILVUS_HOST }}
+          MILVUS_PORT: ${{ secrets.MILVUS_PORT }}
+          MILVUS_USERNAME: ${{ secrets.MILVUS_USERNAME }}
+          MILVUS_PASSWORD: ${{ secrets.MILVUS_PASSWORD }}
+        run: pytest -vv tests

tests/conftest.py

@@ -3,6 +3,7 @@
 import os
 from pathlib import Path
 import subprocess
+import sys
 from tempfile import TemporaryDirectory
 from typing import Generator, cast
 from ibm_watsonx_ai.experiment.autoai.optimizers import RemoteAutoPipelines
@@ -30,7 +31,7 @@
 @pytest.fixture(scope="session", name="context_free_api_client")
 def fixture_context_free_api_client() -> APIClient:
     credentials = Credentials(
-        url=os.environ["WX_URL"], api_key=os.environ["WX_API_KEY"]
+        url=os.environ["WATSONX_URL"], api_key=os.environ["WATSONX_API_KEY"]
     )
 
     return APIClient(credentials)
@@ -84,20 +85,6 @@ def fixture_project_api_client(
     return api_client
 
 
-@pytest.fixture(scope="session", name="env_file_values")
-def fixture_env_file_values(space_id: str) -> dict[str, str]:
-    os_vars_mapping = {
-        "WATSONX_APIKEY": "WX_API_KEY",  # pragma: allowlist secret
-        "WATSONX_API_KEY": "WX_API_KEY",  # pragma: allowlist secret
-        "WATSONX_URL": "WX_URL",
-    }
-
-    env_values = {cli: os.environ[env] for cli, env in os_vars_mapping.items()}
-    env_values["WATSONX_SPACE_ID"] = space_id
-
-    return env_values
-
-
 @pytest.fixture(name="tmp_dir")
 def fixture_tmp_dir() -> Generator[str, None, None]:
     with TemporaryDirectory() as tmp_dir:
@@ -107,9 +94,10 @@ def fixture_tmp_dir() -> Generator[str, None, None]:
 @pytest.fixture(name="test_venv_path")
 def fixture_test_venv_path() -> Path:
     venv_name = ".venv_test"
+    version = sys.version.split(" ", 1)[0]
 
     subprocess.run(
-        [Path(__file__).parent / "create_test_venv_with_uv.sh", venv_name, "3.11"],
+        [Path(__file__).parent / "create_test_venv_with_uv.sh", venv_name, version],
         check=True,
     )
 

tests/container_utils.py

@@ -59,7 +59,7 @@ def delete_container(
 
 
 def delete_old_containers(api_client: APIClient, container_type: ContainerType) -> None:
-    delete_threshold = datetime.now(tz=timezone.utc) - timedelta(days=0)
+    delete_threshold = datetime.now(tz=timezone.utc) - timedelta(days=1)
 
     if container_type == "project":
         container_df = api_client.projects.list()
@@ -88,8 +88,8 @@ def create_new_space(api_client: APIClient) -> tuple[str, str]:
             "resource_crn": os.environ["COS_RESOURCE_INSTANCE_ID"],
         },
         api_client.spaces.ConfigurationMetaNames.COMPUTE: {
-            "name": os.environ["WX_NAME"],
-            "crn": os.environ["WX_IAM_SERVICE_ID_CRN"],
+            "name": os.environ["WATSONX_COMPUTE_NAME"],
+            "crn": os.environ["WATSONX_IAM_SERVICE_ID_CRN"],
         },
         api_client.spaces.ConfigurationMetaNames.TYPE: "wx",
     }
@@ -111,7 +111,9 @@ def create_new_project(api_client: APIClient) -> tuple[str, str]:
     storage_guid = (
         os.environ["COS_RESOURCE_INSTANCE_ID"].replace("::", "").split(":")[-1]
     )
-    compute_guid = os.environ["WX_IAM_SERVICE_ID_CRN"].replace("::", "").split(":")[-1]
+    compute_guid = (
+        os.environ["WATSONX_IAM_SERVICE_ID_CRN"].replace("::", "").split(":")[-1]
+    )
 
     meta_props = {
         api_client.projects.ConfigurationMetaNames.NAME: project_name,
@@ -123,8 +125,8 @@ def create_new_project(api_client: APIClient) -> tuple[str, str]:
         },
         api_client.projects.ConfigurationMetaNames.COMPUTE: {
             "type": "machine_learning",
-            "name": os.environ["WX_NAME"],
-            "crn": os.environ["WX_IAM_SERVICE_ID_CRN"],
+            "name": os.environ["WATSONX_COMPUTE_NAME"],
+            "crn": os.environ["WATSONX_IAM_SERVICE_ID_CRN"],
             "guid": compute_guid,
         },
         api_client.projects.ConfigurationMetaNames.TYPE: "wx",

tests/example.pytest.ini

@@ -0,0 +1,31 @@
+[pytest]
+env =
+    COS_ENDPOINT_URL=https://s3.us-south.cloud-object-storage.appdomain.cloud
+    COS_API_KEY=
+
+    COS_HMAC_ACCESS_KEY_ID=
+    COS_HMAC_SECRET_ACCESS_KEY=
+    COS_ENDPOINTS=https://control.cloud-object-storage.cloud.ibm.com/v2/endpoints
+
+    COS_IAM_API_KEY_DESCRIPTION=
+    COS_IAM_API_KEY_NAME=
+    COS_IAM_ROLE_CRN=
+    COS_IAM_SERVICE_ID_CRN=
+
+    COS_RESOURCE_INSTANCE_ID=
+
+    WATSONX_API_KEY=
+    WATSONX_URL=https://us-south.ml.cloud.ibm.com
+    WATSONX_COMPUTE_NAME=
+    WATSONX_IAM_SERVICE_ID_CRN=
+
+    PSQL_HOST=
+    PSQL_PORT=
+    PSQL_DATABASE=
+    PSQL_USERNAME=
+    PSQL_PASSWORD=
+
+    MILVUS_HOST=
+    MILVUS_PORT=
+    MILVUS_USERNAME=
+    MILVUS_PASSWORD=