diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml new file mode 100644 index 0000000..9a5a8d8 --- /dev/null +++ b/.github/workflows/pr-checks.yaml @@ -0,0 +1,121 @@ +name: PR checks + +on: + pull_request: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + env: + PIP_ROOT_USER_ACTION: ignore + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files + continue-on-error: false + + run-tests: + runs-on: ubuntu-latest + env: + PIP_ROOT_USER_ACTION: ignore + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed templates + id: changed + run: | + # Extract the unique template folder name from every changed path under agents/ + # Works uniformly for base agents, community agents, and MCP templates alike. + # pytest -k matches on substrings of test IDs, so the folder name is sufficient: + # agents/base/langgraph-react-agent → matches test_base_agent[base/langgraph-react-agent] + # agents/community/mcp-autoai-template → matches test_mcp_autoai_template + AGENT_TEMPLATE_PARAMETER_NAMES=$(git diff --name-only origin/main...HEAD \ + | grep -E '^agents/(base|community)/[^/]+/' \ + | awk -F'/' '{print $2"/"$3}' \ + | uniq) + + MCP_TEMPLATE_TEST_NAMES=$(git diff --name-only origin/main...HEAD \ + | grep -E '^agents/(mcp/[^/]+|community/mcp-autoai-template)/' \ + | awk -F'/' '{print $3}' \ + | uniq \ + | tr '-' '_' \ + | sed 's/^/test_/') + + echo "Agent tests to run:" + echo "$AGENT_TEMPLATE_PARAMETER_NAMES" + echo "MCP tests to run:" + echo "$MCP_TEMPLATE_TEST_NAMES" + + TEMPLATE_NAMES=$(printf '%s\n' $AGENT_TEMPLATE_PARAMETER_NAMES $MCP_TEMPLATE_TEST_NAMES | sort -u) + + if [[ -z "$TEMPLATE_NAMES" ]]; then + echo "No agent or MCP templates changed — skipping tests." + echo "k_expr=" >> "$GITHUB_OUTPUT" + else + K_EXPR=$(echo "$TEMPLATE_NAMES" | tr '\n' ' ' | sed 's/ *$//' | sed 's/ / or /g') + echo "k_expr=$K_EXPR" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Python + if: steps.changed.outputs.k_expr != '' + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} + + - name: Install uv + if: steps.changed.outputs.k_expr != '' + uses: astral-sh/setup-uv@v5 + + - name: Install Poetry + if: steps.changed.outputs.k_expr != '' + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install test dependencies + if: steps.changed.outputs.k_expr != '' + run: | + pip install -r tests/requirements.txt + + - name: Run tests + if: steps.changed.outputs.k_expr != '' + env: + WATSONX_URL: ${{ secrets.WATSONX_URL }} + WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + WATSONX_COMPUTE_NAME: ${{ secrets.WATSONX_COMPUTE_NAME }} + WATSONX_IAM_SERVICE_ID_CRN: ${{ secrets.WATSONX_IAM_SERVICE_ID_CRN }} + COS_API_KEY: ${{ secrets.COS_API_KEY }} + COS_HMAC_ACCESS_KEY_ID: ${{ secrets.COS_HMAC_ACCESS_KEY_ID }} + COS_HMAC_SECRET_ACCESS_KEY: ${{ secrets.COS_HMAC_SECRET_ACCESS_KEY }} + COS_IAM_API_KEY_DESCRIPTION: ${{ secrets.COS_IAM_API_KEY_DESCRIPTION }} + COS_IAM_API_KEY_NAME: ${{ secrets.COS_IAM_API_KEY_NAME }} + COS_IAM_ROLE_CRN: ${{ secrets.COS_IAM_ROLE_CRN }} + COS_IAM_SERVICE_ID_CRN: ${{ secrets.COS_IAM_SERVICE_ID_CRN }} + COS_RESOURCE_INSTANCE_ID: ${{ secrets.COS_RESOURCE_INSTANCE_ID }} + PSQL_HOST: ${{ secrets.PSQL_HOST }} + PSQL_PORT: ${{ secrets.PSQL_PORT }} + PSQL_DATABASE: ${{ secrets.PSQL_DATABASE }} + PSQL_USERNAME: ${{ secrets.PSQL_USERNAME }} + PSQL_PASSWORD: ${{ secrets.PSQL_PASSWORD }} + MILVUS_HOST: ${{ secrets.MILVUS_HOST }} + MILVUS_PORT: ${{ secrets.MILVUS_PORT }} + MILVUS_USERNAME: ${{ secrets.MILVUS_USERNAME }} + MILVUS_PASSWORD: ${{ secrets.MILVUS_PASSWORD }} + run: pytest -vv tests -k "${{ steps.changed.outputs.k_expr }}" diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml new file mode 100644 index 0000000..7c1c2d1 --- /dev/null +++ b/.github/workflows/run-tests.yaml @@ -0,0 +1,59 @@ +name: Run tests defined in the repository + +on: + push: + branches: [main] + schedule: + - cron: "0 7 * * 0" + +jobs: + run-tests: + runs-on: ubuntu-latest + env: + PIP_ROOT_USER_ACTION: ignore + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install test dependencies + run: | + pip install -r tests/requirements.txt + + - name: Run tests + env: + WATSONX_URL: ${{ secrets.WATSONX_URL }} + WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + WATSONX_COMPUTE_NAME: ${{ secrets.WATSONX_COMPUTE_NAME }} + WATSONX_IAM_SERVICE_ID_CRN: ${{ secrets.WATSONX_IAM_SERVICE_ID_CRN }} + COS_API_KEY: ${{ secrets.COS_API_KEY }} + COS_HMAC_ACCESS_KEY_ID: ${{ secrets.COS_HMAC_ACCESS_KEY_ID }} + COS_HMAC_SECRET_ACCESS_KEY: ${{ secrets.COS_HMAC_SECRET_ACCESS_KEY }} + COS_IAM_API_KEY_DESCRIPTION: ${{ secrets.COS_IAM_API_KEY_DESCRIPTION }} + COS_IAM_API_KEY_NAME: ${{ secrets.COS_IAM_API_KEY_NAME }} + COS_IAM_ROLE_CRN: ${{ secrets.COS_IAM_ROLE_CRN }} + COS_IAM_SERVICE_ID_CRN: ${{ secrets.COS_IAM_SERVICE_ID_CRN }} + COS_RESOURCE_INSTANCE_ID: ${{ secrets.COS_RESOURCE_INSTANCE_ID }} + PSQL_HOST: ${{ secrets.PSQL_HOST }} + PSQL_PORT: ${{ secrets.PSQL_PORT }} + PSQL_DATABASE: ${{ secrets.PSQL_DATABASE }} + PSQL_USERNAME: ${{ secrets.PSQL_USERNAME }} + PSQL_PASSWORD: ${{ secrets.PSQL_PASSWORD }} + MILVUS_HOST: ${{ secrets.MILVUS_HOST }} + MILVUS_PORT: ${{ secrets.MILVUS_PORT }} + MILVUS_USERNAME: ${{ secrets.MILVUS_USERNAME }} + MILVUS_PASSWORD: ${{ secrets.MILVUS_PASSWORD }} + run: pytest -vv tests diff --git a/tests/conftest.py b/tests/conftest.py index ecc699b..990c349 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ import os from pathlib import Path import subprocess +import sys from tempfile import TemporaryDirectory from typing import Generator, cast from ibm_watsonx_ai.experiment.autoai.optimizers import RemoteAutoPipelines @@ -30,7 +31,7 @@ @pytest.fixture(scope="session", name="context_free_api_client") def fixture_context_free_api_client() -> APIClient: credentials = Credentials( - url=os.environ["WX_URL"], api_key=os.environ["WX_API_KEY"] + url=os.environ["WATSONX_URL"], api_key=os.environ["WATSONX_API_KEY"] ) return APIClient(credentials) @@ -84,20 +85,6 @@ def fixture_project_api_client( return api_client -@pytest.fixture(scope="session", name="env_file_values") -def fixture_env_file_values(space_id: str) -> dict[str, str]: - os_vars_mapping = { - "WATSONX_APIKEY": "WX_API_KEY", # pragma: allowlist secret - "WATSONX_API_KEY": "WX_API_KEY", # pragma: allowlist secret - "WATSONX_URL": "WX_URL", - } - - env_values = {cli: os.environ[env] for cli, env in os_vars_mapping.items()} - env_values["WATSONX_SPACE_ID"] = space_id - - return env_values - - @pytest.fixture(name="tmp_dir") def fixture_tmp_dir() -> Generator[str, None, None]: with TemporaryDirectory() as tmp_dir: @@ -107,9 +94,10 @@ def fixture_tmp_dir() -> Generator[str, None, None]: @pytest.fixture(name="test_venv_path") def fixture_test_venv_path() -> Path: venv_name = ".venv_test" + version = sys.version.split(" ", 1)[0] subprocess.run( - [Path(__file__).parent / "create_test_venv_with_uv.sh", venv_name, "3.11"], + [Path(__file__).parent / "create_test_venv_with_uv.sh", venv_name, version], check=True, ) diff --git a/tests/container_utils.py b/tests/container_utils.py index 2f1e2f9..213f8e0 100644 --- a/tests/container_utils.py +++ b/tests/container_utils.py @@ -59,7 +59,7 @@ def delete_container( def delete_old_containers(api_client: APIClient, container_type: ContainerType) -> None: - delete_threshold = datetime.now(tz=timezone.utc) - timedelta(days=0) + delete_threshold = datetime.now(tz=timezone.utc) - timedelta(days=1) if container_type == "project": container_df = api_client.projects.list() @@ -88,8 +88,8 @@ def create_new_space(api_client: APIClient) -> tuple[str, str]: "resource_crn": os.environ["COS_RESOURCE_INSTANCE_ID"], }, api_client.spaces.ConfigurationMetaNames.COMPUTE: { - "name": os.environ["WX_NAME"], - "crn": os.environ["WX_IAM_SERVICE_ID_CRN"], + "name": os.environ["WATSONX_COMPUTE_NAME"], + "crn": os.environ["WATSONX_IAM_SERVICE_ID_CRN"], }, api_client.spaces.ConfigurationMetaNames.TYPE: "wx", } @@ -111,7 +111,9 @@ def create_new_project(api_client: APIClient) -> tuple[str, str]: storage_guid = ( os.environ["COS_RESOURCE_INSTANCE_ID"].replace("::", "").split(":")[-1] ) - compute_guid = os.environ["WX_IAM_SERVICE_ID_CRN"].replace("::", "").split(":")[-1] + compute_guid = ( + os.environ["WATSONX_IAM_SERVICE_ID_CRN"].replace("::", "").split(":")[-1] + ) meta_props = { api_client.projects.ConfigurationMetaNames.NAME: project_name, @@ -123,8 +125,8 @@ def create_new_project(api_client: APIClient) -> tuple[str, str]: }, api_client.projects.ConfigurationMetaNames.COMPUTE: { "type": "machine_learning", - "name": os.environ["WX_NAME"], - "crn": os.environ["WX_IAM_SERVICE_ID_CRN"], + "name": os.environ["WATSONX_COMPUTE_NAME"], + "crn": os.environ["WATSONX_IAM_SERVICE_ID_CRN"], "guid": compute_guid, }, api_client.projects.ConfigurationMetaNames.TYPE: "wx", diff --git a/tests/example.pytest.ini b/tests/example.pytest.ini new file mode 100644 index 0000000..b638318 --- /dev/null +++ b/tests/example.pytest.ini @@ -0,0 +1,31 @@ +[pytest] +env = + COS_ENDPOINT_URL=https://s3.us-south.cloud-object-storage.appdomain.cloud + COS_API_KEY= + + COS_HMAC_ACCESS_KEY_ID= + COS_HMAC_SECRET_ACCESS_KEY= + COS_ENDPOINTS=https://control.cloud-object-storage.cloud.ibm.com/v2/endpoints + + COS_IAM_API_KEY_DESCRIPTION= + COS_IAM_API_KEY_NAME= + COS_IAM_ROLE_CRN= + COS_IAM_SERVICE_ID_CRN= + + COS_RESOURCE_INSTANCE_ID= + + WATSONX_API_KEY= + WATSONX_URL=https://us-south.ml.cloud.ibm.com + WATSONX_COMPUTE_NAME= + WATSONX_IAM_SERVICE_ID_CRN= + + PSQL_HOST= + PSQL_PORT= + PSQL_DATABASE= + PSQL_USERNAME= + PSQL_PASSWORD= + + MILVUS_HOST= + MILVUS_PORT= + MILVUS_USERNAME= + MILVUS_PASSWORD= diff --git a/tests/test_agents.py b/tests/test_agents.py index 1c80611..1c082e2 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -3,7 +3,14 @@ import pytest -from utils import AGENTS_PATH, clone_agent_template, create_env_file, use_cli, run_cli +from utils import ( + AGENTS_PATH, + clone_agent_template, + create_env_file, + get_env_vars, + use_cli, + run_cli, +) logger = logging.getLogger(__name__) @@ -11,6 +18,7 @@ class TestAgents: SKIPPED_TESTS = { + "base/crewai-websearch-agent": "CrewAI is not compatible with genai-A25-py3.12 software specification", "community/langgraph-graph-rag": "Neo4j credentials required to run the AI service in the Cloud", "community/langgraph-tavily-tool": "Tavily credentials required to run the AI service in the Cloud", } @@ -25,6 +33,8 @@ class TestAgents: 'tool_config_vectorIndexId = "{}"': ("vector_index_id", "fixture"), } + USER_MESSAGE_CONTENT = "Use one of your available tools with any example input and tell me what it returned." + @staticmethod def _get_agent_names(dir_name: str) -> list[str]: agents_path = AGENTS_PATH / dir_name @@ -37,7 +47,7 @@ def _get_agent_names(dir_name: str) -> list[str]: ] def _create_config_toml_file( - self, env_file_values: dict[str, str], request: pytest.FixtureRequest + self, env_vars: dict[str, str], request: pytest.FixtureRequest ) -> None: with open("config.toml.example", encoding="utf-8") as file: config_toml_content = file.read() @@ -49,7 +59,7 @@ def _create_config_toml_file( value, value_type = replacement match value_type: case "env": - value = env_file_values[value] + value = env_vars[value] case "fixture": value = request.getfixturevalue(value) @@ -80,11 +90,7 @@ def _run_template_unit_tests(self, venv_path: Path) -> None: def _run_template_invoke(self, venv_path: Path) -> None: run_cli( venv_path, - [ - "template", - "invoke", - "Call your tool with some example value and tell me what you asked for and what you received.", - ], + ["template", "invoke", self.USER_MESSAGE_CONTENT], input=b"y", # for package installation ) @@ -99,7 +105,7 @@ def _run_service_get(self, venv_path: Path, deployment_id: str) -> None: run_cli(venv_path, ["service", "get", deployment_id]) def _run_service_invoke(self, venv_path: Path) -> None: - run_cli(venv_path, ["service", "invoke", "Hello"]) + run_cli(venv_path, ["service", "invoke", self.USER_MESSAGE_CONTENT]) def _run_service_delete(self, venv_path: Path, deployment_id: str) -> None: run_cli( @@ -113,7 +119,6 @@ def _setup_template( venv_path: Path, agent_name: str, tmp_dir: str, - env_file_values: dict[str, str], monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, ) -> None: @@ -121,9 +126,10 @@ def _setup_template( pytest.skip(self.SKIPPED_TESTS[agent_name]) clone_agent_template(venv_path, tmp_dir, agent_name, monkeypatch) - create_env_file(env_file_values) - self._create_config_toml_file(env_file_values, request) + env_vars = get_env_vars() + create_env_file(env_vars) + self._create_config_toml_file(env_vars, request) def _template_tests(self, venv_path: Path, agent_name: str) -> None: if use_cli(): @@ -141,11 +147,11 @@ def _service_tests(self, venv_path: Path) -> None: self._run_service_invoke(venv_path) self._run_service_delete(venv_path, deployment_id) + @pytest.mark.usefixtures("space_id") @pytest.mark.parametrize("agent_name", _get_agent_names("base")) def test_base_agent( self, agent_name: str, - env_file_values: dict[str, str], tmp_dir: str, test_venv_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -155,18 +161,17 @@ def test_base_agent( test_venv_path, agent_name, tmp_dir, - env_file_values, monkeypatch, request, ) self._template_tests(test_venv_path, agent_name) self._service_tests(test_venv_path) + @pytest.mark.usefixtures("space_id") @pytest.mark.parametrize("agent_name", _get_agent_names("community")) def test_community_agent( self, agent_name: str, - env_file_values: dict[str, str], tmp_dir: str, test_venv_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -176,7 +181,6 @@ def test_community_agent( test_venv_path, agent_name, tmp_dir, - env_file_values, monkeypatch, request, ) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 98f69c4..ae85d6c 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -5,7 +5,7 @@ from typing import Generator import pytest -from utils import clone_agent_template, create_env_file, run_cli +from utils import clone_agent_template, create_env_file, get_env_vars, run_cli logger = logging.getLogger(__name__) @@ -59,7 +59,6 @@ def test_mcp_autoai_template( self, test_venv_path: Path, tmp_dir: str, - env_file_values: dict[str, str], credit_risk_deployment_id: str, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -69,10 +68,10 @@ def test_mcp_autoai_template( run_cli(test_venv_path, ["install", "-r", "requirements.txt"], "pip") - create_env_file( - env_file_values - | {"WATSONX_CREDIT_RISK_DEPLOYMENT_ID": credit_risk_deployment_id} + env_vars = get_env_vars( + {"WATSONX_CREDIT_RISK_DEPLOYMENT_ID": credit_risk_deployment_id} ) + create_env_file(env_vars) with self._run_server(test_venv_path): for prompt, expected_outputs in self.EXPECTED_OUTPUTS_BY_PROMPT.items(): diff --git a/tests/utils.py b/tests/utils.py index d58c5b9..5dd59bf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,6 +9,13 @@ AGENTS_PATH = Path(__file__).parents[1] / "agents" +ENV_VARS_MAPPING = { + "WATSONX_APIKEY": "WATSONX_API_KEY", # pragma: allowlist secret + "WATSONX_API_KEY": "WATSONX_API_KEY", # pragma: allowlist secret + "WATSONX_URL": "WATSONX_URL", + "WATSONX_SPACE_ID": "WATSONX_SPACE_ID", +} + logger = logging.getLogger(__name__) @@ -68,11 +75,20 @@ def clone_agent_template( return target_dir -def create_env_file(env_file_values: dict[str, str]) -> None: +def get_env_vars(additional_env_vars: dict[str, str] | None = None) -> dict[str, str]: + env_vars = {cli: os.environ[env] for cli, env in ENV_VARS_MAPPING.items()} + + if additional_env_vars: + env_vars.update(additional_env_vars) + + return env_vars + + +def create_env_file(env_vars: dict[str, str]) -> None: with open("template.env", encoding="utf-8") as file: env_file_content = file.read() - for key, value in env_file_values.items(): + for key, value in env_vars.items(): env_file_content = re.sub( rf"{key}=[^\n]*\n", f"{key}={value}\n", env_file_content )