diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index fb4737052..f5db04f32 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,3 +1,7 @@ # Unreleased ## Summary + +## Feature + +* #691: Started customization of PTB workflows by defining the YML schema diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 47ac24ff0..c66fa5d92 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -279,3 +279,20 @@ def github_template_dict(self) -> dict[str, Any]: "os_version": self.os_version, "python_versions": self.python_versions, } + + @computed_field # type: ignore[misc] + @property + def github_workflow_patcher_yaml(self) -> Path | None: + """ + For customizing the GitHub workflow templates provided by the PTB, + a project can define a `.workflow-patcher.yml` file containing instructions to + delete or modify jobs in the PTB template. Modification includes replacing and + inserting steps. + + This feature is a work-in-progress that will be completed with: + https://github.com/exasol/python-toolbox/issues/690 + """ + workflow_patcher_yaml = self.root_path / ".workflow-patcher.yml" + if workflow_patcher_yaml.exists(): + return workflow_patcher_yaml + return None diff --git a/exasol/toolbox/util/workflows/patch_workflow.py b/exasol/toolbox/util/workflows/patch_workflow.py new file mode 100644 index 000000000..c4bc10d5a --- /dev/null +++ b/exasol/toolbox/util/workflows/patch_workflow.py @@ -0,0 +1,59 @@ +from enum import Enum +from pathlib import Path +from typing import Any + +from pydantic import ( + BaseModel, + ConfigDict, + Field, +) +from ruamel.yaml import CommentedMap + +from exasol.toolbox.util.workflows.render_yaml import YamlRenderer + + +class ActionType(str, Enum): + INSERT_AFTER = "INSERT_AFTER" + REPLACE = "REPLACE" + + +class StepContent(BaseModel): + model_config = ConfigDict(extra="allow") # This allows extra fields + + name: str + id: str + uses: str | None = None + run: str | None = None + with_: dict[str, Any] | None = Field(None, alias="with") + env: dict[str, str] | None = None + + +class StepCustomization(BaseModel): + action: ActionType + job: str + step_id: str + content: StepContent + + +class Workflow(BaseModel): + name: str + remove_jobs: list[str] = Field(default_factory=list) + step_customizations: list[StepCustomization] = Field(default_factory=list) + + +class WorkflowPatcherConfig(BaseModel): + workflows: list[Workflow] + + +class WorkflowPatcher(YamlRenderer): + """ + The :class:`WorkflowPatcher` enables users to define a YAML file + to customize PTB-provided workflows by removing or modifying jobs in the file. + A job can be modified by replacing or inserting steps. + The provided YAML file must meet the conditions of :class:`WorkflowPatcherConfig`. + """ + + def get_yaml_dict(self, file_path: Path) -> CommentedMap: + loaded_yaml = super().get_yaml_dict(file_path) + WorkflowPatcherConfig.model_validate(loaded_yaml) + return loaded_yaml diff --git a/exasol/toolbox/util/workflows/process_template.py b/exasol/toolbox/util/workflows/process_template.py new file mode 100644 index 000000000..febebaa35 --- /dev/null +++ b/exasol/toolbox/util/workflows/process_template.py @@ -0,0 +1,19 @@ +from pathlib import Path + +from exasol.toolbox.util.workflows.render_yaml import YamlRenderer + + +class WorkflowRenderer(YamlRenderer): + """ + The :class:`WorkflowRenderer` renders a workflow template provided by the PTB into + a final workflow. It renders the final workflow by: + - resolving Jinja variables. + - standardizing formatting via ruamel.yaml for a consistent output. + """ + + def render(self, file_path: Path) -> str: + """ + Render the template to the contents of a valid GitHub workflow. + """ + workflow_dict = self.get_yaml_dict(file_path) + return self.get_as_string(workflow_dict) diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py new file mode 100644 index 000000000..23b57dd57 --- /dev/null +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -0,0 +1,69 @@ +import io +from dataclasses import dataclass +from inspect import cleandoc +from pathlib import Path +from typing import Any + +from jinja2 import Environment +from ruamel.yaml import ( + YAML, + CommentedMap, +) + +jinja_env = Environment( + variable_start_string="((", variable_end_string="))", autoescape=True +) + + +@dataclass(frozen=True) +class YamlRenderer: + """ + The :class:`YamlRenderer` provides a standardised interface for rendering YAML + files within the PTB. To simplify configuration and reduce manual coordination, + use Jinja variables as defined in :meth:`BaseConfig.github_template_dict` in your + YAML files. + """ + + github_template_dict: dict[str, Any] + + @staticmethod + def _get_standard_yaml() -> YAML: + """ + Prepare standard YAML class. + """ + yaml = YAML() + yaml.width = 200 + yaml.preserve_quotes = True + yaml.sort_base_mapping_type_on_output = False # type: ignore + yaml.indent(mapping=2, sequence=4, offset=2) + return yaml + + def _render_with_jinja(self, input_str: str) -> str: + """ + Render the template with Jinja. + """ + jinja_template = jinja_env.from_string(input_str) + return jinja_template.render(self.github_template_dict) + + def get_yaml_dict(self, file_path: Path) -> CommentedMap: + """ + Load a file as a CommentedMap (dictionary form of a YAML), after + rendering it with Jinja. + """ + with file_path.open("r", encoding="utf-8") as stream: + raw_content = stream.read() + + workflow_string = self._render_with_jinja(raw_content) + + yaml = self._get_standard_yaml() + return yaml.load(workflow_string) + + def get_as_string(self, yaml_dict: CommentedMap) -> str: + """ + Output a YAML string. + """ + yaml = self._get_standard_yaml() + with io.StringIO() as stream: + yaml.dump(yaml_dict, stream) + workflow_string = stream.getvalue() + return cleandoc(workflow_string) diff --git a/exasol/toolbox/util/workflows/template_processing.py b/exasol/toolbox/util/workflows/template_processing.py deleted file mode 100644 index e78deba45..000000000 --- a/exasol/toolbox/util/workflows/template_processing.py +++ /dev/null @@ -1,44 +0,0 @@ -from dataclasses import dataclass -from typing import Any - -from jinja2 import Environment - -jinja_env = Environment( - variable_start_string="((", variable_end_string="))", autoescape=True -) - -import io -from inspect import cleandoc - -from ruamel.yaml import YAML - - -@dataclass(frozen=True) -class TemplateRenderer: - template_str: str - github_template_dict: dict[str, Any] - - def _render_with_jinja(self, input_str: str) -> str: - """ - Render the template with Jinja. - """ - jinja_template = jinja_env.from_string(input_str) - return jinja_template.render(self.github_template_dict) - - def render_to_workflow(self) -> str: - """ - Render the template to the contents of a valid GitHub workflow. - """ - yaml = YAML() - yaml.width = 200 - yaml.preserve_quotes = True - yaml.sort_base_mapping_type_on_output = False # type: ignore - yaml.indent(mapping=2, sequence=4, offset=2) - - workflow_string = self._render_with_jinja(self.template_str) - workflow_dict = yaml.load(workflow_string) - - stream = io.StringIO() - yaml.dump(workflow_dict, stream) - workflow_string = stream.getvalue() - return cleandoc(workflow_string) diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index 83964a2ea..fc54b653d 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -6,7 +6,7 @@ ConfigDict, ) -from exasol.toolbox.util.workflows.template_processing import TemplateRenderer +from exasol.toolbox.util.workflows.process_template import WorkflowRenderer class Workflow(BaseModel): @@ -20,11 +20,10 @@ def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any raise FileNotFoundError(file_path) try: - raw_content = file_path.read_text() - template_renderer = TemplateRenderer( - template_str=raw_content, github_template_dict=github_template_dict + workflow_renderer = WorkflowRenderer( + github_template_dict=github_template_dict ) - workflow = template_renderer.render_to_workflow() + workflow = workflow_renderer.render(file_path=file_path) return cls(content=workflow) except Exception as e: raise ValueError(f"Error rendering file: {file_path}") from e diff --git a/test/unit/config_test.py b/test/unit/config_test.py index cba1e54b4..ee647c282 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -34,6 +34,7 @@ def test_works_as_defined(test_project_config_factory): "dist", "venv", ), + "github_workflow_patcher_yaml": None, "github_template_dict": { "dependency_manager_version": "2.3.0", "minimum_python_version": "3.10", diff --git a/test/unit/util/workflows/patch_workflow_test.py b/test/unit/util/workflows/patch_workflow_test.py new file mode 100644 index 000000000..aab1acd76 --- /dev/null +++ b/test/unit/util/workflows/patch_workflow_test.py @@ -0,0 +1,90 @@ +from dataclasses import dataclass +from inspect import cleandoc +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from exasol.toolbox.util.workflows.patch_workflow import ( + ActionType, + WorkflowPatcher, +) +from noxconfig import PROJECT_CONFIG + + +@dataclass(frozen=True) +class ExampleYaml: + remove_jobs = """ + workflows: + - name: "checks.yml" + remove_jobs: + - documentation + """ + step_customization = """ + workflows: + - name: "checks.yml" + step_customizations: + - action: {action} + job: Tests + step_id: checkout-repo + content: + name: SCM Checkout + id: checkout-repo + uses: actions/checkout@v6 + with: + fetch-depth: 0 + """ + + +@pytest.fixture +def workflow_patcher() -> WorkflowPatcher: + return WorkflowPatcher(github_template_dict=PROJECT_CONFIG.github_template_dict) + + +@pytest.fixture +def workflow_patcher_yaml(tmp_path: Path) -> Path: + return tmp_path / ".workflow-patcher.yml" + + +class TestWorkflowPatcher: + @staticmethod + def test_remove_jobs(workflow_patcher_yaml, workflow_patcher): + content = cleandoc(ExampleYaml.remove_jobs) + workflow_patcher_yaml.write_text(content) + + yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml) + + assert workflow_patcher.get_as_string(yaml_dict) == content + + @staticmethod + @pytest.mark.parametrize("action", ActionType) + def test_step_customizations(workflow_patcher_yaml, action, workflow_patcher): + content = cleandoc(ExampleYaml.step_customization.format(action=action.value)) + workflow_patcher_yaml.write_text(content) + + yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml) + + assert workflow_patcher.get_as_string(yaml_dict) == content + + +class TestStepCustomization: + @staticmethod + def test_allows_extra_field(workflow_patcher_yaml, workflow_patcher): + content = f""" + {ExampleYaml.step_customization.format(action="REPLACE")} + extra-field: "test" + """ + content = cleandoc(content) + workflow_patcher_yaml.write_text(content) + + yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml) + + assert workflow_patcher.get_as_string(yaml_dict) == content + + @staticmethod + def test_raises_error_for_unknown_action(workflow_patcher_yaml, workflow_patcher): + content = cleandoc(ExampleYaml.step_customization.format(action="UNKNOWN")) + workflow_patcher_yaml.write_text(content) + + with pytest.raises(ValidationError, match="Input should be"): + workflow_patcher.get_yaml_dict(workflow_patcher_yaml) diff --git a/test/unit/util/workflows/template_processing_test.py b/test/unit/util/workflows/render_yaml_test.py similarity index 63% rename from test/unit/util/workflows/template_processing_test.py rename to test/unit/util/workflows/render_yaml_test.py index 93b6b2dfe..035b7fd1c 100644 --- a/test/unit/util/workflows/template_processing_test.py +++ b/test/unit/util/workflows/render_yaml_test.py @@ -1,12 +1,19 @@ from inspect import cleandoc -from exasol.toolbox.util.workflows.template_processing import TemplateRenderer +import pytest + +from exasol.toolbox.util.workflows.render_yaml import YamlRenderer from noxconfig import PROJECT_CONFIG +@pytest.fixture +def yaml_renderer() -> YamlRenderer: + return YamlRenderer(github_template_dict=PROJECT_CONFIG.github_template_dict) + + class TestTemplateRenderer: @staticmethod - def test_works_for_general_case(): + def test_works_for_general_case(tmp_path, yaml_renderer): input_yaml = """ name: Build & Publish @@ -22,15 +29,15 @@ def test_works_for_general_case(): permissions: contents: write """ + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) - assert template_renderer.render_to_workflow() == cleandoc(input_yaml) + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(input_yaml) @staticmethod - def test_fixes_extra_horizontal_whitespace(): + def test_fixes_extra_horizontal_whitespace(tmp_path, yaml_renderer): # required has 2 extra spaces input_yaml = """ name: Build & Publish @@ -52,14 +59,15 @@ def test_fixes_extra_horizontal_whitespace(): required: true """ - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) - assert template_renderer.render_to_workflow() == cleandoc(expected_yaml) + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) + + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_keeps_comments(): + def test_keeps_comments(tmp_path, yaml_renderer): input_yaml = """ steps: # Comment in nested area @@ -76,15 +84,15 @@ def test_keeps_comments(): # Comment in step """ - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) - assert template_renderer.render_to_workflow() == cleandoc(expected_yaml) + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_keeps_quotes_for_variables_as_is(): + def test_keeps_quotes_for_variables_as_is(tmp_path, yaml_renderer): input_yaml = """ - name: Build Artifacts run: poetry build @@ -121,15 +129,15 @@ def test_keeps_quotes_for_variables_as_is(): dist/* """ - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) - assert template_renderer.render_to_workflow() == cleandoc(expected_yaml) + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_updates_jinja_variables(): + def test_updates_jinja_variables(tmp_path, yaml_renderer): input_yaml = """ - name: Setup Python & Poetry Environment uses: exasol/python-toolbox/.github/actions/python-environment@v5 @@ -145,15 +153,15 @@ def test_updates_jinja_variables(): poetry-version: "2.3.0" """ - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) - assert template_renderer.render_to_workflow() == cleandoc(expected_yaml) + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(expected_yaml) @staticmethod - def test_preserves_list_format(): + def test_preserves_list_format(tmp_path, yaml_renderer): input_yaml = """ on: pull_request: @@ -170,9 +178,9 @@ def test_preserves_list_format(): python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] """ - template_renderer = TemplateRenderer( - template_str=cleandoc(input_yaml), - github_template_dict=PROJECT_CONFIG.github_template_dict, - ) + file_path = tmp_path / "dummy.yml" + content = cleandoc(input_yaml) + file_path.write_text(content) - assert template_renderer.render_to_workflow() == cleandoc(input_yaml) + yaml_dict = yaml_renderer.get_yaml_dict(file_path) + assert yaml_renderer.get_as_string(yaml_dict) == cleandoc(input_yaml)