From b8a25b2e3d3e4e7bed2f3b4b713e50689b13d160 Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Wed, 18 Feb 2026 17:28:07 +1000 Subject: [PATCH 1/2] feat: extend build command information in buildspec Signed-off-by: behnazh-w --- .../build_command_patcher.py | 128 +++++------ .../common_spec/base_spec.py | 36 ++- .../build_spec_generator/common_spec/core.py | 33 +-- .../common_spec/maven_spec.py | 50 +++-- .../common_spec/pypi_spec.py | 33 +-- .../dockerfile/pypi_dockerfile_output.py | 2 +- .../reproducible_central.py | 6 +- .../common_spec/test_core.py | 6 +- .../dockerfile/test_dockerfile_output.py | 6 +- .../dockerfile/test_pypi_dockerfile_output.py | 4 +- .../test_reproducible_central.py | 13 +- .../test_build_command_patcher.py | 209 +++++------------- .../expected_macaron.buildspec | 1 - .../computer-k8s/expected_default.buildspec | 25 ++- .../expected_default.buildspec | 17 +- .../expected_default.buildspec | 11 +- .../pypi_toga/expected_default.buildspec | 21 +- 17 files changed, 278 insertions(+), 323 deletions(-) delete mode 100644 tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec diff --git a/src/macaron/build_spec_generator/build_command_patcher.py b/src/macaron/build_spec_generator/build_command_patcher.py index 224ec5715..72eb76f00 100644 --- a/src/macaron/build_spec_generator/build_command_patcher.py +++ b/src/macaron/build_spec_generator/build_command_patcher.py @@ -66,80 +66,74 @@ } -def _patch_commands( - cmds_sequence: Sequence[list[str]], +def _patch_command( + cmd: list[str], cli_parsers: Sequence[CLICommandParser], patches: Mapping[ PatchCommandBuildTool, Mapping[str, PatchValueType | None], ], -) -> list[CLICommand] | None: - """Patch the sequence of build commands, using the provided CLICommandParser instances. +) -> CLICommand | None: + """Patch the build command, using the provided CLICommandParser instances. - For each command in `cmds_sequence`, it will be checked against all CLICommandParser instances until there is + The command will be checked against all CLICommandParser instances to find one that can parse it, then a patch from ``patches`` is applied for this command if provided. If a command doesn't have any corresponding ``CLICommandParser`` instance it will be parsed as UnparsedCLICommand, which just holds the original command as a list of string, without any changes. """ - result: list[CLICommand] = [] - for cmd in cmds_sequence: - # Checking if the command is a valid non-empty list. - if not cmd: - continue - effective_cli_parser = None - for cli_parser in cli_parsers: - if cli_parser.is_build_tool(cmd[0]): - effective_cli_parser = cli_parser - break - - if not effective_cli_parser: - result.append(UnparsedCLICommand(original_cmds=cmd)) - continue - - try: - cli_command = effective_cli_parser.parse(cmd) - except CommandLineParseError as error: - logger.error( - "Failed to patch the cli command %s. Error %s.", - " ".join(cmd), - error, - ) - return None - - patch = patches.get(effective_cli_parser.build_tool, None) - if not patch: - result.append(cli_command) - continue - - try: - new_cli_command = effective_cli_parser.apply_patch( - cli_command=cli_command, - patch_options=patch, - ) - except PatchBuildCommandError as error: - logger.error( - "Failed to patch the build command %s. Error %s.", - " ".join(cmd), - error, - ) - return None - - result.append(new_cli_command) - - return result - - -def patch_commands( - cmds_sequence: Sequence[list[str]], + # Checking if the command is a valid non-empty list. + if not cmd: + return None + + effective_cli_parser = None + for cli_parser in cli_parsers: + if cli_parser.is_build_tool(cmd[0]): + effective_cli_parser = cli_parser + break + + if not effective_cli_parser: + return UnparsedCLICommand(original_cmds=cmd) + + try: + cli_command = effective_cli_parser.parse(cmd) + except CommandLineParseError as error: + logger.error( + "Failed to patch the cli command %s. Error %s.", + " ".join(cmd), + error, + ) + return None + + patch = patches.get(effective_cli_parser.build_tool, None) + if not patch: + return cli_command + + try: + patched_command: CLICommand = effective_cli_parser.apply_patch( + cli_command=cli_command, + patch_options=patch, + ) + return patched_command + except PatchBuildCommandError as error: + logger.error( + "Failed to patch the build command %s. Error %s.", + " ".join(cmd), + error, + ) + return None + + +def patch_command( + cmd: list[str], patches: Mapping[ PatchCommandBuildTool, Mapping[str, PatchValueType | None], ], -) -> list[list[str]] | None: - """Patch a sequence of CLI commands. +) -> list[str] | None: + """Patch a CLI command. - For each command in this command sequence: + Possible scenarios: - If the command is not a build command, or it's a tool we do not support, it will be left intact. @@ -159,21 +153,17 @@ def patch_commands( Returns ------- - list[list[str]] | None - The patched command sequence or None if there is an error. The errors that can happen if any command - which we support is invalid in ``cmds_sequence``, or the patch value is valid. + list[str] | None + The patched command or None if there is an error. An error happens if a command, + or the patch value is valid. """ - result = [] - patch_cli_commands = _patch_commands( - cmds_sequence=cmds_sequence, + patch_cli_command = _patch_command( + cmd=cmd, cli_parsers=[MVN_CLI_PARSER, GRADLE_CLI_PARSER], patches=patches, ) - if patch_cli_commands is None: + if patch_cli_command is None: return None - for patch_cmd in patch_cli_commands: - result.append(patch_cmd.to_cmds()) - - return result + return patch_cli_command.to_cmds() diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py index 6477801fd..126b7001b 100644 --- a/src/macaron/build_spec_generator/common_spec/base_spec.py +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -9,6 +9,32 @@ from packageurl import PackageURL +class SpecBuildCommandDict(TypedDict, total=False): + """ + Initialize build command section of the build specification. + + It contains helpful information related to a build command. + """ + + #: The build tool. + build_tool: Required[str] + + #: The build tool version. + build_tool_version: NotRequired[str] + + #: The build configuration path + build_tool_path: NotRequired[str] + + #: The build command. + command: Required[list[str]] + + #: The pre-build commands. + pre_build_cmds: NotRequired[list[list[str]]] + + #: The post-build commands. + post_build_cmds: NotRequired[list[list[str]]] + + class BaseBuildSpecDict(TypedDict, total=False): """ Initialize base build specification. @@ -58,8 +84,8 @@ class BaseBuildSpecDict(TypedDict, total=False): #: List of build dependencies, which includes tests. build_dependencies: NotRequired[list[str]] - #: List of shell commands to build the project. - build_commands: NotRequired[list[list[str]]] + #: List of shell commands and related information to build the project. + build_commands: NotRequired[list[SpecBuildCommandDict]] #: List of shell commands to test the project. test_commands: NotRequired[list[list[str]]] @@ -106,7 +132,7 @@ def resolve_fields(self, purl: PackageURL) -> None: def get_default_build_commands( self, build_tool_names: list[str], - ) -> list[list[str]]: + ) -> list[SpecBuildCommandDict]: """Return the default build commands for the build tools. Parameters @@ -116,8 +142,8 @@ def get_default_build_commands( Returns ------- - list[list[str]] - The build command as a list[list[str]]. + list[SpecBuildCommandDict] + The build command and relevant information as a list[SpecBuildCommandDict]. Raises ------ diff --git a/src/macaron/build_spec_generator/common_spec/core.py b/src/macaron/build_spec_generator/common_spec/core.py index 4c2cf1ecd..b6e283dcd 100644 --- a/src/macaron/build_spec_generator/common_spec/core.py +++ b/src/macaron/build_spec_generator/common_spec/core.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the logic to generate a build spec in a generic format that can be transformed if needed.""" @@ -13,7 +13,7 @@ import sqlalchemy.orm from packageurl import PackageURL -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.common_spec.maven_spec import MavenBuildSpec from macaron.build_spec_generator.common_spec.pypi_spec import PyPIBuildSpec from macaron.build_spec_generator.macaron_db_extractor import ( @@ -75,8 +75,8 @@ def format_build_command_info(build_command_info: list[GenericBuildCommandInfo]) str The prettified output. """ - pretty_formatted_ouput = [pprint.pformat(build_command_info) for build_command_info in build_command_info] - return "\n".join(pretty_formatted_ouput) + pretty_formatted_output = [pprint.pformat(build_command_info) for build_command_info in build_command_info] + return "\n".join(pretty_formatted_output) def remove_shell_quote(cmd: list[str]) -> list[str]: @@ -351,18 +351,22 @@ def gen_generic_build_spec( if build_tools is not None: build_tool_names = [build_tool.value for build_tool in build_tools] - build_command_info = get_build_command_info( + db_build_command_info = get_build_command_info( component_id=latest_component.id, session=session, ) - logger.info( - "Attempted to find build command from the database. Result: %s", - build_command_info or "Cannot find any.", - ) - - selected_build_command = build_command_info.command if build_command_info else [] - lang_version = get_language_version(build_command_info) if build_command_info else "" + lang_version = None + spec_build_commad_info = None + if db_build_command_info: + logger.info( + "Attempted to find build command from the database. Result: %s", + db_build_command_info or "Cannot find any.", + ) + lang_version = get_language_version(db_build_command_info) if db_build_command_info else "" + spec_build_commad_info = SpecBuildCommandDict( + build_tool=db_build_command_info.build_tool_name, command=db_build_command_info.command + ) base_build_spec_dict = BaseBuildSpecDict( { @@ -378,8 +382,11 @@ def gen_generic_build_spec( "purl": str(purl), "language": target_language, "build_tools": build_tool_names, - "build_commands": [selected_build_command] if selected_build_command else [], + "build_commands": ( + [spec_build_commad_info] if spec_build_commad_info and spec_build_commad_info["command"] else [] + ), } ) + ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl) return base_build_spec_dict diff --git a/src/macaron/build_spec_generator/common_spec/maven_spec.py b/src/macaron/build_spec_generator/common_spec/maven_spec.py index de0b4c5df..2741f0923 100644 --- a/src/macaron/build_spec_generator/common_spec/maven_spec.py +++ b/src/macaron/build_spec_generator/common_spec/maven_spec.py @@ -8,8 +8,8 @@ from packageurl import PackageURL -from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict +from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_command +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.common_spec.jdk_finder import find_jdk_version_from_central_maven_repo from macaron.build_spec_generator.common_spec.jdk_version_normalizer import normalize_jdk_version @@ -33,7 +33,7 @@ def __init__(self, data: BaseBuildSpecDict): def get_default_build_commands( self, build_tool_names: list[str], - ) -> list[list[str]]: + ) -> list[SpecBuildCommandDict]: """Return the default build commands for the build tools. Parameters @@ -43,28 +43,34 @@ def get_default_build_commands( Returns ------- - list[list[str]] - The build command as a list[list[str]]. + list[SpecBuildCommandDict] + The build command as a list[SpecBuildCommandDict]. """ - default_build_commands = [] + default_build_cmd_list = [] for build_tool_name in build_tool_names: match build_tool_name: case "maven": - default_build_commands.append("mvn clean package".split()) + default_build_cmd_list.append( + SpecBuildCommandDict(build_tool=build_tool_name, command="mvn clean package".split()) + ) case "gradle": - default_build_commands.append("./gradlew clean assemble publishToMavenLocal".split()) + default_build_cmd_list.append( + SpecBuildCommandDict( + build_tool=build_tool_name, command="./gradlew clean assemble publishToMavenLocal".split() + ) + ) case _: pass - if not default_build_commands: + if not default_build_cmd_list: logger.debug( "There is no default build command available for the build tools %s.", build_tool_names, ) - return default_build_commands + return default_build_cmd_list def resolve_fields(self, purl: PackageURL) -> None: """ @@ -108,16 +114,14 @@ def resolve_fields(self, purl: PackageURL) -> None: self.data["language_version"] = [major_jdk_version] # Resolve and patch build commands. - selected_build_commands = self.data["build_commands"] or self.get_default_build_commands( - self.data["build_tools"] - ) - patched_build_commands = patch_commands( - cmds_sequence=selected_build_commands, - patches=CLI_COMMAND_PATCHES, - ) - if not patched_build_commands: - logger.debug("Failed to patch build command sequences %s", selected_build_commands) - self.data["build_commands"] = [] - return - - self.data["build_commands"] = patched_build_commands + if not self.data["build_commands"]: + self.data["build_commands"] = self.get_default_build_commands(self.data["build_tools"]) + + for build_command_info in self.data["build_commands"]: + if build_command_info["command"] and ( + patched_cmd := patch_command( + cmd=build_command_info["command"], + patches=CLI_COMMAND_PATCHES, + ) + ): + build_command_info["command"] = patched_cmd diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py index ee67578c9..0726e75f6 100644 --- a/src/macaron/build_spec_generator/common_spec/pypi_spec.py +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -14,7 +14,7 @@ from packaging.specifiers import InvalidSpecifier from packaging.utils import InvalidWheelFilename, parse_wheel_filename -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict, SpecBuildCommandDict from macaron.config.defaults import defaults from macaron.errors import SourceCodeError, WheelTagError from macaron.json_tools import json_extract @@ -43,7 +43,7 @@ def __init__(self, data: BaseBuildSpecDict): def get_default_build_commands( self, build_tool_names: list[str], - ) -> list[list[str]]: + ) -> list[SpecBuildCommandDict]: """Return the default build commands for the build tools. Parameters @@ -53,37 +53,44 @@ def get_default_build_commands( Returns ------- - list[list[str]] - The build command as a list[list[str]]. + list[SpecBuildCommandDict] + The build command as a list[SpecBuildCommandDict]. """ - default_build_commands = [] - + default_build_cmd_list = [] for build_tool_name in build_tool_names: match build_tool_name: case "pip": - default_build_commands.append("python -m build --wheel -n".split()) + default_build_cmd_list.append( + SpecBuildCommandDict(build_tool=build_tool_name, command="python -m build --wheel -n".split()) + ) case "poetry": - default_build_commands.append("poetry build".split()) + default_build_cmd_list.append( + SpecBuildCommandDict(build_tool=build_tool_name, command="poetry build".split()) + ) case "flit": # We might also want to deal with existence flit.ini, we can do so via # "python -m flit.tomlify" - default_build_commands.append("flit build".split()) + default_build_cmd_list.append( + SpecBuildCommandDict(build_tool=build_tool_name, command="flit build".split()) + ) case "hatch": - default_build_commands.append("hatch build".split()) + default_build_cmd_list.append( + SpecBuildCommandDict(build_tool=build_tool_name, command="hatch build".split()) + ) case "conda": # TODO: update this if a build command can be used for conda. pass case _: pass - if not default_build_commands: + if not default_build_cmd_list: logger.debug( "There is no default build command available for the build tools %s.", build_tool_names, ) - return default_build_commands + return default_build_cmd_list def resolve_fields(self, purl: PackageURL) -> None: """ @@ -108,7 +115,7 @@ def resolve_fields(self, purl: PackageURL) -> None: upstream_artifacts: dict[str, list[str]] = {} pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info) - patched_build_commands: list[list[str]] = [] + patched_build_commands: list[SpecBuildCommandDict] = [] build_backends_set: set[str] = set() parsed_build_requires: dict[str, str] = {} sdist_build_requires: dict[str, str] = {} diff --git a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py index 67d1c6308..24b9e406d 100644 --- a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py +++ b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py @@ -63,7 +63,7 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: f"pip install {buildspec['build_tools'][0]} && if test -f \"flit.ini\"; then python -m flit.tomlify; fi && " ) - modern_build_command = build_tool_install + " ".join(x for x in buildspec["build_commands"][0]) + modern_build_command = build_tool_install + " ".join(x for x in buildspec["build_commands"][0]["command"]) legacy_build_command = ( 'if test -f "setup.py"; then pip install wheel && python setup.py bdist_wheel; ' "else python -m build --wheel -n; fi" diff --git a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py index 5a6ec8389..ceaea7e8d 100644 --- a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py +++ b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the logic to generate a build spec in the Reproducible Central format.""" @@ -96,7 +96,9 @@ def gen_reproducible_central_build_spec(build_spec: BaseBuildSpecDict) -> str | "newline": build_spec["newline"], "buildinfo": f"target/{build_spec['artifact_id']}-{build_spec['version']}.buildinfo", "jdk": build_spec["language_version"][0], - "command": compose_shell_commands(build_spec["build_commands"]), + "command": compose_shell_commands( + [b_info["command"] for b_info in build_spec["build_commands"] if b_info["command"]] + ), } return STRING_TEMPLATE.format_map(template_format_values) diff --git a/tests/build_spec_generator/common_spec/test_core.py b/tests/build_spec_generator/common_spec/test_core.py index a0620c869..538d13695 100644 --- a/tests/build_spec_generator/common_spec/test_core.py +++ b/tests/build_spec_generator/common_spec/test_core.py @@ -6,7 +6,7 @@ import pytest from packageurl import PackageURL -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.common_spec.core import ( ECOSYSTEMS, LANGUAGES, @@ -185,7 +185,7 @@ def test_get_language_version( "purl": "pkg:maven/foo/bar@1.0.0", "language": LANGUAGES.MAVEN.value, "build_tools": ["ant"], - "build_commands": [["ant", "dist"]], + "build_commands": [SpecBuildCommandDict(build_tool="ant", command=["ant", "dist"])], } ), id="unsupported build tool for maven", @@ -225,7 +225,7 @@ def test_get_language_version( "purl": "pkg:pypi/bar@1.0.0", "language": LANGUAGES.PYPI.value, "build_tools": ["uv"], - "build_commands": [["python", "-m", "build"]], + "build_commands": [SpecBuildCommandDict(build_tool="uv", command=["python", "-m", "build"])], } ), id="unsupported build tool for pypi", diff --git a/tests/build_spec_generator/dockerfile/test_dockerfile_output.py b/tests/build_spec_generator/dockerfile/test_dockerfile_output.py index f78b566f2..c8c55be29 100644 --- a/tests/build_spec_generator/dockerfile/test_dockerfile_output.py +++ b/tests/build_spec_generator/dockerfile/test_dockerfile_output.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ @@ -7,7 +7,7 @@ import pytest -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.dockerfile import dockerfile_output from macaron.errors import GenerateBuildSpecError @@ -28,7 +28,7 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: "build_tools": ["maven"], "newline": "lf", "language_version": ["17"], - "build_commands": [["mvn", "package"]], + "build_commands": [SpecBuildCommandDict(build_tool="maven", command=["mvn", "package"])], "purl": "pkg:maven/com.oracle/example-artifact@1.2.3", } ) diff --git a/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py b/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py index 4c8902325..0c3523474 100644 --- a/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py +++ b/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py @@ -7,7 +7,7 @@ import pytest -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.dockerfile.pypi_dockerfile_output import gen_dockerfile @@ -29,7 +29,7 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: "language": "python", "has_binaries": False, "build_tools": ["pip"], - "build_commands": [["python", "-m", "build"]], + "build_commands": [SpecBuildCommandDict(build_tool="pip", command=["python", "-m", "build"])], "build_requires": {"setuptools": "==80.9.0", "wheel": ""}, "build_backends": ["setuptools.build_meta"], "upstream_artifacts": { diff --git a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py index f28b93f66..e20d429af 100644 --- a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py +++ b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py @@ -1,11 +1,11 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains tests for Reproducible Central build spec generation.""" import pytest -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict from macaron.build_spec_generator.common_spec.core import compose_shell_commands from macaron.build_spec_generator.reproducible_central.reproducible_central import gen_reproducible_central_build_spec from macaron.errors import GenerateBuildSpecError @@ -27,7 +27,7 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: "build_tools": ["maven"], "newline": "lf", "language_version": ["17"], - "build_commands": [["mvn", "package"]], + "build_commands": [SpecBuildCommandDict(build_tool="maven", command=["mvn", "package"])], "purl": "pkg:maven/com.oracle/example-artifact@1.2.3", } ) @@ -78,8 +78,11 @@ def test_build_tool_name_variants(base_build_spec: BaseBuildSpecDict, build_tool def test_compose_shell_commands_integration(base_build_spec: BaseBuildSpecDict) -> None: """Test that the correct compose_shell_commands function is used.""" - base_build_spec["build_commands"] = [["mvn", "clean", "package"], ["echo", "done"]] + base_build_spec["build_commands"] = [ + SpecBuildCommandDict(build_tool="maven", command=["mvn", "clean", "package"]), + SpecBuildCommandDict(build_tool="maven", command=["mvn", "deploy"]), + ] content = gen_reproducible_central_build_spec(base_build_spec) - expected_commands = compose_shell_commands([["mvn", "clean", "package"], ["echo", "done"]]) + expected_commands = compose_shell_commands([["mvn", "clean", "package"], ["mvn", "deploy"]]) assert content assert f'command="{expected_commands}"' in content diff --git a/tests/build_spec_generator/test_build_command_patcher.py b/tests/build_spec_generator/test_build_command_patcher.py index dad1f04ee..b1efc261b 100644 --- a/tests/build_spec_generator/test_build_command_patcher.py +++ b/tests/build_spec_generator/test_build_command_patcher.py @@ -8,10 +8,8 @@ import pytest from macaron.build_spec_generator.build_command_patcher import ( - CLICommand, - CLICommandParser, PatchValueType, - _patch_commands, + _patch_command, ) from macaron.build_spec_generator.cli_command_parser import PatchCommandBuildTool from macaron.build_spec_generator.cli_command_parser.gradle_cli_parser import ( @@ -22,7 +20,6 @@ MavenCLICommandParser, MavenOptionPatchValueType, ) -from macaron.build_spec_generator.cli_command_parser.unparsed_cli_command import UnparsedCLICommand @pytest.mark.parametrize( @@ -119,15 +116,14 @@ def test_patch_mvn_cli_command( expected: str, ) -> None: """Test the patch maven cli command on valid input.""" - patch_cmds = _patch_commands( - cmds_sequence=[original.split()], + patch_cmd = _patch_command( + cmd=original.split(), cli_parsers=[maven_cli_parser], patches={PatchCommandBuildTool.MAVEN: patch_options}, ) - assert patch_cmds - assert len(patch_cmds) == 1 + assert patch_cmd - patch_mvn_cli_command = maven_cli_parser.parse(patch_cmds.pop().to_cmds()) + patch_mvn_cli_command = maven_cli_parser.parse(patch_cmd.to_cmds()) expected_mvn_cli_command = maven_cli_parser.parse(expected.split()) assert patch_mvn_cli_command == expected_mvn_cli_command @@ -173,11 +169,11 @@ def test_patch_mvn_cli_command_error( invalid_patch: dict[str, MavenOptionPatchValueType | None], ) -> None: """Test patch mvn cli command patching with invalid patch.""" - cmd_list = "mvn -s ../.github/maven-settings.xml install -Pexamples,noRun".split() + original_cmd = "mvn -s ../.github/maven-settings.xml install -Pexamples,noRun".split() assert ( - _patch_commands( - cmds_sequence=[cmd_list], + _patch_command( + cmd=original_cmd, cli_parsers=[maven_cli_parser], patches={ PatchCommandBuildTool.MAVEN: invalid_patch, @@ -281,15 +277,14 @@ def test_patch_gradle_cli_command( expected: str, ) -> None: """Test the patch gradle cli command on valid input.""" - patch_cmds = _patch_commands( - cmds_sequence=[original.split()], + patch_cmd = _patch_command( + cmd=original.split(), cli_parsers=[gradle_cli_parser], patches={PatchCommandBuildTool.GRADLE: patch_options}, ) - assert patch_cmds - assert len(patch_cmds) == 1 + assert patch_cmd - patch_gradle_cli_command = gradle_cli_parser.parse(patch_cmds.pop().to_cmds()) + patch_gradle_cli_command = gradle_cli_parser.parse(patch_cmd.to_cmds()) expected_gradle_cli_command = gradle_cli_parser.parse(expected.split()) assert patch_gradle_cli_command == expected_gradle_cli_command @@ -353,10 +348,10 @@ def test_patch_gradle_cli_command_error( invalid_patch: dict[str, GradleOptionPatchValueType | None], ) -> None: """Test patch mvn cli command patching with invalid patch.""" - cmd_list = "gradle clean build --no-build-cache --debug --console plain -Dorg.gradle.parallel=true".split() + original_cmd = "gradle clean build --no-build-cache --debug --console plain -Dorg.gradle.parallel=true".split() assert ( - _patch_commands( - cmds_sequence=[cmd_list], + _patch_command( + cmd=original_cmd, cli_parsers=[gradle_cli_parser], patches={ PatchCommandBuildTool.GRADLE: invalid_patch, @@ -367,128 +362,51 @@ def test_patch_gradle_cli_command_error( @pytest.mark.parametrize( - ("cmds_sequence", "patches", "expected"), + ("original", "patch_options", "expected"), [ pytest.param( - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], - { - PatchCommandBuildTool.MAVEN: { - "--debug": True, - }, - PatchCommandBuildTool.GRADLE: { - "--debug": True, - }, - }, - [ - "mvn clean package --debug".split(), - "gradle clean build --debug".split(), - ], - id="apply_multiple_types_of_patches", - ), - pytest.param( - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], + "make setup", { - PatchCommandBuildTool.MAVEN: { - "--debug": True, - }, + "--threads": None, + "--no-transfer-progress": None, + "--define": None, }, - [ - "mvn clean package --debug".split(), - "gradle clean build".split(), - ], - id="apply_one_type_of_patch_to_multiple_commands", + "make setup", + id="make_command", ), pytest.param( - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], - {}, - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], - id="apply_no_patch_to_multiple_build_commands", - ), - pytest.param( - [ - "make setup".split(), - "mvn clean package".split(), - "gradle clean build".split(), - "make clean".split(), - ], + "./configure", { - PatchCommandBuildTool.MAVEN: { - "--debug": True, - }, - PatchCommandBuildTool.GRADLE: { - "--debug": True, - }, + "--threads": None, + "--no-transfer-progress": None, + "--define": None, }, - [ - "make setup".split(), - "mvn clean package --debug".split(), - "gradle clean build --debug".split(), - "make clean".split(), - ], - id="command_that_we_cannot_parse_stay_the_same", + "./configure", + id="configure_command", ), ], ) -def test_patching_multiple_commands( +def test_patch_arbitrary_command( maven_cli_parser: MavenCLICommandParser, - gradle_cli_parser: GradleCLICommandParser, - cmds_sequence: list[list[str]], - patches: Mapping[ - PatchCommandBuildTool, - Mapping[str, PatchValueType | None], - ], - expected: list[list[str]], + original: str, + patch_options: Mapping[str, MavenOptionPatchValueType | None], + expected: str, ) -> None: - """Test patching multiple commands.""" - patch_cli_commands = _patch_commands( - cmds_sequence=cmds_sequence, - cli_parsers=[maven_cli_parser, gradle_cli_parser], - patches=patches, + """Test the patch function for arbitrary commands.""" + patched_cmd = _patch_command( + cmd=original.split(), + cli_parsers=[maven_cli_parser], + patches={PatchCommandBuildTool.MAVEN: patch_options}, ) - - assert patch_cli_commands - - expected_cli_commands: list[CLICommand] = [] - cli_parsers: list[CLICommandParser] = [maven_cli_parser, gradle_cli_parser] - for cmd in expected: - effective_cli_parser = None - for cli_parser in cli_parsers: - if cli_parser.is_build_tool(cmd[0]): - effective_cli_parser = cli_parser - break - - if effective_cli_parser: - expected_cli_commands.append(effective_cli_parser.parse(cmd)) - else: - expected_cli_commands.append( - UnparsedCLICommand( - original_cmds=cmd, - ) - ) - - assert patch_cli_commands == expected_cli_commands + assert patched_cmd + assert patched_cmd.to_cmds() == expected.split() @pytest.mark.parametrize( - ("cmds_sequence", "patches"), + ("cmd", "patches"), [ pytest.param( - [ - "mvn --this-is-not-a-mvn-option".split(), - "gradle clean build".split(), - ], + "mvn --this-is-not-a-mvn-option".split(), { PatchCommandBuildTool.MAVEN: { "--debug": True, @@ -500,10 +418,7 @@ def test_patching_multiple_commands( id="incorrect_mvn_command", ), pytest.param( - [ - "mvn clean package".split(), - "gradle clean build --not-a-gradle-command".split(), - ], + "gradle clean build --not-a-gradle-command".split(), { PatchCommandBuildTool.MAVEN: { "--debug": True, @@ -515,45 +430,39 @@ def test_patching_multiple_commands( id="incorrect_gradle_command", ), pytest.param( - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], + "mvn clean package".split(), { PatchCommandBuildTool.MAVEN: { "--not-a-valid-option": True, }, }, - id="incorrrect_patch_option_long_name", + id="incorrect_patch_option_long_name", ), pytest.param( - [ - "mvn clean package".split(), - "gradle clean build".split(), - ], + "mvn clean package".split(), { PatchCommandBuildTool.MAVEN: { # --debug expects a boolean or a None value. "--debug": 10, }, }, - id="incorrrect_patch_value", + id="incorrect_patch_value", ), ], ) -def test_patching_multiple_commands_error( +def test_multiple_patches_error( maven_cli_parser: MavenCLICommandParser, gradle_cli_parser: GradleCLICommandParser, - cmds_sequence: list[list[str]], + cmd: list[str], patches: Mapping[ PatchCommandBuildTool, Mapping[str, PatchValueType | None], ], ) -> None: - """Test error cases for patching multiple commands.""" + """Test error cases for multiple patches and parsers.""" assert ( - _patch_commands( - cmds_sequence=cmds_sequence, + _patch_command( + cmd=cmd, cli_parsers=[maven_cli_parser, gradle_cli_parser], patches=patches, ) @@ -562,23 +471,19 @@ def test_patching_multiple_commands_error( @pytest.mark.parametrize( - ("original_cmd_sequence"), + ("original_cmd"), [ pytest.param( [], - id="empty sequence", - ), - pytest.param( - [[]], id="empty command", ), ], ) -def test_empty_command(maven_cli_parser: MavenCLICommandParser, original_cmd_sequence: list[list[str]]) -> None: - """Test the patch command for empty commands.""" - patch_cmds = _patch_commands( - cmds_sequence=original_cmd_sequence, +def test_empty_command(maven_cli_parser: MavenCLICommandParser, original_cmd: list[str]) -> None: + """Test the patch command for an empty command.""" + patch_cmd = _patch_command( + cmd=original_cmd, cli_parsers=[maven_cli_parser], patches={PatchCommandBuildTool.MAVEN: {}}, ) - assert patch_cmds == [] + assert patch_cmd is None diff --git a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec deleted file mode 100644 index 2fe2e28c8..000000000 --- a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec +++ /dev/null @@ -1 +0,0 @@ -{"macaron_version": "0.17.0", "group_id": "io.github.behnazh-w.demo", "artifact_id": "core", "version": "2.0.3", "git_repo": "https://github.com/behnazh-w/example-maven-provenance", "git_tag": "597be192fb50f03b86c34f1bfc494fea1eab264f", "newline": "lf", "language_version": "17", "ecosystem": "maven", "purl": "pkg:maven/io.github.behnazh-w.demo/core@2.0.3", "language": "java", "build_tool": "maven", "build_commands": [["./mvnw", "-DskipTests=true", "-Dmaven.test.skip=true", "-Dmaven.site.skip=true", "-Drat.skip=true", "-Dmaven.javadoc.skip=true", "clean", "package"]]} diff --git a/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec index 86325ad7f..93c0898b8 100644 --- a/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec +++ b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec @@ -1,5 +1,5 @@ { - "macaron_version": "0.18.0", + "macaron_version": "0.20.0", "group_id": "org.apache.hugegraph", "artifact_id": "computer-k8s", "version": "1.0.0", @@ -16,15 +16,18 @@ "maven" ], "build_commands": [ - [ - "mvn", - "-DskipTests=true", - "-Dmaven.test.skip=true", - "-Dmaven.site.skip=true", - "-Drat.skip=true", - "-Dmaven.javadoc.skip=true", - "clean", - "package" - ] + { + "build_tool": "maven", + "command": [ + "mvn", + "-DskipTests=true", + "-Dmaven.test.skip=true", + "-Dmaven.site.skip=true", + "-Drat.skip=true", + "-Dmaven.javadoc.skip=true", + "clean", + "package" + ] + } ] } diff --git a/tests/integration/cases/pypi_cachetools/expected_default.buildspec b/tests/integration/cases/pypi_cachetools/expected_default.buildspec index 87859fbd4..53ae6d8f4 100644 --- a/tests/integration/cases/pypi_cachetools/expected_default.buildspec +++ b/tests/integration/cases/pypi_cachetools/expected_default.buildspec @@ -16,13 +16,16 @@ "pip" ], "build_commands": [ - [ - "python", - "-m", - "build", - "--wheel", - "-n" - ] + { + "build_tool": "pip", + "command": [ + "python", + "-m", + "build", + "--wheel", + "-n" + ] + } ], "has_binaries": false, "build_requires": { diff --git a/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec b/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec index de0634640..79071d6c3 100644 --- a/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec +++ b/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec @@ -17,10 +17,13 @@ "flit" ], "build_commands": [ - [ - "flit", - "build" - ] + { + "build_tool": "flit", + "command": [ + "flit", + "build" + ] + } ], "has_binaries": false, "build_requires": { diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec index ac873e87f..29e2f4cf8 100644 --- a/tests/integration/cases/pypi_toga/expected_default.buildspec +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -17,19 +17,22 @@ "pip" ], "build_commands": [ - [ - "python", - "-m", - "build", - "--wheel", - "-n" - ] + { + "build_tool": "pip", + "command": [ + "python", + "-m", + "build", + "--wheel", + "-n" + ] + } ], "has_binaries": false, "build_requires": { "setuptools": "==80.3.1", - "setuptools_dynamic_dependencies": "==1.0.0", - "setuptools_scm": "==8.3.1" + "setuptools_scm": "==8.3.1", + "setuptools_dynamic_dependencies": "==1.0.0" }, "build_backends": [ "setuptools.build_meta" From 6c57565162f325b0ebd1a65eaf253838d0358e8b Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Fri, 20 Feb 2026 17:56:06 +1000 Subject: [PATCH 2/2] wip --- .../common_spec/base_spec.py | 7 ------- src/macaron/slsa_analyzer/analyzer.py | 5 ++++- .../build_tool/base_build_tool.py | 20 +++++++++++++++++++ src/macaron/slsa_analyzer/build_tool/pip.py | 12 +++++++---- .../slsa_analyzer/checks/build_tool_check.py | 20 ++++++++++++++----- .../slsa_analyzer/checks/check_result.py | 17 +++++++++++++++- 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py index 126b7001b..9a63a6fbb 100644 --- a/src/macaron/build_spec_generator/common_spec/base_spec.py +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -28,13 +28,6 @@ class SpecBuildCommandDict(TypedDict, total=False): #: The build command. command: Required[list[str]] - #: The pre-build commands. - pre_build_cmds: NotRequired[list[list[str]]] - - #: The post-build commands. - post_build_cmds: NotRequired[list[list[str]]] - - class BaseBuildSpecDict(TypedDict, total=False): """ Initialize base build specification. diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index a76e45e1b..cd603c637 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -1050,14 +1050,17 @@ def _determine_build_tools(self, analyze_ctx: AnalyzeContext, git_service: BaseG continue if build_tool.match_purl_type(analyze_ctx.component.type): + if build_tool.name != "pip": + continue logger.info( "Checking if the repo %s uses build tool %s", analyze_ctx.component.repository.complete_name, build_tool.name, ) - if build_tool.is_detected(analyze_ctx.component.repository.fs_path): + if (build_tool_configs := build_tool.is_detected(analyze_ctx.component.repository.fs_path)): logger.info("The repo uses %s build tool.", build_tool.name) + build_tool.set_build_tool_configurations(build_tool_configs) analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) if not analyze_ctx.dynamic_data["build_spec"]["tools"]: diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index d6f7f9d99..561f5e8ca 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -202,6 +202,7 @@ def __init__(self, name: str, language: BuildLanguage, purl_type: str) -> None: self.wrapper_files: list[str] = [] self.runtime_options = RuntimeOptions() self.path_filters: list[str] = [] + self.build_tool_configs: list[tuple[str, float, str | None]] = [] def __str__(self) -> str: return self.name @@ -261,6 +262,25 @@ def get_dep_analyzer(self) -> DependencyAnalyzer: """ return NoneDependencyAnalyzer() + def set_build_tool_configurations(self, build_tool_configs: list[tuple[str, float, str | None]]) -> None: + """Set the build tool configurations for the instance. + + Parameters + ---------- + build_tool_configs : list of tuple of (str, float, str or None) + A list containing configuration tuples for each build tool. + Each tuple consists of: + - str: The path to the build tool configuration file. + - float: The confidence score between 0 and 1 for identifying the correct build tool configuration. + - str or None: An optional build tool version. + + Returns + ------- + None + """ + self.build_tool_configs = build_tool_configs + + def get_build_dirs(self, repo_path: str) -> Iterable[Path]: """Find directories in the repository that have their own build scripts. diff --git a/src/macaron/slsa_analyzer/build_tool/pip.py b/src/macaron/slsa_analyzer/build_tool/pip.py index 2ee2752c7..c6225ea1c 100644 --- a/src/macaron/slsa_analyzer/build_tool/pip.py +++ b/src/macaron/slsa_analyzer/build_tool/pip.py @@ -56,19 +56,23 @@ def is_detected(self, repo_path: str) -> bool: bool True if this build tool is detected, else False. """ + results: list[tuple[str, float, str | None]] = [] # (config_path, confidence_score, build_tool_version) + confidence_score = 1.0 for config_name in self.build_configs: if config_path := file_exists(repo_path, config_name, filters=self.path_filters): if os.path.basename(config_path) == "pyproject.toml": # Check the build-system section. If it doesn't exist, by default setuptools should be used. if pyproject.get_build_system(config_path) is None: - return True + results.append((str(config_path.relative_to(repo_path)), confidence_score, None)) for tool in self.build_requires + self.build_backend: if pyproject.build_system_contains_tool(tool, config_path): - return True + results.append((str(config_path.relative_to(repo_path)), confidence_score, None)) + break else: # TODO: For other build configuration files, like setup.py, we need to improve the logic. - return True - return False + results.append((str(config_path.relative_to(repo_path)), confidence_score, None)) + confidence_score = confidence_score / 2 * 100 + return results def get_dep_analyzer(self) -> DependencyAnalyzer: """Create a DependencyAnalyzer for the build tool. diff --git a/src/macaron/slsa_analyzer/checks/build_tool_check.py b/src/macaron/slsa_analyzer/checks/build_tool_check.py index 8432b014e..e0b301873 100644 --- a/src/macaron/slsa_analyzer/checks/build_tool_check.py +++ b/src/macaron/slsa_analyzer/checks/build_tool_check.py @@ -27,11 +27,20 @@ class BuildToolFacts(CheckFacts): #: The primary key. id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + #: The language of the artifact built by build tool. + language: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) + #: The build tool name. build_tool_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) - #: The language of the artifact built by build tool. - language: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) + #: The build tool version. + build_tool_version: Mapped[str | None] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) + + #: The build tool configuration path. + build_tool_path: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) + + #: The build tool configuration path link. + build_tool_path_link: Mapped[str | None] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) __mapper_args__ = { "polymorphic_identity": "_build_tool_check", @@ -72,9 +81,10 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: result_tables: list[CheckFacts] = [] for tool in build_tools: - result_tables.append( - BuildToolFacts(build_tool_name=tool.name, language=tool.language.value, confidence=Confidence.HIGH) - ) + for build_tool_path, score, _ in tool.build_tool_configs: + result_tables.append( + BuildToolFacts(build_tool_name=tool.name, build_tool_path = build_tool_path, language=tool.language.value, confidence=Confidence.get_confidence_level(score)) + ) return CheckResultData( result_tables=result_tables, diff --git a/src/macaron/slsa_analyzer/checks/check_result.py b/src/macaron/slsa_analyzer/checks/check_result.py index f9d5c1ad0..7dba4af7e 100644 --- a/src/macaron/slsa_analyzer/checks/check_result.py +++ b/src/macaron/slsa_analyzer/checks/check_result.py @@ -137,7 +137,22 @@ def normalize(cls, evidence_weight_map: EvidenceWeightMap) -> "Confidence": normalized_score = score / max_score - # Return the confidence level that is closest to the normalized score. + return cls.get_confidence_level(normalized_score) + + @classmethod + def get_confidence_level(cls, normalized_score) -> "Confidence": + """Return the Confidence level closest to a given normalized score. + + Parameters + ---------- + normalized_score : float + A score normalized to the range expected by the Confidence values. + + Returns + ------- + Confidence + The Confidence enum member whose value is closest to the given normalized score. + """ return min(cls, key=lambda c: abs(c.value - normalized_score))