Merge pull request #37 from Codeplain-ai/release/prepare-testing-script

Tjaž Eržen · web-flow · commit 002fcf4cd2e0 · 2025-12-05T12:13:20.000+01:00
Dedicated Environment-Preparation Script for Conformance Tests
diff --git a/examples/example_hello_world_react/config.yaml b/examples/example_hello_world_react/config.yaml
@@ -1,4 +1,2 @@
-unittests-script: ../../test_scripts/run_unittests_react.sh
 conformance-tests-script: ../../test_scripts/run_conformance_tests_cypress.sh
 verbose: true # verbose flag, defaults to false if not set
-
diff --git a/plain2code_arguments.py b/plain2code_arguments.py
@@ -175,6 +175,13 @@ def create_parser():
         "1) First argument: path to a folder (e.g. 'build') containing generated source code, "
         "2) Second argument: path to a subfolder of the conformance tests folder (e.g. 'conformance_tests/subfoldername') containing test files.",
     )
+
+    parser.add_argument(
+        "--prepare-environment-script",
+        type=str,
+        help="Path to a shell script that prepares the testing environment. The script should accept the build folder path as its first argument (default: 'build').",
+    )
+
     parser.add_argument(
         "--api",
         type=str,
diff --git a/render_machine/actions/fix_conformance_test.py b/render_machine/actions/fix_conformance_test.py
@@ -115,6 +115,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any |
                         response_files,
                         style=console.OUTPUT_STYLE,
                     )
+                render_context.conformance_tests_running_context.should_prepare_testing_environment = True
                 return self.IMPLEMENTATION_CODE_UPDATED, None
             else:
                 return self.IMPLEMENTATION_CODE_NOT_UPDATED, None
diff --git a/render_machine/actions/prepare_testing_environment.py b/render_machine/actions/prepare_testing_environment.py
@@ -0,0 +1,30 @@
+from typing import Any
+
+import render_machine.render_utils as render_utils
+from plain2code_console import console
+from render_machine.actions.base_action import BaseAction
+from render_machine.render_context import RenderContext
+
+
+class PrepareTestingEnvironment(BaseAction):
+    SUCCESSFUL_OUTCOME = "testing_environment_prepared"
+    FAILED_OUTCOME = "testing_environment_preparation_failed"
+
+    def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
+        if render_context.args.verbose:
+            console.info(
+                f"[b]Running testing environment preparation script {render_context.args.prepare_environment_script} for build folder {render_context.args.build_folder}.[/b]"
+            )
+        exit_code, _ = render_utils.execute_script(
+            render_context.args.prepare_environment_script,
+            [render_context.args.build_folder],
+            render_context.args.verbose,
+            "Testing Environment Preparation",
+        )
+
+        render_context.conformance_tests_running_context.should_prepare_testing_environment = False
+
+        if exit_code == 0:
+            return self.SUCCESSFUL_OUTCOME, None
+        else:
+            return self.FAILED_OUTCOME, None
diff --git a/render_machine/actions/run_conformance_tests.py b/render_machine/actions/run_conformance_tests.py
@@ -24,11 +24,11 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
             console.info(
                 f"\n[b]Running conformance tests script {render_context.args.conformance_tests_script} for {conformance_tests_folder_name} (functional requirement {render_context.conformance_tests_running_context.current_testing_frid}).[/b]"
             )
-        exit_code, conformance_tests_issue = render_utils.execute_test_script(
+        exit_code, conformance_tests_issue = render_utils.execute_script(
             render_context.args.conformance_tests_script,
             [render_context.args.build_folder, conformance_tests_folder_name],
             render_context.args.verbose,
-            "conformance",
+            "Conformance Tests",
         )
 
         if exit_code == 0:
diff --git a/render_machine/actions/run_unit_tests.py b/render_machine/actions/run_unit_tests.py
@@ -18,11 +18,11 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
             console.info(
                 f"[b]Running unit tests script {render_context.args.unittests_script}.[/b] (attempt: {render_context.unit_tests_running_context.fix_attempts + 1})"
             )
-        exit_code, unittests_issue = render_utils.execute_test_script(
+        exit_code, unittests_issue = render_utils.execute_script(
             render_context.args.unittests_script,
             [render_context.args.build_folder],
             render_context.args.verbose,
-            "unit",
+            "Unit Tests",
         )
 
         if exit_code == 0:
diff --git a/render_machine/render_context.py b/render_machine/render_context.py
@@ -175,6 +175,13 @@ def start_refactoring_code(self):
                 )
             self.machine.dispatch(triggers.PROCEED_FRID_PROCESSING)
 
+    def start_testing_environment_preparation(self):
+        if (
+            self.args.prepare_environment_script is None
+            or not self.conformance_tests_running_context.should_prepare_testing_environment
+        ):
+            self.machine.dispatch(triggers.MARK_TESTING_ENVIRONMENT_PREPARED)
+
     def start_conformance_tests_processing(self):
         console.info("\n[b]Implementing conformance tests...[/b]\n")
         conformance_tests_json = self.conformance_tests_utils.get_conformance_tests_json()
@@ -185,6 +192,7 @@ def start_conformance_tests_processing(self):
             fix_attempts=0,
             conformance_tests_json=conformance_tests_json,
             conformance_tests_render_attempts=0,
+            should_prepare_testing_environment=True,
         )
 
     def finish_conformance_tests_processing(self):
diff --git a/render_machine/render_types.py b/render_machine/render_types.py
@@ -33,3 +33,4 @@ class ConformanceTestsRunningContext:
     # - current_testing_frid == frid  noqa: E800
     # - conformance_test_phase_index == 0 (conformance tests phase)
     current_testing_frid_high_level_implementation_plan: Optional[str] = None
+    should_prepare_testing_environment: bool = False
diff --git a/render_machine/render_utils.py b/render_machine/render_utils.py
@@ -7,7 +7,7 @@
 import plain_spec
 from plain2code_console import console
 
-TEST_SCRIPT_EXECUTION_TIMEOUT = 120
+SCRIPT_EXECUTION_TIMEOUT = 120
 TIMEOUT_ERROR_EXIT_CODE = 124
 
 
@@ -40,53 +40,53 @@ def print_inputs(render_context, existing_files_content, message):
     )
 
 
-def execute_test_script(test_script, scripts_args, verbose, test_type):
+def execute_script(script, scripts_args, verbose, script_type):
     try:
         start_time = time.time()
         result = subprocess.run(
-            [file_utils.add_current_path_if_no_path(test_script)] + scripts_args,
+            [file_utils.add_current_path_if_no_path(script)] + scripts_args,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             text=True,
-            timeout=TEST_SCRIPT_EXECUTION_TIMEOUT,
+            timeout=SCRIPT_EXECUTION_TIMEOUT,
         )
         elapsed_time = time.time() - start_time
-        # Log the info about the tests
+        # Log the info about the script execution
         if verbose:
-            with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".test_output") as temp_file:
-                temp_file.write("\n═════════════════════════ Test Script Output ═════════════════════════\n")
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".script_output") as temp_file:
+                temp_file.write(f"\n═════════════════════════ {script_type} Script Output ═════════════════════════\n")
                 temp_file.write(result.stdout)
                 temp_file.write("\n══════════════════════════════════════════════════════════════════════\n")
                 temp_file_path = temp_file.name
                 if result.returncode != 0:
-                    temp_file.write(f"Test script {test_script} failed with exit code {result.returncode}.\n")
+                    temp_file.write(f"{script_type} script {script} failed with exit code {result.returncode}.\n")
                 else:
-                    temp_file.write(f"Test script {test_script} successfully passed.\n")
-                temp_file.write(f"Test execution time: {elapsed_time:.2f} seconds.\n")
+                    temp_file.write(f"{script_type} script {script} successfully passed.\n")
+                temp_file.write(f"{script_type} script execution time: {elapsed_time:.2f} seconds.\n")
 
-            console.info(f"[b]Test output stored in: {temp_file_path}[/b]")
+            console.info(f"[b]{script_type} script output stored in: {temp_file_path}[/b]")
 
             if result.returncode != 0:
                 console.info(
-                    f"[b]The {test_type} tests have failed. Initiating the patching mode to automatically correct the discrepancies.[/b]\n"
+                    f"[b]The {script_type} script has failed. Initiating the patching mode to automatically correct the discrepancies.[/b]\n"
                 )
             else:
-                console.info(f"[b]All {test_type} tests passed successfully.[/b]\n")
+                console.info(f"[b]All {script_type} script passed successfully.[/b]\n")
 
         return result.returncode, result.stdout
     except subprocess.TimeoutExpired as e:
         # Store timeout output in a temporary file
         if verbose:
-            with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".test_timeout") as temp_file:
-                temp_file.write(f"Test script {test_script} timed out after {TEST_SCRIPT_EXECUTION_TIMEOUT} seconds.")
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".script_timeout") as temp_file:
+                temp_file.write(f"{script_type} script {script} timed out after {SCRIPT_EXECUTION_TIMEOUT} seconds.")
                 if e.stdout:
                     decoded_output = e.stdout.decode("utf-8") if isinstance(e.stdout, bytes) else e.stdout
-                    temp_file.write(f"Test script partial output before the timeout:\n{decoded_output}")
+                    temp_file.write(f"{script_type} script partial output before the timeout:\n{decoded_output}")
                 else:
-                    temp_file.write("Test script did not produce any output before the timeout.")
+                    temp_file.write(f"{script_type} script did not produce any output before the timeout.")
                 temp_file_path = temp_file.name
             console.warning(
-                f"The {test_type} test timed out after {TEST_SCRIPT_EXECUTION_TIMEOUT} seconds. Test output stored in: {temp_file_path}\n"
+                f"The {script_type} script timed out after {SCRIPT_EXECUTION_TIMEOUT} seconds. {script_type} script output stored in: {temp_file_path}\n"
             )
 
-        return TIMEOUT_ERROR_EXIT_CODE, f"Tests did not finish in {TEST_SCRIPT_EXECUTION_TIMEOUT} seconds."
+        return TIMEOUT_ERROR_EXIT_CODE, f"{script_type} script did not finish in {SCRIPT_EXECUTION_TIMEOUT} seconds."
diff --git a/render_machine/state_machine_config.py b/render_machine/state_machine_config.py
@@ -18,6 +18,7 @@
 from render_machine.actions.fix_conformance_test import FixConformanceTest
 from render_machine.actions.fix_unit_tests import FixUnitTests
 from render_machine.actions.prepare_repositories import PrepareRepositories
+from render_machine.actions.prepare_testing_environment import PrepareTestingEnvironment
 from render_machine.actions.refactor_code import RefactorCode
 from render_machine.actions.render_conformance_tests import RenderConformanceTests
 from render_machine.actions.render_functional_requirement import RenderFunctionalRequirement
@@ -88,7 +89,8 @@ def get_action_map(self) -> Dict[str, Any]:
                 git_utils.REFACTORED_CODE_COMMIT_MESSAGE
             ),
             f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTING_INITIALISED.value}": RenderConformanceTests(),
-            f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}": RunConformanceTests(),
+            f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_GENERATED.value}": PrepareTestingEnvironment(),
+            f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_ENV_PREPARED.value}": RunConformanceTests(),
             f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_FAILED.value}": FixConformanceTest(),
             f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.POSTPROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTS_READY_FOR_SUMMARY.value}": SummarizeConformanceTests(),
             f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.POSTPROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTS_READY_FOR_COMMIT.value}": CommitConformanceTestsChanges(
@@ -120,6 +122,8 @@ def get_action_result_triggers_map(self) -> Dict[str, str]:
             CommitImplementationCodeChanges.SUCCESSFUL_OUTCOME: triggers.PROCEED_FRID_PROCESSING,
             CreateDist.SUCCESSFUL_OUTCOME: triggers.FINISH_RENDER,
             RenderConformanceTests.SUCCESSFUL_OUTCOME: triggers.MARK_CONFORMANCE_TESTS_READY,
+            PrepareTestingEnvironment.SUCCESSFUL_OUTCOME: triggers.MARK_TESTING_ENVIRONMENT_PREPARED,
+            PrepareTestingEnvironment.FAILED_OUTCOME: triggers.HANDLE_ERROR,
             RunConformanceTests.SUCCESSFUL_OUTCOME: triggers.MOVE_TO_NEXT_CONFORMANCE_TEST,
             RunConformanceTests.FAILED_OUTCOME: triggers.MARK_CONFORMANCE_TESTS_FAILED,
             RunConformanceTests.UNRECOVERABLE_ERROR_OUTCOME: triggers.HANDLE_ERROR,
@@ -177,7 +181,11 @@ def get_processing_conformance_tests_states(self, render_context: RenderContext)
                     "name": States.CONFORMANCE_TESTING_INITIALISED.value,
                     "on_enter": "start_conformance_tests_for_frid",
                 },
-                States.CONFORMANCE_TEST_READY.value,
+                {
+                    "name": States.CONFORMANCE_TEST_GENERATED.value,
+                    "on_enter": "start_testing_environment_preparation",
+                },
+                States.CONFORMANCE_TEST_ENV_PREPARED.value,
                 {
                     "name": States.CONFORMANCE_TEST_FAILED.value,
                     "on_enter": "start_fixing_conformance_tests",
@@ -345,10 +353,15 @@ def get_transitions(self) -> List[Dict[str, str]]:
             {
                 "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTING_INITIALISED.value}",
                 "trigger": triggers.MARK_CONFORMANCE_TESTS_READY,
-                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}",
+                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_GENERATED.value}",
+            },
+            {
+                "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_GENERATED.value}",
+                "trigger": triggers.MARK_TESTING_ENVIRONMENT_PREPARED,
+                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_ENV_PREPARED.value}",
             },
             {
-                "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}",
+                "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_ENV_PREPARED.value}",
                 "trigger": triggers.MARK_CONFORMANCE_TESTS_FAILED,
                 "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_FAILED.value}",
             },
@@ -358,14 +371,14 @@ def get_transitions(self) -> List[Dict[str, str]]:
                 "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTING_INITIALISED.value}",
             },
             {
-                "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}",
+                "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_ENV_PREPARED.value}",
                 "trigger": triggers.MOVE_TO_NEXT_CONFORMANCE_TEST,
                 "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TESTING_INITIALISED.value}",
             },
             {
                 "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_FAILED.value}",
                 "trigger": triggers.MARK_CONFORMANCE_TESTS_READY,
-                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}",
+                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_ENV_PREPARED.value}",
             },
             {
                 "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_FAILED.value}",
@@ -375,7 +388,7 @@ def get_transitions(self) -> List[Dict[str, str]]:
             {
                 "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.PROCESSING_UNIT_TESTS.value}_{States.UNIT_TESTS_READY.value}",
                 "trigger": triggers.MARK_UNIT_TESTS_PASSED,
-                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_READY.value}",
+                "dest": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.CONFORMANCE_TEST_GENERATED.value}",
             },
             {
                 "source": f"{States.IMPLEMENTING_FRID.value}_{States.PROCESSING_CONFORMANCE_TESTS.value}_{States.PROCESSING_UNIT_TESTS.value}_{States.UNIT_TESTS_READY.value}",
diff --git a/render_machine/states.py b/render_machine/states.py
@@ -38,7 +38,8 @@ class States(Enum):
     # Conformance test processing states
     PROCESSING_CONFORMANCE_TESTS = "processingConformanceTests"
     CONFORMANCE_TESTING_INITIALISED = "conformanceTestingInitialised"
-    CONFORMANCE_TEST_READY = "conformanceTestReady"
+    CONFORMANCE_TEST_GENERATED = "conformanceTestGenerated"
+    CONFORMANCE_TEST_ENV_PREPARED = "conformanceTestEnvironmentPrepared"
     CONFORMANCE_TEST_FAILED = "conformanceTestFailed"
 
     # Postprocessing conformance tests states
diff --git a/render_machine/triggers.py b/render_machine/triggers.py
@@ -19,6 +19,7 @@
 RESTART_FRID_PROCESSING = "restart_frid_processing"
 START_NEW_REFACTORING_ITERATION = "start_new_refactoring_iteration"
 MARK_CONFORMANCE_TESTS_READY = "mark_conformance_tests_ready"
+MARK_TESTING_ENVIRONMENT_PREPARED = "mark_testing_environment_prepared"
 MARK_CONFORMANCE_TESTS_FAILED = "mark_conformance_tests_failed"
 MOVE_TO_NEXT_CONFORMANCE_TEST = "move_to_next_conformance_test"
 MARK_ALL_CONFORMANCE_TESTS_PASSED = "mark_all_conformance_tests_passed"
diff --git a/standard_template_library/golang-console-app-template.plain b/standard_template_library/golang-console-app-template.plain
@@ -2,8 +2,6 @@
 
 - :App: is a console application.
 
-- :Implementation: is the technical implementation of :App:.
-
 
 ***Non-Functional Requirements:***
 
diff --git a/test_scripts/run_unittests_react.sh b/test_scripts/run_unittests_react.sh
@@ -63,4 +63,4 @@ if [ $TEST_EXIT_CODE -ne 0 ]; then
   exit $TEST_EXIT_CODE
 fi
 
-exit $UNRECOVERABLE_ERROR_EXIT_CODE
+exit $TEST_EXIT_CODE

Original file line number	Diff line number	Diff line change
`@@ -115,6 +115,7 @@ def execute(self, render_context: RenderContext, previous_action_payload: Any \|`
`115`	`115`	`response_files,`
`116`	`116`	`style=console.OUTPUT_STYLE,`
`117`	`117`	`)`
	`118`	`+ render_context.conformance_tests_running_context.should_prepare_testing_environment = True`
`118`	`119`	`return self.IMPLEMENTATION_CODE_UPDATED, None`
`119`	`120`	`else:`
`120`	`121`	`return self.IMPLEMENTATION_CODE_NOT_UPDATED, None`
Original file line number	Diff line number	Diff line change
`@@ -24,11 +24,11 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any \|`
`24`	`24`	`console.info(`
`25`	`25`	`f"\n[b]Running conformance tests script {render_context.args.conformance_tests_script} for {conformance_tests_folder_name} (functional requirement {render_context.conformance_tests_running_context.current_testing_frid}).[/b]"`
`26`	`26`	`)`
`27`		`- exit_code, conformance_tests_issue = render_utils.execute_test_script(`
	`27`	`+ exit_code, conformance_tests_issue = render_utils.execute_script(`
`28`	`28`	`render_context.args.conformance_tests_script,`
`29`	`29`	`[render_context.args.build_folder, conformance_tests_folder_name],`
`30`	`30`	`render_context.args.verbose,`
`31`		`- "conformance",`
	`31`	`+ "Conformance Tests",`
`32`	`32`	`)`
`33`	`33`
`34`	`34`	`if exit_code == 0:`
Original file line number	Diff line number	Diff line change
`@@ -18,11 +18,11 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any \|`
`18`	`18`	`console.info(`
`19`	`19`	`f"[b]Running unit tests script {render_context.args.unittests_script}.[/b] (attempt: {render_context.unit_tests_running_context.fix_attempts + 1})"`
`20`	`20`	`)`
`21`		`- exit_code, unittests_issue = render_utils.execute_test_script(`
	`21`	`+ exit_code, unittests_issue = render_utils.execute_script(`
`22`	`22`	`render_context.args.unittests_script,`
`23`	`23`	`[render_context.args.build_folder],`
`24`	`24`	`render_context.args.verbose,`
`25`		`- "unit",`
	`25`	`+ "Unit Tests",`
`26`	`26`	`)`
`27`	`27`
`28`	`28`	`if exit_code == 0:`