From 3e84ec15af1126fe9c73f93b68e55df21c0a93bd Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:25:19 +0000 Subject: [PATCH 01/10] the script now runs a KICS scan command --- e2e-report.html | 425 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 425 insertions(+) create mode 100644 e2e-report.html diff --git a/e2e-report.html b/e2e-report.html new file mode 100644 index 00000000000..bb488914a39 --- /dev/null +++ b/e2e-report.html @@ -0,0 +1,425 @@ +KICS Scan Result
KICS E2E REPORT03/23/2026 13:11KICS.IO

E2E Tests Result: fail

Time Elapsed: 1273.29s

Kics Docker Image:

Filters

PASSED +161
FAILED +1
ALL +162
Test: _should_exclude_provided_paths_[E2E-CLI-027]_0 +Elapsed: 225.26s +Status: pass
Test: _should_generate_and_save_a_log_file_with_log_level_[E2E-CLI-039]_0 +Elapsed: 122.69s +Status: pass
Test: _should_perform_a_scan_and_create_a_result_and_payload_file_[E2E-CLI-033]_0 +Elapsed: 322.69s +Status: pass
Test: scan_should_generate_equivalent_payloads_for_OpenAPI_YAML_and_JSON_files_with_circular_references_[E2E-CLI-099]_0 +Elapsed: 93.53s +Status: pass
Test: scan_should_generate_equivalent_payloads_for_OpenAPI_YAML_and_JSON_files_with_circular_references_[E2E-CLI-099]_1 +Elapsed: 93.25s +Status: pass
Test: should_check_if_output_path_is_invalid_[E2E-CLI-082]_0 +Elapsed: 1.51s +Status: pass
Test: should_check_if_output_path_is_valid_[E2E-CLI-081]_0 +Elapsed: 25.33s +Status: pass
Test: should_create_a_payload_file_[E2E-CLI-005]_0 +Elapsed: 105.12s +Status: pass
Test: should_disable_colored_output_in_the_CLI_[E2E-CLI-015]_0 +Elapsed: 16.82s +Status: pass
Test: should_display_CPU_usage_in_the_CLI_[E2E-CLI-022]_0 +Elapsed: 14.57s +Status: pass
Test: should_display_an_error_message_about_unknown_argument_[E2E-CLI-010]_0 +Elapsed: 1.5s +Status: pass
Test: should_display_an_error_of_invalid_flag_combination_[E2E-CLI-004]_0 +Elapsed: 0.79s +Status: pass
Test: should_display_an_error_of_invalid_flag_combination_[E2E-CLI-004]_1 +Elapsed: 0.72s +Status: pass
Test: should_display_an_error_regarding_missing_--results_flag_[E2E-CLI-060]_0 +Elapsed: 2.22s +Status: pass
Test: should_display_an_error_regarding_missing_-p_flag_[E2E-CLI-003]_0 +Elapsed: 1.12s +Status: pass
Test: should_display_line_references_in_the_payload_file_[E2E-CLI-047]_0 +Elapsed: 321.71s +Status: pass
Test: should_display_memory_usage_in_the_CLI_[E2E-CLI-023]_0 +Elapsed: 19.25s +Status: pass
Test: should_display_minimal-ui_[E2E-CLI-012]_0 +Elapsed: 14.23s +Status: pass
Test: should_display_the_kics_help_text_[E2E-CLI-001]_0 +Elapsed: 0.2s +Status: pass
Test: should_display_the_kics_remediate_help_text_[E2E-CLI-059]_0 +Elapsed: 1.6s +Status: pass
Test: should_display_the_kics_scan_help_text_[E2E-CLI-002]_0 +Elapsed: 0.63s +Status: pass
Test: should_display_the_kics_version_[E2E-CLI-024]_0 +Elapsed: 1.57s +Status: pass
Test: should_display_verbose_information_in_the_CLI_[E2E-CLI-017]_0 +Elapsed: 14.54s +Status: pass
Test: should_download_and_scan_the_provided_git_path_[E2E-CLI-041]_0 +Elapsed: 94.22s +Status: pass
Test: should_download_and_scan_the_provided_http_path/file_[E2E-CLI-042]_0 +Elapsed: 10.87s +Status: pass
Test: should_exclude_provided_categories_[E2E-CLI-018]_0 +Elapsed: 121.7s +Status: pass
Test: should_exclude_provided_queries_[E2E-CLI-020]_0 +Elapsed: 14.01s +Status: pass
Test: should_exclude_provided_similarity_ID_results_[E2E-CLI-035]_0 +Elapsed: 18.02s +Status: pass
Test: should_exclude_provided_similarity_ID_results_[E2E-CLI-035]_1 +Elapsed: 1.26s +Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_0 +Elapsed: 94.56s +Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_1 +Elapsed: 15.88s +Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_2 +Elapsed: 1.65s +Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_3 +Elapsed: 32.7s +Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_0 +Elapsed: 1.11s +Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_1 +Elapsed: 0.89s +Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_2 +Elapsed: 122.92s +Status: pass
Test: should_export_the_result_files_to_provided_path_[E2E-CLI-030]_0 +Elapsed: 19.88s +Status: pass
Test: should_export_the_results_based_on_different_formats_[E2E-CLI-031]_0 +Elapsed: 145.1s +Status: pass
Test: should_export_the_results_based_on_different_formats_[E2E-CLI-031]_1 +Elapsed: 9.16s +Status: pass
Test: should_export_the_results_based_on_report_formats_[E2E-CLI-040]_0 +Elapsed: 125s +Status: pass
Test: should_export_the_results_based_on_the_formats_provided_by_this_flag,_with_critical_severity_[E2E-CLI-090]_0 +Elapsed: 8.52s +Status: pass
Test: should_fail-on_provided_values_[E2E-CLI-025]_0 +Elapsed: 17.37s +Status: pass
Test: should_fail-on_provided_values_[E2E-CLI-025]_1 +Elapsed: 21.08s +Status: pass
Test: should_fetch_descriptions_from_environment_[E2E-CLI-046]_0 +Elapsed: 18.86s +Status: pass
Test: should_generate_a_valid_ID_[E2E-CLI-006]_0 +Elapsed: 0.91s +Status: pass
Test: should_generate_and_save_a_log_file_[E2E-CLI-038]_0 +Elapsed: 117.71s +Status: pass
Test: should_hide_all_output_text_in_CLI_[E2E-CLI-008]_0 +Elapsed: 75.09s +Status: pass
Test: should_hide_the_progress_bar_in_the_CLI_[E2E-CLI-009]_0 +Elapsed: 14.38s +Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_0 +Elapsed: 308.11s +Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_1 +Elapsed: 315.66s +Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_2 +Elapsed: 20.17s +Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_3 +Elapsed: 61.92s +Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_4 +Elapsed: 375.26s +Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_0 +Elapsed: 0.55s +Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_1 +Elapsed: 1.88s +Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_2 +Elapsed: 113.78s +Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_3 +Elapsed: 130.93s +Status: pass
Test: should_include_bill_of_materials_in_results_output_[E2E-CLI-050]_0 +Elapsed: 325.92s +Status: pass
Test: should_include_bill_of_materials_in_results_output_[E2E-CLI-050]_1 +Elapsed: 1.34s +Status: pass
Test: should_list_all_supported_platforms_[E2E-CLI-013]_0 +Elapsed: 0.8s +Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_0 +Elapsed: 134.19s +Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_1 +Elapsed: 4.52s +Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_2 +Elapsed: 1.02s +Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_0 +Elapsed: 5.22s +Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_1 +Elapsed: 5.88s +Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_2 +Elapsed: 5.22s +Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_3 +Elapsed: 2.14s +Status: pass
Test: should_load_and_execute_queries_from_multiple_paths_[E2E-CLI-054]_0 +Elapsed: 8.16s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_0 +Elapsed: 324.73s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_1 +Elapsed: 321.29s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_2 +Elapsed: 325.99s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_3 +Elapsed: 7.44s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_4 +Elapsed: 10.21s +Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_5 +Elapsed: 7.17s +Status: pass
Test: should_load_descriptions_from_a_custom_server_[E2E-CLI-052]_0 +Elapsed: 12.76s +Status: pass
Test: should_load_descriptions_from_a_custom_server_[E2E-CLI-052]_1 +Elapsed: 310.5s +Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_0 +Elapsed: 10.55s +Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_1 +Elapsed: 125.89s +Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_2 +Elapsed: 2.77s +Status: pass
Test: should_modify_log_format_messages_in_the_CLI_[E2E-CLI-028]_0 +Elapsed: 23.86s +Status: pass
Test: should_modify_the_default_preview-lines_value_[E2E-CLI-014]_0 +Elapsed: 13.89s +Status: pass
Test: should_not_display_messages_in_the_CLI_[E2E-CLI-034]_0 +Elapsed: 18.63s +Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_0 +Elapsed: 10.21s +Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_1 +Elapsed: 19.9s +Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_2 +Elapsed: 16.3s +Status: pass
Test: should_perform_a_scan,_present_two_results,_without_the_same_similarity_id_[E2E-CLI-077]_0 +Elapsed: 9.38s +Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-086]_0 +Elapsed: 333.14s +Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-087]_0 +Elapsed: 333.77s +Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-088]_0 +Elapsed: 348.53s +Status: pass
Test: should_perform_a_scan_and_not_ignore_the_entire_project_[E2E-CLI-092]_0 +Elapsed: 131.87s +Status: pass
Test: should_perform_a_scan_and_return_three_different_similarity_ids_on_the_results_[E2E-CLI-078]_0 +Elapsed: 8.19s +Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_block_[E2E-CLI-084]_0 +Elapsed: 13.02s +Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_file_[E2E-CLI-083]_0 +Elapsed: 9.49s +Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_query_[E2E-CLI-085]_0 +Elapsed: 11.74s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_0 +Elapsed: 17.26s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_1 +Elapsed: 12.92s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_2 +Elapsed: 12.63s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_3 +Elapsed: 1.32s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_4 +Elapsed: 11.92s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_5 +Elapsed: 12.05s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_6 +Elapsed: 9.43s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_7 +Elapsed: 0.9s +Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_8 +Elapsed: 0.87s +Status: pass
Test: should_perform_a_scan_on_bicep_file_with_existing_resources_and_return_exit_code_0_[E2E-CLI-103]_0 +Elapsed: 21.66s +Status: pass
Test: should_perform_a_scan_on_bicep_files_and_create_a_result_and_payload_file_[E2E-CLI-091]_0 +Elapsed: 25.89s +Status: pass
Test: should_perform_a_scan_on_zip_file_with_UTF-16_files_and_return_exit_code_40_[E2E-CLI-102]_0 +Elapsed: 338.59s +Status: pass
Test: should_perform_a_scan_saving_the_reports_in_sarif_format,_showing_no_cwe_field_on_results_[E2E-CLI-080]_0 +Elapsed: 2.32s +Status: pass
Test: should_perform_a_scan_saving_the_reports_in_sarif_format,_showing_the_cwe_on_results_[E2E-CLI-079]_0 +Elapsed: 3.55s +Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_critical_severity_and_return_exit_code_60_[E2E-CLI-089]_0 +Elapsed: 2.71s +Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_similarity_ids_unique,_showing_the_old_similarity_id_[E2E-CLI-096]_0 +Elapsed: 3.55s +Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_similarity_ids_unique_without_showing_the_old_similarity_id_[E2E-CLI-097]_0 +Elapsed: 7.53s +Status: pass
Test: should_perform_a_scan_without_detecting_anything_since_no_files_are_scanned_because_of_max_file_size_[E2E-CLI-076]_0 +Elapsed: 2.2s +Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_1]_0 +Elapsed: 0.97s +Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_2]_0 +Elapsed: 1.79s +Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_3]_0 +Elapsed: 2.19s +Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_4]_0 +Elapsed: 1.58s +Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_5]_0 +Elapsed: 0.88s +Status: pass
Test: should_perform_a_simple_scan_[E2E-CLI-007]_0 +Elapsed: 71.47s +Status: pass
Test: should_perform_a_valid_analyze_[E2E-CLI-066]_0 +Elapsed: 2.28s +Status: pass
Test: should_perform_a_valid_scan_[E2E-CLI-065]_0 +Elapsed: 6.3s +Status: pass
Test: should_perform_a_valid_scan_and_and_detect_ansible_[E2E-CLI-075]_0 +Elapsed: 10.12s +Status: pass
Test: should_perform_a_valid_scan_and_get_the_variables_using_a_variables_path_as_a_comment_[E2E-CLI-062]_0 +Elapsed: 345.95s +Status: pass
Test: should_perform_a_valid_scan_and_get_the_variables_using_a_variables_path_as_a_flag_[E2E-CLI-063]_0 +Elapsed: 342.68s +Status: pass
Test: should_perform_a_valid_scan_and_ignore_the_experimental_queries_[E2E-CLI-069]_0 +Elapsed: 11.41s +Status: pass
Test: should_perform_a_valid_scan_and_not_ignore_the_experimental_queries_[E2E-CLI-070]_0 +Elapsed: 5.93s +Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-071]_0 +Elapsed: 8.19s +Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-073]_0 +Elapsed: 7.25s +Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-094]_0 +Elapsed: 3.13s +Status: pass
Test: should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0 +Elapsed: 23.58s +Status: fail +
View Details
Test: Test_E2E_CLI/should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0
--- FAIL: Test_E2E_CLI/should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0 (23.58s)
View extra elements
Expected in fixtures: +{ +"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", +"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 5, +"queryName": "Healthcheck Instruction Missing", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" +} +}
Actual output +{ +"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", +"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 1, +"queryName": "Healthcheck Instruction Missing", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" +} +}
Show full Actual and Expected
Expected in fixtures: +"actual_value": "The 'Dockerfile' does not contain any 'USER' instruction", +"expected_value": "The 'Dockerfile' should contain the 'USER' instruction", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 1, +"queryName": "Missing User Instruction", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "821b0a105dd9ebb6b14489251615d94813a578feb68624113a52ba9c1668ecb7" +} +{ +"actual_value": "FROM alpine:latest'", +"expected_value": "FROM alpine:latest:'version' where version should not be 'latest'", +"file_name": "", +"issue_type": "IncorrectValue", +"line": 1, +"queryName": "Image Version Using 'latest'", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "a32938ae4d056569e5cfda73594f6cb276d4f9ee14a848fb6b642b14defee892" +} +{ +"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", +"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 5, +"queryName": "Healthcheck Instruction Missing", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" +}
Actual output: +"actual_value": "The 'Dockerfile' does not contain any 'USER' instruction", +"expected_value": "The 'Dockerfile' should contain the 'USER' instruction", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 1, +"queryName": "Missing User Instruction", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "821b0a105dd9ebb6b14489251615d94813a578feb68624113a52ba9c1668ecb7" +} +{ +"actual_value": "FROM alpine:latest'", +"expected_value": "FROM alpine:latest:'version' where version should not be 'latest'", +"file_name": "", +"issue_type": "IncorrectValue", +"line": 1, +"queryName": "Image Version Using 'latest'", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "a32938ae4d056569e5cfda73594f6cb276d4f9ee14a848fb6b642b14defee892" +} +{ +"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", +"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", +"file_name": "", +"issue_type": "MissingAttribute", +"line": 1, +"queryName": "Healthcheck Instruction Missing", +"search_key": "FROM={{alpine:latest}}", +"search_line": -1, +"search_value": "", +"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" +}
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-072]_0 +Elapsed: 8.68s +Status: pass
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-074]_0 +Elapsed: 19.11s +Status: pass
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-095]_0 +Elapsed: 3.67s +Status: pass
Test: should_perform_a_valid_scan_and_return_one_HIGH_result_[E2E-CLI-098]_0 +Elapsed: 3.99s +Status: pass
Test: should_perform_a_valid_scan_but_ignore_broken_symlinks_and_symlinks_that_create_endless_loops_[E2E-CLI-067]_0 +Elapsed: 4.7s +Status: pass
Test: should_perform_a_valid_scan_while_evaluating_the_terraform_functions_with_unknown_type_[E2E-CLI-071]_0 +Elapsed: 356.01s +Status: pass
Test: should_perform_a_valid_scan_with_--exclude-type_flag_[E2E-CLI-061]_0 +Elapsed: 27.23s +Status: pass
Test: should_perform_a_valid_scan_with_-t_flag_[E2E-CLI-011]_0 +Elapsed: 16.38s +Status: pass
Test: should_perform_a_valid_scan_with_json/yaml_file_with_##_in_non_ref_[E2E-CLI-064]_0 +Elapsed: 160.31s +Status: pass
Test: should_remediate_all_remediation_found_[E2E-CLI-057]_0 +Elapsed: 41.88s +Status: pass
Test: should_remediate_the_recommendations_pointed_in_include-ids_flag_[E2E-CLI-058]_0 +Elapsed: 18.38s +Status: pass
Test: should_remove_empty_files_from_helm_install_render_[E2E-CLI-104]_0 +Elapsed: 6.11s +Status: pass
Test: should_resolve_openapi_files_and_return_results_in_different_files_[E2E-CLI-055]_0 +Elapsed: 104.85s +Status: pass
Test: should_run_a_scan_in_multiple_paths_[E2E-CLI-019]_0 +Elapsed: 116.9s +Status: pass
Test: should_run_only_provided_queries_and_exclude_provided_results_[E2E-CLI-037]_0 +Elapsed: 8.05s +Status: pass
Test: should_run_only_provided_queries_and_exclude_provided_results_[E2E-CLI-037]_1 +Elapsed: 13.3s +Status: pass
Test: should_scan_a_folder_with_FHIR_files_and_skip_them_successfully_[E2E-CLI-100]_0 +Elapsed: 4.66s +Status: pass
Test: should_scan_a_folder_with_azure-pipelines-vscode_files_and_skip_them_successfully_[E2E-CLI-100]_0 +Elapsed: 10.82s +Status: pass
Test: should_scan_a_folder_with_blacklisted_files,_with_flag_--exclude-type,_and_skip_them_successfully_[E2E-CLI-100]_0 +Elapsed: 15.9s +Status: pass
Test: should_scan_a_folder_with_blacklisted_files,_with_flag_--type,_and_skip_them_successfully_[E2E-CLI-100]_0 +Elapsed: 2.47s +Status: pass
Test: should_set_the_results_output_name_[E2E-CLI-032]_0 +Elapsed: 144.37s +Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_0 +Elapsed: 0.57s +Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_1 +Elapsed: 1.1s +Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_2 +Elapsed: 0.9s +Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_3 +Elapsed: 0.88s +Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_0 +Elapsed: 17.5s +Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_1 +Elapsed: 21.53s +Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_2 +Elapsed: 3.34s +Status: pass
Test: should_validate_the_kics_result_status_code_[E2E-CLI-021]_0 +Elapsed: 118.32s +Status: pass
Test: should_validate_the_kics_result_status_code_[E2E-CLI-021]_1 +Elapsed: 7s +Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_exclude_from_scan_[E2E-CLI-101]_0 +Elapsed: 1.49s +Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_scan,_long_version_[E2E-CLI-101]_0 +Elapsed: 1.4s +Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_scan,_short_version_[E2E-CLI-101]_0 +Elapsed: 1.44s +Status: pass
\ No newline at end of file From d0a31d50cab86247cb34273b6f7538c709be7d80 Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 24 Mar 2026 15:47:52 +0000 Subject: [PATCH 02/10] updated script to fill positive_expected_result file --- .../generate-positive-expected-result.py | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 .github/scripts/generate-positive-expected-results/generate-positive-expected-result.py diff --git a/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py b/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py new file mode 100644 index 00000000000..4d87b61dacd --- /dev/null +++ b/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py @@ -0,0 +1,226 @@ +import argparse +import json +import os +import subprocess +import sys +import time + +FIELD_ORDER = [ + "queryName", "severity", "line", "fileName", + "resourceType", "resourceName", "searchKey", "searchValue", + "expectedValue", "actualValue", "issueType", "similarityID", "search_line", +] + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) +QUERIES_DIR = os.path.join(REPO_ROOT, "assets", "queries") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run a KICS scan for a given query.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--run-all", action="store_true", help="Run scans for all queries under assets/queries.") + group.add_argument("--queryID", help="The query ID to scan.") + parser.add_argument("--queryPath", help="The base path of the query (required without --run-all).") + return parser.parse_args() + + +def build_command(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> list[str]: + main_go = os.path.join(REPO_ROOT, "cmd", "console", "main.go") + + return [ + "go", "run", main_go, + "scan", + "-p", scan_path, + "-o", output_path, + "--output-name", output_name, + "-i", query_id, + "-d", payload_path, + "-v", + "--experimental-queries", + "--bom", + "--enable-openapi-refs", + "--ignore-on-exit", "results" + ] + + +def run_scan(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> int: + command = build_command(query_id, scan_path, payload_path, output_path, output_name) + + print("Running command:") + print(" ".join(command)) + print("-" * 60) + + try: + result = subprocess.run(command, cwd=REPO_ROOT, check=True) + return result.returncode + except subprocess.CalledProcessError as e: + print(f"\n[ERROR] Scan failed with return code {e.returncode}.", file=sys.stderr) + return e.returncode + except FileNotFoundError: + print("\n[ERROR] 'go' not found. Make sure Go is installed and in your PATH.", file=sys.stderr) + return 1 + + +def find_positive_tests(query_path: str) -> list[tuple[str, str]]: + """ + Return a sorted list of (label, scan_path) for each positive test in test/. + + Handles two layouts: + - File: test/positiveX. → label='positiveX', scan_path=the file + - Directory: test/positiveX/ → for each positiveX_Y. inside, + label='positiveX_Y', scan_path=the file + """ + test_dir = os.path.join(query_path, "test") + if not os.path.isdir(test_dir): + return [] + + positives = [] + for entry in os.listdir(test_dir): + if not entry.startswith("positive"): + continue + full_path = os.path.join(test_dir, entry) + if os.path.isdir(full_path): + # Directory: positiveX/ — scan each file inside individually + for file in os.listdir(full_path): + file_path = os.path.join(full_path, file) + if not os.path.isfile(file_path): + continue + label = os.path.splitext(file)[0] # e.g. 'positive2_1' + after = label[len("positive"):] + if not after or not after[0].isdigit(): # skip positive_expected_result etc. + continue + positives.append((label, file_path)) + else: + # File: positiveX. + suffix = entry[len("positive"):].split(".")[0] + if not suffix.isdigit(): + continue + positives.append((f"positive{suffix}", full_path)) + + positives.sort(key=lambda x: x[0]) + return positives + + +def run_query_scans(query_id: str, query_path: str) -> list[tuple[str, str, int]]: + positives = find_positive_tests(query_path) + if not positives: + print(f"[WARN] No positive tests found in {query_path}/test, skipping.", file=sys.stderr) + return [] + + payloads_dir = os.path.join(query_path, "payloads") + os.makedirs(payloads_dir, exist_ok=True) + + output_path = os.path.join(query_path, "results") + os.sep + os.makedirs(output_path, exist_ok=True) + + failed = [] + for label, scan_path in positives: + payload_path = os.path.join(payloads_dir, f"{label}.json") + output_name = f"{label}.json" + print(f"\n -> {label}: {os.path.relpath(scan_path, REPO_ROOT)}") + rc = run_scan(query_id, scan_path, payload_path, output_path, output_name) + if rc != 0: + failed.append((scan_path, payload_path, rc)) + + collect_and_write_expected_results(query_path) + return failed + + +def collect_and_write_expected_results(query_path: str) -> None: + """ + Read all positive*.json result files from results/, extract findings, + sort by (fileName, line), and write test/positive_expected_result.json. + """ + results_dir = os.path.join(query_path, "results") + if not os.path.isdir(results_dir): + return + + entries = [] + for filename in sorted(os.listdir(results_dir)): + if not filename.startswith("positive") or not filename.endswith(".json"): + continue + with open(os.path.join(results_dir, filename), encoding="utf-8") as f: + data = json.load(f) + + for query in data.get("queries", []): + query_name = query.get("query_name", "") + severity = query.get("severity", "") + for file_entry in query.get("files", []): + entry = { + "queryName": query_name, + "severity": severity, + "line": file_entry.get("line", 0), + "fileName": os.path.basename(file_entry.get("file_name", "")), + "resourceType": file_entry.get("resource_type", ""), + "resourceName": file_entry.get("resource_name", ""), + "searchKey": file_entry.get("search_key", ""), + "searchValue": file_entry.get("search_value", ""), + "expectedValue":file_entry.get("expected_value", ""), + "actualValue": file_entry.get("actual_value", ""), + "issueType": file_entry.get("issue_type", ""), + "similarityID": file_entry.get("similarity_id", ""), + "search_line": file_entry.get("search_line", 0), + } + entries.append({k: entry[k] for k in FIELD_ORDER}) + + entries.sort(key=lambda x: ( + x["fileName"], x["line"], x["issueType"], x["searchKey"], x["similarityID"] + )) + + out_path = os.path.join(query_path, "test", "positive_expected_result.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(entries, f, indent=2) + f.write("\n") + + print(f" -> Written {len(entries)} entries to {os.path.relpath(out_path, REPO_ROOT)}") + + +def iter_queries(): + """Yield (query_id, query_path) for every query found under assets/queries.""" + for dirpath, _, filenames in os.walk(QUERIES_DIR): + if "metadata.json" not in filenames: + continue + metadata = os.path.join(dirpath, "metadata.json") + with open(metadata, encoding="utf-8") as f: + data = json.load(f) + query_id = data.get("id") + if not query_id: + print(f"[WARN] No 'id' field in {metadata}, skipping.", file=sys.stderr) + continue + yield query_id, dirpath + + +def main(): + args = parse_args() + + if args.run_all: + all_failed = [] + queries = list(iter_queries()) + print(f"Found {len(queries)} queries. Starting scans...\n") + time.sleep(5) # mudar para menos, isto é só para efeitos de debug + for query_id, query_path in queries: + print(f"\n=== {os.path.relpath(query_path, REPO_ROOT)} ({query_id}) ===") + failed = run_query_scans(query_id, query_path) + all_failed.extend(failed) + + print("\n" + "=" * 60) + if all_failed: + print(f"[SUMMARY] {len(all_failed)} scan(s) failed:") + for scan_path, payload_path, rc in all_failed: + print(f" - {os.path.relpath(scan_path, REPO_ROOT)} → exit {rc}") + sys.exit(1) + else: + print(f"[SUMMARY] All scans completed successfully.") + sys.exit(0) + else: + if not args.queryPath: + print("[ERROR] --queryPath is required when not using --run-all.", file=sys.stderr) + sys.exit(1) + query_path = os.path.normpath(os.path.join(REPO_ROOT, args.queryPath)) + failed = run_query_scans(args.queryID, query_path) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() From ba8691f2fb6315a24997841894707d71be37ea6b Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:07:14 +0000 Subject: [PATCH 03/10] reverted some changes --- .../generate-positive-expected-result.py | 226 ---------- e2e-report.html | 425 ------------------ 2 files changed, 651 deletions(-) delete mode 100644 .github/scripts/generate-positive-expected-results/generate-positive-expected-result.py delete mode 100644 e2e-report.html diff --git a/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py b/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py deleted file mode 100644 index 4d87b61dacd..00000000000 --- a/.github/scripts/generate-positive-expected-results/generate-positive-expected-result.py +++ /dev/null @@ -1,226 +0,0 @@ -import argparse -import json -import os -import subprocess -import sys -import time - -FIELD_ORDER = [ - "queryName", "severity", "line", "fileName", - "resourceType", "resourceName", "searchKey", "searchValue", - "expectedValue", "actualValue", "issueType", "similarityID", "search_line", -] - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) -QUERIES_DIR = os.path.join(REPO_ROOT, "assets", "queries") - - -def parse_args(): - parser = argparse.ArgumentParser(description="Run a KICS scan for a given query.") - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("--run-all", action="store_true", help="Run scans for all queries under assets/queries.") - group.add_argument("--queryID", help="The query ID to scan.") - parser.add_argument("--queryPath", help="The base path of the query (required without --run-all).") - return parser.parse_args() - - -def build_command(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> list[str]: - main_go = os.path.join(REPO_ROOT, "cmd", "console", "main.go") - - return [ - "go", "run", main_go, - "scan", - "-p", scan_path, - "-o", output_path, - "--output-name", output_name, - "-i", query_id, - "-d", payload_path, - "-v", - "--experimental-queries", - "--bom", - "--enable-openapi-refs", - "--ignore-on-exit", "results" - ] - - -def run_scan(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> int: - command = build_command(query_id, scan_path, payload_path, output_path, output_name) - - print("Running command:") - print(" ".join(command)) - print("-" * 60) - - try: - result = subprocess.run(command, cwd=REPO_ROOT, check=True) - return result.returncode - except subprocess.CalledProcessError as e: - print(f"\n[ERROR] Scan failed with return code {e.returncode}.", file=sys.stderr) - return e.returncode - except FileNotFoundError: - print("\n[ERROR] 'go' not found. Make sure Go is installed and in your PATH.", file=sys.stderr) - return 1 - - -def find_positive_tests(query_path: str) -> list[tuple[str, str]]: - """ - Return a sorted list of (label, scan_path) for each positive test in test/. - - Handles two layouts: - - File: test/positiveX. → label='positiveX', scan_path=the file - - Directory: test/positiveX/ → for each positiveX_Y. inside, - label='positiveX_Y', scan_path=the file - """ - test_dir = os.path.join(query_path, "test") - if not os.path.isdir(test_dir): - return [] - - positives = [] - for entry in os.listdir(test_dir): - if not entry.startswith("positive"): - continue - full_path = os.path.join(test_dir, entry) - if os.path.isdir(full_path): - # Directory: positiveX/ — scan each file inside individually - for file in os.listdir(full_path): - file_path = os.path.join(full_path, file) - if not os.path.isfile(file_path): - continue - label = os.path.splitext(file)[0] # e.g. 'positive2_1' - after = label[len("positive"):] - if not after or not after[0].isdigit(): # skip positive_expected_result etc. - continue - positives.append((label, file_path)) - else: - # File: positiveX. - suffix = entry[len("positive"):].split(".")[0] - if not suffix.isdigit(): - continue - positives.append((f"positive{suffix}", full_path)) - - positives.sort(key=lambda x: x[0]) - return positives - - -def run_query_scans(query_id: str, query_path: str) -> list[tuple[str, str, int]]: - positives = find_positive_tests(query_path) - if not positives: - print(f"[WARN] No positive tests found in {query_path}/test, skipping.", file=sys.stderr) - return [] - - payloads_dir = os.path.join(query_path, "payloads") - os.makedirs(payloads_dir, exist_ok=True) - - output_path = os.path.join(query_path, "results") + os.sep - os.makedirs(output_path, exist_ok=True) - - failed = [] - for label, scan_path in positives: - payload_path = os.path.join(payloads_dir, f"{label}.json") - output_name = f"{label}.json" - print(f"\n -> {label}: {os.path.relpath(scan_path, REPO_ROOT)}") - rc = run_scan(query_id, scan_path, payload_path, output_path, output_name) - if rc != 0: - failed.append((scan_path, payload_path, rc)) - - collect_and_write_expected_results(query_path) - return failed - - -def collect_and_write_expected_results(query_path: str) -> None: - """ - Read all positive*.json result files from results/, extract findings, - sort by (fileName, line), and write test/positive_expected_result.json. - """ - results_dir = os.path.join(query_path, "results") - if not os.path.isdir(results_dir): - return - - entries = [] - for filename in sorted(os.listdir(results_dir)): - if not filename.startswith("positive") or not filename.endswith(".json"): - continue - with open(os.path.join(results_dir, filename), encoding="utf-8") as f: - data = json.load(f) - - for query in data.get("queries", []): - query_name = query.get("query_name", "") - severity = query.get("severity", "") - for file_entry in query.get("files", []): - entry = { - "queryName": query_name, - "severity": severity, - "line": file_entry.get("line", 0), - "fileName": os.path.basename(file_entry.get("file_name", "")), - "resourceType": file_entry.get("resource_type", ""), - "resourceName": file_entry.get("resource_name", ""), - "searchKey": file_entry.get("search_key", ""), - "searchValue": file_entry.get("search_value", ""), - "expectedValue":file_entry.get("expected_value", ""), - "actualValue": file_entry.get("actual_value", ""), - "issueType": file_entry.get("issue_type", ""), - "similarityID": file_entry.get("similarity_id", ""), - "search_line": file_entry.get("search_line", 0), - } - entries.append({k: entry[k] for k in FIELD_ORDER}) - - entries.sort(key=lambda x: ( - x["fileName"], x["line"], x["issueType"], x["searchKey"], x["similarityID"] - )) - - out_path = os.path.join(query_path, "test", "positive_expected_result.json") - with open(out_path, "w", encoding="utf-8") as f: - json.dump(entries, f, indent=2) - f.write("\n") - - print(f" -> Written {len(entries)} entries to {os.path.relpath(out_path, REPO_ROOT)}") - - -def iter_queries(): - """Yield (query_id, query_path) for every query found under assets/queries.""" - for dirpath, _, filenames in os.walk(QUERIES_DIR): - if "metadata.json" not in filenames: - continue - metadata = os.path.join(dirpath, "metadata.json") - with open(metadata, encoding="utf-8") as f: - data = json.load(f) - query_id = data.get("id") - if not query_id: - print(f"[WARN] No 'id' field in {metadata}, skipping.", file=sys.stderr) - continue - yield query_id, dirpath - - -def main(): - args = parse_args() - - if args.run_all: - all_failed = [] - queries = list(iter_queries()) - print(f"Found {len(queries)} queries. Starting scans...\n") - time.sleep(5) # mudar para menos, isto é só para efeitos de debug - for query_id, query_path in queries: - print(f"\n=== {os.path.relpath(query_path, REPO_ROOT)} ({query_id}) ===") - failed = run_query_scans(query_id, query_path) - all_failed.extend(failed) - - print("\n" + "=" * 60) - if all_failed: - print(f"[SUMMARY] {len(all_failed)} scan(s) failed:") - for scan_path, payload_path, rc in all_failed: - print(f" - {os.path.relpath(scan_path, REPO_ROOT)} → exit {rc}") - sys.exit(1) - else: - print(f"[SUMMARY] All scans completed successfully.") - sys.exit(0) - else: - if not args.queryPath: - print("[ERROR] --queryPath is required when not using --run-all.", file=sys.stderr) - sys.exit(1) - query_path = os.path.normpath(os.path.join(REPO_ROOT, args.queryPath)) - failed = run_query_scans(args.queryID, query_path) - sys.exit(1 if failed else 0) - - -if __name__ == "__main__": - main() diff --git a/e2e-report.html b/e2e-report.html deleted file mode 100644 index bb488914a39..00000000000 --- a/e2e-report.html +++ /dev/null @@ -1,425 +0,0 @@ -KICS Scan Result
KICS E2E REPORT03/23/2026 13:11KICS.IO

E2E Tests Result: fail

Time Elapsed: 1273.29s

Kics Docker Image:

Filters

PASSED -161
FAILED -1
ALL -162
Test: _should_exclude_provided_paths_[E2E-CLI-027]_0 -Elapsed: 225.26s -Status: pass
Test: _should_generate_and_save_a_log_file_with_log_level_[E2E-CLI-039]_0 -Elapsed: 122.69s -Status: pass
Test: _should_perform_a_scan_and_create_a_result_and_payload_file_[E2E-CLI-033]_0 -Elapsed: 322.69s -Status: pass
Test: scan_should_generate_equivalent_payloads_for_OpenAPI_YAML_and_JSON_files_with_circular_references_[E2E-CLI-099]_0 -Elapsed: 93.53s -Status: pass
Test: scan_should_generate_equivalent_payloads_for_OpenAPI_YAML_and_JSON_files_with_circular_references_[E2E-CLI-099]_1 -Elapsed: 93.25s -Status: pass
Test: should_check_if_output_path_is_invalid_[E2E-CLI-082]_0 -Elapsed: 1.51s -Status: pass
Test: should_check_if_output_path_is_valid_[E2E-CLI-081]_0 -Elapsed: 25.33s -Status: pass
Test: should_create_a_payload_file_[E2E-CLI-005]_0 -Elapsed: 105.12s -Status: pass
Test: should_disable_colored_output_in_the_CLI_[E2E-CLI-015]_0 -Elapsed: 16.82s -Status: pass
Test: should_display_CPU_usage_in_the_CLI_[E2E-CLI-022]_0 -Elapsed: 14.57s -Status: pass
Test: should_display_an_error_message_about_unknown_argument_[E2E-CLI-010]_0 -Elapsed: 1.5s -Status: pass
Test: should_display_an_error_of_invalid_flag_combination_[E2E-CLI-004]_0 -Elapsed: 0.79s -Status: pass
Test: should_display_an_error_of_invalid_flag_combination_[E2E-CLI-004]_1 -Elapsed: 0.72s -Status: pass
Test: should_display_an_error_regarding_missing_--results_flag_[E2E-CLI-060]_0 -Elapsed: 2.22s -Status: pass
Test: should_display_an_error_regarding_missing_-p_flag_[E2E-CLI-003]_0 -Elapsed: 1.12s -Status: pass
Test: should_display_line_references_in_the_payload_file_[E2E-CLI-047]_0 -Elapsed: 321.71s -Status: pass
Test: should_display_memory_usage_in_the_CLI_[E2E-CLI-023]_0 -Elapsed: 19.25s -Status: pass
Test: should_display_minimal-ui_[E2E-CLI-012]_0 -Elapsed: 14.23s -Status: pass
Test: should_display_the_kics_help_text_[E2E-CLI-001]_0 -Elapsed: 0.2s -Status: pass
Test: should_display_the_kics_remediate_help_text_[E2E-CLI-059]_0 -Elapsed: 1.6s -Status: pass
Test: should_display_the_kics_scan_help_text_[E2E-CLI-002]_0 -Elapsed: 0.63s -Status: pass
Test: should_display_the_kics_version_[E2E-CLI-024]_0 -Elapsed: 1.57s -Status: pass
Test: should_display_verbose_information_in_the_CLI_[E2E-CLI-017]_0 -Elapsed: 14.54s -Status: pass
Test: should_download_and_scan_the_provided_git_path_[E2E-CLI-041]_0 -Elapsed: 94.22s -Status: pass
Test: should_download_and_scan_the_provided_http_path/file_[E2E-CLI-042]_0 -Elapsed: 10.87s -Status: pass
Test: should_exclude_provided_categories_[E2E-CLI-018]_0 -Elapsed: 121.7s -Status: pass
Test: should_exclude_provided_queries_[E2E-CLI-020]_0 -Elapsed: 14.01s -Status: pass
Test: should_exclude_provided_similarity_ID_results_[E2E-CLI-035]_0 -Elapsed: 18.02s -Status: pass
Test: should_exclude_provided_similarity_ID_results_[E2E-CLI-035]_1 -Elapsed: 1.26s -Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_0 -Elapsed: 94.56s -Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_1 -Elapsed: 15.88s -Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_2 -Elapsed: 1.65s -Status: pass
Test: should_exclude_queries_by_given_severities_[E2E-CLI-044]_3 -Elapsed: 32.7s -Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_0 -Elapsed: 1.11s -Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_1 -Elapsed: 0.89s -Status: pass
Test: should_execute_only_queries_of_specific_cloud_provider_[E2E-CLI-043]_2 -Elapsed: 122.92s -Status: pass
Test: should_export_the_result_files_to_provided_path_[E2E-CLI-030]_0 -Elapsed: 19.88s -Status: pass
Test: should_export_the_results_based_on_different_formats_[E2E-CLI-031]_0 -Elapsed: 145.1s -Status: pass
Test: should_export_the_results_based_on_different_formats_[E2E-CLI-031]_1 -Elapsed: 9.16s -Status: pass
Test: should_export_the_results_based_on_report_formats_[E2E-CLI-040]_0 -Elapsed: 125s -Status: pass
Test: should_export_the_results_based_on_the_formats_provided_by_this_flag,_with_critical_severity_[E2E-CLI-090]_0 -Elapsed: 8.52s -Status: pass
Test: should_fail-on_provided_values_[E2E-CLI-025]_0 -Elapsed: 17.37s -Status: pass
Test: should_fail-on_provided_values_[E2E-CLI-025]_1 -Elapsed: 21.08s -Status: pass
Test: should_fetch_descriptions_from_environment_[E2E-CLI-046]_0 -Elapsed: 18.86s -Status: pass
Test: should_generate_a_valid_ID_[E2E-CLI-006]_0 -Elapsed: 0.91s -Status: pass
Test: should_generate_and_save_a_log_file_[E2E-CLI-038]_0 -Elapsed: 117.71s -Status: pass
Test: should_hide_all_output_text_in_CLI_[E2E-CLI-008]_0 -Elapsed: 75.09s -Status: pass
Test: should_hide_the_progress_bar_in_the_CLI_[E2E-CLI-009]_0 -Elapsed: 14.38s -Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_0 -Elapsed: 308.11s -Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_1 -Elapsed: 315.66s -Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_2 -Elapsed: 20.17s -Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_3 -Elapsed: 61.92s -Status: pass
Test: should_ignore_files/code-blocks/code-lines_during_the_scan_[E2E-CLI-053]_4 -Elapsed: 375.26s -Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_0 -Elapsed: 0.55s -Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_1 -Elapsed: 1.88s -Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_2 -Elapsed: 113.78s -Status: pass
Test: should_ignore_on_exit_provided_flags_[E2E-CLI-026]_3 -Elapsed: 130.93s -Status: pass
Test: should_include_bill_of_materials_in_results_output_[E2E-CLI-050]_0 -Elapsed: 325.92s -Status: pass
Test: should_include_bill_of_materials_in_results_output_[E2E-CLI-050]_1 -Elapsed: 1.34s -Status: pass
Test: should_list_all_supported_platforms_[E2E-CLI-013]_0 -Elapsed: 0.8s -Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_0 -Elapsed: 134.19s -Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_1 -Elapsed: 4.52s -Status: pass
Test: should_load_a_config_file_[E2E-CLI-029]_2 -Elapsed: 1.02s -Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_0 -Elapsed: 5.22s -Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_1 -Elapsed: 5.88s -Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_2 -Elapsed: 5.22s -Status: pass
Test: should_load_and_execute_queries_from_a_custom_path_[E2E-CLI-051]_3 -Elapsed: 2.14s -Status: pass
Test: should_load_and_execute_queries_from_multiple_paths_[E2E-CLI-054]_0 -Elapsed: 8.16s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_0 -Elapsed: 324.73s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_1 -Elapsed: 321.29s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_2 -Elapsed: 325.99s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_3 -Elapsed: 7.44s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_4 -Elapsed: 10.21s -Status: pass
Test: should_load_custom_secrets_rules_from_provided_path_[E2E-CLI-048]_5 -Elapsed: 7.17s -Status: pass
Test: should_load_descriptions_from_a_custom_server_[E2E-CLI-052]_0 -Elapsed: 12.76s -Status: pass
Test: should_load_descriptions_from_a_custom_server_[E2E-CLI-052]_1 -Elapsed: 310.5s -Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_0 -Elapsed: 10.55s -Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_1 -Elapsed: 125.89s -Status: pass
Test: should_load_libraries_from_the_provided_path_[E2E-CLI-049]_2 -Elapsed: 2.77s -Status: pass
Test: should_modify_log_format_messages_in_the_CLI_[E2E-CLI-028]_0 -Elapsed: 23.86s -Status: pass
Test: should_modify_the_default_preview-lines_value_[E2E-CLI-014]_0 -Elapsed: 13.89s -Status: pass
Test: should_not_display_messages_in_the_CLI_[E2E-CLI-034]_0 -Elapsed: 18.63s -Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_0 -Elapsed: 10.21s -Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_1 -Elapsed: 19.9s -Status: pass
Test: should_not_execute_secret_queries_[E2E-CLI-045]_2 -Elapsed: 16.3s -Status: pass
Test: should_perform_a_scan,_present_two_results,_without_the_same_similarity_id_[E2E-CLI-077]_0 -Elapsed: 9.38s -Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-086]_0 -Elapsed: 333.14s -Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-087]_0 -Elapsed: 333.77s -Status: pass
Test: should_perform_a_scan_and_finish_successfully_[E2E-CLI-088]_0 -Elapsed: 348.53s -Status: pass
Test: should_perform_a_scan_and_not_ignore_the_entire_project_[E2E-CLI-092]_0 -Elapsed: 131.87s -Status: pass
Test: should_perform_a_scan_and_return_three_different_similarity_ids_on_the_results_[E2E-CLI-078]_0 -Elapsed: 8.19s -Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_block_[E2E-CLI-084]_0 -Elapsed: 13.02s -Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_file_[E2E-CLI-083]_0 -Elapsed: 9.49s -Status: pass
Test: should_perform_a_scan_and_return_zero_results_ignoring_the_query_[E2E-CLI-085]_0 -Elapsed: 11.74s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_0 -Elapsed: 17.26s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_1 -Elapsed: 12.92s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_2 -Elapsed: 12.63s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_3 -Elapsed: 1.32s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_4 -Elapsed: 11.92s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_5 -Elapsed: 12.05s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_6 -Elapsed: 9.43s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_7 -Elapsed: 0.9s -Status: pass
Test: should_perform_a_scan_including_only_specific_queries_[E2E-CLI-036]_8 -Elapsed: 0.87s -Status: pass
Test: should_perform_a_scan_on_bicep_file_with_existing_resources_and_return_exit_code_0_[E2E-CLI-103]_0 -Elapsed: 21.66s -Status: pass
Test: should_perform_a_scan_on_bicep_files_and_create_a_result_and_payload_file_[E2E-CLI-091]_0 -Elapsed: 25.89s -Status: pass
Test: should_perform_a_scan_on_zip_file_with_UTF-16_files_and_return_exit_code_40_[E2E-CLI-102]_0 -Elapsed: 338.59s -Status: pass
Test: should_perform_a_scan_saving_the_reports_in_sarif_format,_showing_no_cwe_field_on_results_[E2E-CLI-080]_0 -Elapsed: 2.32s -Status: pass
Test: should_perform_a_scan_saving_the_reports_in_sarif_format,_showing_the_cwe_on_results_[E2E-CLI-079]_0 -Elapsed: 3.55s -Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_critical_severity_and_return_exit_code_60_[E2E-CLI-089]_0 -Elapsed: 2.71s -Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_similarity_ids_unique,_showing_the_old_similarity_id_[E2E-CLI-096]_0 -Elapsed: 3.55s -Status: pass
Test: should_perform_a_scan_successfully_giving_results_with_similarity_ids_unique_without_showing_the_old_similarity_id_[E2E-CLI-097]_0 -Elapsed: 7.53s -Status: pass
Test: should_perform_a_scan_without_detecting_anything_since_no_files_are_scanned_because_of_max_file_size_[E2E-CLI-076]_0 -Elapsed: 2.2s -Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_1]_0 -Elapsed: 0.97s -Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_2]_0 -Elapsed: 1.79s -Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_3]_0 -Elapsed: 2.19s -Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_4]_0 -Elapsed: 1.58s -Status: pass
Test: should_perform_a_scans_successfully_giving_results_with_old_severity_and_return_exit_code_according_to_old_severity_[E2E-CLI-093_5]_0 -Elapsed: 0.88s -Status: pass
Test: should_perform_a_simple_scan_[E2E-CLI-007]_0 -Elapsed: 71.47s -Status: pass
Test: should_perform_a_valid_analyze_[E2E-CLI-066]_0 -Elapsed: 2.28s -Status: pass
Test: should_perform_a_valid_scan_[E2E-CLI-065]_0 -Elapsed: 6.3s -Status: pass
Test: should_perform_a_valid_scan_and_and_detect_ansible_[E2E-CLI-075]_0 -Elapsed: 10.12s -Status: pass
Test: should_perform_a_valid_scan_and_get_the_variables_using_a_variables_path_as_a_comment_[E2E-CLI-062]_0 -Elapsed: 345.95s -Status: pass
Test: should_perform_a_valid_scan_and_get_the_variables_using_a_variables_path_as_a_flag_[E2E-CLI-063]_0 -Elapsed: 342.68s -Status: pass
Test: should_perform_a_valid_scan_and_ignore_the_experimental_queries_[E2E-CLI-069]_0 -Elapsed: 11.41s -Status: pass
Test: should_perform_a_valid_scan_and_not_ignore_the_experimental_queries_[E2E-CLI-070]_0 -Elapsed: 5.93s -Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-071]_0 -Elapsed: 8.19s -Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-073]_0 -Elapsed: 7.25s -Status: pass
Test: should_perform_a_valid_scan_and_not_resolve_references_[E2E-CLI-094]_0 -Elapsed: 3.13s -Status: pass
Test: should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0 -Elapsed: 23.58s -Status: fail -
View Details
Test: Test_E2E_CLI/should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0
--- FAIL: Test_E2E_CLI/should_perform_a_valid_scan_and_recover_from_a_corrupted_dockerfile_[E2E-CLI-068]_0 (23.58s)
View extra elements
Expected in fixtures: -{ -"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", -"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 5, -"queryName": "Healthcheck Instruction Missing", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" -} -}
Actual output -{ -"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", -"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 1, -"queryName": "Healthcheck Instruction Missing", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" -} -}
Show full Actual and Expected
Expected in fixtures: -"actual_value": "The 'Dockerfile' does not contain any 'USER' instruction", -"expected_value": "The 'Dockerfile' should contain the 'USER' instruction", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 1, -"queryName": "Missing User Instruction", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "821b0a105dd9ebb6b14489251615d94813a578feb68624113a52ba9c1668ecb7" -} -{ -"actual_value": "FROM alpine:latest'", -"expected_value": "FROM alpine:latest:'version' where version should not be 'latest'", -"file_name": "", -"issue_type": "IncorrectValue", -"line": 1, -"queryName": "Image Version Using 'latest'", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "a32938ae4d056569e5cfda73594f6cb276d4f9ee14a848fb6b642b14defee892" -} -{ -"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", -"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 5, -"queryName": "Healthcheck Instruction Missing", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" -}
Actual output: -"actual_value": "The 'Dockerfile' does not contain any 'USER' instruction", -"expected_value": "The 'Dockerfile' should contain the 'USER' instruction", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 1, -"queryName": "Missing User Instruction", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "821b0a105dd9ebb6b14489251615d94813a578feb68624113a52ba9c1668ecb7" -} -{ -"actual_value": "FROM alpine:latest'", -"expected_value": "FROM alpine:latest:'version' where version should not be 'latest'", -"file_name": "", -"issue_type": "IncorrectValue", -"line": 1, -"queryName": "Image Version Using 'latest'", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "a32938ae4d056569e5cfda73594f6cb276d4f9ee14a848fb6b642b14defee892" -} -{ -"actual_value": "Dockerfile doesn't contain instruction 'HEALTHCHECK'", -"expected_value": "Dockerfile should contain instruction 'HEALTHCHECK'", -"file_name": "", -"issue_type": "MissingAttribute", -"line": 1, -"queryName": "Healthcheck Instruction Missing", -"search_key": "FROM={{alpine:latest}}", -"search_line": -1, -"search_value": "", -"similarity_id": "ec1a1e998229f963313752def87ce5027e89997cfb4180c9b356414566f5a029" -}
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-072]_0 -Elapsed: 8.68s -Status: pass
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-074]_0 -Elapsed: 19.11s -Status: pass
Test: should_perform_a_valid_scan_and_resolve_references_[E2E-CLI-095]_0 -Elapsed: 3.67s -Status: pass
Test: should_perform_a_valid_scan_and_return_one_HIGH_result_[E2E-CLI-098]_0 -Elapsed: 3.99s -Status: pass
Test: should_perform_a_valid_scan_but_ignore_broken_symlinks_and_symlinks_that_create_endless_loops_[E2E-CLI-067]_0 -Elapsed: 4.7s -Status: pass
Test: should_perform_a_valid_scan_while_evaluating_the_terraform_functions_with_unknown_type_[E2E-CLI-071]_0 -Elapsed: 356.01s -Status: pass
Test: should_perform_a_valid_scan_with_--exclude-type_flag_[E2E-CLI-061]_0 -Elapsed: 27.23s -Status: pass
Test: should_perform_a_valid_scan_with_-t_flag_[E2E-CLI-011]_0 -Elapsed: 16.38s -Status: pass
Test: should_perform_a_valid_scan_with_json/yaml_file_with_##_in_non_ref_[E2E-CLI-064]_0 -Elapsed: 160.31s -Status: pass
Test: should_remediate_all_remediation_found_[E2E-CLI-057]_0 -Elapsed: 41.88s -Status: pass
Test: should_remediate_the_recommendations_pointed_in_include-ids_flag_[E2E-CLI-058]_0 -Elapsed: 18.38s -Status: pass
Test: should_remove_empty_files_from_helm_install_render_[E2E-CLI-104]_0 -Elapsed: 6.11s -Status: pass
Test: should_resolve_openapi_files_and_return_results_in_different_files_[E2E-CLI-055]_0 -Elapsed: 104.85s -Status: pass
Test: should_run_a_scan_in_multiple_paths_[E2E-CLI-019]_0 -Elapsed: 116.9s -Status: pass
Test: should_run_only_provided_queries_and_exclude_provided_results_[E2E-CLI-037]_0 -Elapsed: 8.05s -Status: pass
Test: should_run_only_provided_queries_and_exclude_provided_results_[E2E-CLI-037]_1 -Elapsed: 13.3s -Status: pass
Test: should_scan_a_folder_with_FHIR_files_and_skip_them_successfully_[E2E-CLI-100]_0 -Elapsed: 4.66s -Status: pass
Test: should_scan_a_folder_with_azure-pipelines-vscode_files_and_skip_them_successfully_[E2E-CLI-100]_0 -Elapsed: 10.82s -Status: pass
Test: should_scan_a_folder_with_blacklisted_files,_with_flag_--exclude-type,_and_skip_them_successfully_[E2E-CLI-100]_0 -Elapsed: 15.9s -Status: pass
Test: should_scan_a_folder_with_blacklisted_files,_with_flag_--type,_and_skip_them_successfully_[E2E-CLI-100]_0 -Elapsed: 2.47s -Status: pass
Test: should_set_the_results_output_name_[E2E-CLI-032]_0 -Elapsed: 144.37s -Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_0 -Elapsed: 0.57s -Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_1 -Elapsed: 1.1s -Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_2 -Elapsed: 0.9s -Status: pass
Test: should_throw_error_messages_for_kics'_flags_[E2E-CLI-016]_3 -Elapsed: 0.88s -Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_0 -Elapsed: 17.5s -Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_1 -Elapsed: 21.53s -Status: pass
Test: should_timeout_queries_when_reaching_the_timeout_limit_[E2E-CLI-056]_2 -Elapsed: 3.34s -Status: pass
Test: should_validate_the_kics_result_status_code_[E2E-CLI-021]_0 -Elapsed: 118.32s -Status: pass
Test: should_validate_the_kics_result_status_code_[E2E-CLI-021]_1 -Elapsed: 7s -Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_exclude_from_scan_[E2E-CLI-101]_0 -Elapsed: 1.49s -Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_scan,_long_version_[E2E-CLI-101]_0 -Elapsed: 1.4s -Status: pass
Test: test_bicep_scan_fail_with_list_of_platform_types_to_scan,_short_version_[E2E-CLI-101]_0 -Elapsed: 1.44s -Status: pass
\ No newline at end of file From 690b1b59105565a1e9c7192e5776ad62078c8f7b Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:08:03 +0000 Subject: [PATCH 04/10] added script --- .../generate_positive_expected_result.py | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 .github/scripts/generate-positive-expected-results/generate_positive_expected_result.py diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py new file mode 100644 index 00000000000..4d87b61dacd --- /dev/null +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -0,0 +1,226 @@ +import argparse +import json +import os +import subprocess +import sys +import time + +FIELD_ORDER = [ + "queryName", "severity", "line", "fileName", + "resourceType", "resourceName", "searchKey", "searchValue", + "expectedValue", "actualValue", "issueType", "similarityID", "search_line", +] + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) +QUERIES_DIR = os.path.join(REPO_ROOT, "assets", "queries") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run a KICS scan for a given query.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--run-all", action="store_true", help="Run scans for all queries under assets/queries.") + group.add_argument("--queryID", help="The query ID to scan.") + parser.add_argument("--queryPath", help="The base path of the query (required without --run-all).") + return parser.parse_args() + + +def build_command(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> list[str]: + main_go = os.path.join(REPO_ROOT, "cmd", "console", "main.go") + + return [ + "go", "run", main_go, + "scan", + "-p", scan_path, + "-o", output_path, + "--output-name", output_name, + "-i", query_id, + "-d", payload_path, + "-v", + "--experimental-queries", + "--bom", + "--enable-openapi-refs", + "--ignore-on-exit", "results" + ] + + +def run_scan(query_id: str, scan_path: str, payload_path: str, output_path: str, output_name: str) -> int: + command = build_command(query_id, scan_path, payload_path, output_path, output_name) + + print("Running command:") + print(" ".join(command)) + print("-" * 60) + + try: + result = subprocess.run(command, cwd=REPO_ROOT, check=True) + return result.returncode + except subprocess.CalledProcessError as e: + print(f"\n[ERROR] Scan failed with return code {e.returncode}.", file=sys.stderr) + return e.returncode + except FileNotFoundError: + print("\n[ERROR] 'go' not found. Make sure Go is installed and in your PATH.", file=sys.stderr) + return 1 + + +def find_positive_tests(query_path: str) -> list[tuple[str, str]]: + """ + Return a sorted list of (label, scan_path) for each positive test in test/. + + Handles two layouts: + - File: test/positiveX. → label='positiveX', scan_path=the file + - Directory: test/positiveX/ → for each positiveX_Y. inside, + label='positiveX_Y', scan_path=the file + """ + test_dir = os.path.join(query_path, "test") + if not os.path.isdir(test_dir): + return [] + + positives = [] + for entry in os.listdir(test_dir): + if not entry.startswith("positive"): + continue + full_path = os.path.join(test_dir, entry) + if os.path.isdir(full_path): + # Directory: positiveX/ — scan each file inside individually + for file in os.listdir(full_path): + file_path = os.path.join(full_path, file) + if not os.path.isfile(file_path): + continue + label = os.path.splitext(file)[0] # e.g. 'positive2_1' + after = label[len("positive"):] + if not after or not after[0].isdigit(): # skip positive_expected_result etc. + continue + positives.append((label, file_path)) + else: + # File: positiveX. + suffix = entry[len("positive"):].split(".")[0] + if not suffix.isdigit(): + continue + positives.append((f"positive{suffix}", full_path)) + + positives.sort(key=lambda x: x[0]) + return positives + + +def run_query_scans(query_id: str, query_path: str) -> list[tuple[str, str, int]]: + positives = find_positive_tests(query_path) + if not positives: + print(f"[WARN] No positive tests found in {query_path}/test, skipping.", file=sys.stderr) + return [] + + payloads_dir = os.path.join(query_path, "payloads") + os.makedirs(payloads_dir, exist_ok=True) + + output_path = os.path.join(query_path, "results") + os.sep + os.makedirs(output_path, exist_ok=True) + + failed = [] + for label, scan_path in positives: + payload_path = os.path.join(payloads_dir, f"{label}.json") + output_name = f"{label}.json" + print(f"\n -> {label}: {os.path.relpath(scan_path, REPO_ROOT)}") + rc = run_scan(query_id, scan_path, payload_path, output_path, output_name) + if rc != 0: + failed.append((scan_path, payload_path, rc)) + + collect_and_write_expected_results(query_path) + return failed + + +def collect_and_write_expected_results(query_path: str) -> None: + """ + Read all positive*.json result files from results/, extract findings, + sort by (fileName, line), and write test/positive_expected_result.json. + """ + results_dir = os.path.join(query_path, "results") + if not os.path.isdir(results_dir): + return + + entries = [] + for filename in sorted(os.listdir(results_dir)): + if not filename.startswith("positive") or not filename.endswith(".json"): + continue + with open(os.path.join(results_dir, filename), encoding="utf-8") as f: + data = json.load(f) + + for query in data.get("queries", []): + query_name = query.get("query_name", "") + severity = query.get("severity", "") + for file_entry in query.get("files", []): + entry = { + "queryName": query_name, + "severity": severity, + "line": file_entry.get("line", 0), + "fileName": os.path.basename(file_entry.get("file_name", "")), + "resourceType": file_entry.get("resource_type", ""), + "resourceName": file_entry.get("resource_name", ""), + "searchKey": file_entry.get("search_key", ""), + "searchValue": file_entry.get("search_value", ""), + "expectedValue":file_entry.get("expected_value", ""), + "actualValue": file_entry.get("actual_value", ""), + "issueType": file_entry.get("issue_type", ""), + "similarityID": file_entry.get("similarity_id", ""), + "search_line": file_entry.get("search_line", 0), + } + entries.append({k: entry[k] for k in FIELD_ORDER}) + + entries.sort(key=lambda x: ( + x["fileName"], x["line"], x["issueType"], x["searchKey"], x["similarityID"] + )) + + out_path = os.path.join(query_path, "test", "positive_expected_result.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(entries, f, indent=2) + f.write("\n") + + print(f" -> Written {len(entries)} entries to {os.path.relpath(out_path, REPO_ROOT)}") + + +def iter_queries(): + """Yield (query_id, query_path) for every query found under assets/queries.""" + for dirpath, _, filenames in os.walk(QUERIES_DIR): + if "metadata.json" not in filenames: + continue + metadata = os.path.join(dirpath, "metadata.json") + with open(metadata, encoding="utf-8") as f: + data = json.load(f) + query_id = data.get("id") + if not query_id: + print(f"[WARN] No 'id' field in {metadata}, skipping.", file=sys.stderr) + continue + yield query_id, dirpath + + +def main(): + args = parse_args() + + if args.run_all: + all_failed = [] + queries = list(iter_queries()) + print(f"Found {len(queries)} queries. Starting scans...\n") + time.sleep(5) # mudar para menos, isto é só para efeitos de debug + for query_id, query_path in queries: + print(f"\n=== {os.path.relpath(query_path, REPO_ROOT)} ({query_id}) ===") + failed = run_query_scans(query_id, query_path) + all_failed.extend(failed) + + print("\n" + "=" * 60) + if all_failed: + print(f"[SUMMARY] {len(all_failed)} scan(s) failed:") + for scan_path, payload_path, rc in all_failed: + print(f" - {os.path.relpath(scan_path, REPO_ROOT)} → exit {rc}") + sys.exit(1) + else: + print(f"[SUMMARY] All scans completed successfully.") + sys.exit(0) + else: + if not args.queryPath: + print("[ERROR] --queryPath is required when not using --run-all.", file=sys.stderr) + sys.exit(1) + query_path = os.path.normpath(os.path.join(REPO_ROOT, args.queryPath)) + failed = run_query_scans(args.queryID, query_path) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() From f2108c6c69a875c39e42de43744ee08d161d8ba6 Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Mon, 30 Mar 2026 15:05:23 +0100 Subject: [PATCH 05/10] changed KICS documentation --- docs/creating-queries.md | 94 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/docs/creating-queries.md b/docs/creating-queries.md index 5471685e7ba..cacdf8ab640 100644 --- a/docs/creating-queries.md +++ b/docs/creating-queries.md @@ -297,6 +297,100 @@ If the **query.rego** file implements more than one query, the **metadata.json** } ``` +Filling positive_expected_result.json: + +The `positive_expected_result.json` file is a JSON array where each entry represents a single expected finding from a positive test file. Each entry supports the following fields: + +- `queryName` the name of the query as defined in `metadata.json` +- `severity` the severity level of the finding (`CRITICAL`, `HIGH`, `MEDIUM`, `LOW`, or `INFO`) +- `line` the line number in the positive test file where the vulnerability is detected +- `fileName` the name of the positive test file (e.g., `positive1.tf`, `positive.yaml`) +- `resourceType` the type of the resource flagged by the finding (e.g., `aws_cloudtrail`, `community.aws.elb_application_lb`) +- `resourceName` the name or label of the specific resource instance +- `searchKey` the search key path used by KICS to locate the vulnerability in the original document +- `searchValue` an additional value used to distinguish findings when multiple results point to the same line +- `expectedValue` a description of the expected (secure) value +- `actualValue` a description of the actual (insecure) value detected +- `issueType` the type of issue: `IncorrectValue`, `MissingAttribute`, or `RedundantAttribute` +- `similarityID` a hash that uniquely identifies the finding, used for deduplication and tracking +- `search_line` the search line path used by KICS for line detection; set to `-1` when not applicable + +Example: + +```json +[ + { + "queryName": "Authentication Without MFA", + "severity": "LOW", + "line": 2, + "fileName": "positive.yaml", + "resourceType": "community.aws.sts_assume_role", + "resourceName": "Assume an existing role", + "searchKey": "name={{Assume an existing role}}.{{community.aws.sts_assume_role}}", + "searchValue": "mfa_token", + "expectedValue": "sts_assume_role.mfa_token should be set", + "actualValue": "sts_assume_role.mfa_token is undefined", + "issueType": "MissingAttribute", + "similarityID": "0863129177e5f7d0f0fc55d63426f810f58f35c1270b64f4b57fbd1d8a3639cc", + "search_line": 2 + }, + { + "queryName": "Authentication Without MFA", + "severity": "LOW", + "line": 9, + "fileName": "positive.yaml", + "resourceType": "sts_assume_role", + "resourceName": "Hello", + "searchKey": "name={{Hello}}.{{sts_assume_role}}", + "searchValue": "mfa_serial_number", + "expectedValue": "sts_assume_role.mfa_serial_number should be set", + "actualValue": "sts_assume_role.mfa_serial_number is undefined", + "issueType": "MissingAttribute", + "similarityID": "89628f77eee62d856d5523656cdcbc1be1bfca9a1aaed79ffa9871979c947202", + "search_line": 9 + } +] +``` + +Instead of filling this file manually, you can use the helper script provided at `.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py`. The script runs a KICS scan against each positive test file, collects the findings, and produces a correctly formatted `positive_expected_result.json`. + +**Important:** The script must be run from the **script's own directory** (`.github/scripts/generate-positive-expected-results/`), since it resolves the repository root and all other paths relative to its own location. It also requires **Go** to be installed and available in your `PATH`. + +The script supports two modes of operation: + +**Single query mode** — requires both `--queryID` and `--queryPath`: + +```bash +cd .github/scripts/generate-positive-expected-results/ +python generate_positive_expected_result.py \ + --queryID \ + --queryPath +``` + +For example: + +```bash +cd .github/scripts/generate-positive-expected-results/ +python generate_positive_expected_result.py \ + --queryID "8173d5eb-96b5-4aa6-a71b-ecfa153c123d" \ + --queryPath "assets/queries/terraform/aws/cloudtrail_multi_region_disabled" +``` + +**All queries mode** — scans every query under `assets/queries/`: + +```bash +cd .github/scripts/generate-positive-expected-results/ +python generate_positive_expected_result.py --run-all +``` + +| Flag | Required | Description | +|---|---|---| +| `--queryID` | Yes (unless `--run-all`) | The UUID of the query to scan, found in the query's `metadata.json` under the `id` field. | +| `--queryPath` | Yes (unless `--run-all`) | The relative path (from the repository root) to the query directory containing `query.rego` and `metadata.json`. | +| `--run-all` | No | Iterates over all queries under `assets/queries/`, reading each `metadata.json` to obtain the query ID automatically. Mutually exclusive with `--queryID`. | + +The script discovers positive test files in the query's `test/` directory, runs a KICS scan for each one, collects and merges the findings, sorts them by file name, line number, issue type, search key, and similarity ID, and writes the result to `test/positive_expected_result.json`. + Filling query.rego: - `documentId` id of the sample where the vulnerability occurs From 487a6fa3b7c16663af16c339583ee23d7a34034b Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:12:11 +0100 Subject: [PATCH 06/10] added --kics_compute_new_simid flag --- .../generate_positive_expected_result.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py index 4d87b61dacd..1147a0643b7 100644 --- a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -40,7 +40,8 @@ def build_command(query_id: str, scan_path: str, payload_path: str, output_path: "--experimental-queries", "--bom", "--enable-openapi-refs", - "--ignore-on-exit", "results" + "--kics_compute_new_simid" + #"--ignore-on-exit", "results" ] From 00a603155d42967a1e8fd5899dd4b8e27e1dd9d9 Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:48:21 +0100 Subject: [PATCH 07/10] added --kics_compute_new_simid --- .../generate_positive_expected_result.py | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py index 1147a0643b7..5bcba2fb5c8 100644 --- a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -3,7 +3,6 @@ import os import subprocess import sys -import time FIELD_ORDER = [ "queryName", "severity", "line", "fileName", @@ -11,6 +10,8 @@ "expectedValue", "actualValue", "issueType", "similarityID", "search_line", ] +KICS_RESULT_CODES = {0, 20, 30, 40, 50, 60} + SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) QUERIES_DIR = os.path.join(REPO_ROOT, "assets", "queries") @@ -40,8 +41,7 @@ def build_command(query_id: str, scan_path: str, payload_path: str, output_path: "--experimental-queries", "--bom", "--enable-openapi-refs", - "--kics_compute_new_simid" - #"--ignore-on-exit", "results" + "--kics_compute_new_simid" ] @@ -53,11 +53,10 @@ def run_scan(query_id: str, scan_path: str, payload_path: str, output_path: str, print("-" * 60) try: - result = subprocess.run(command, cwd=REPO_ROOT, check=True) + result = subprocess.run(command, cwd=REPO_ROOT) + if result.returncode not in KICS_RESULT_CODES: + print(f"\n[ERROR] Scan failed with return code {result.returncode}.", file=sys.stderr) return result.returncode - except subprocess.CalledProcessError as e: - print(f"\n[ERROR] Scan failed with return code {e.returncode}.", file=sys.stderr) - return e.returncode except FileNotFoundError: print("\n[ERROR] 'go' not found. Make sure Go is installed and in your PATH.", file=sys.stderr) return 1 @@ -93,21 +92,21 @@ def find_positive_tests(query_path: str) -> list[tuple[str, str]]: continue positives.append((label, file_path)) else: - # File: positiveX. + # File: positive. or positiveX. suffix = entry[len("positive"):].split(".")[0] - if not suffix.isdigit(): - continue + if suffix and not suffix.isdigit(): + continue # skip positive_expected_result.json etc. positives.append((f"positive{suffix}", full_path)) positives.sort(key=lambda x: x[0]) return positives -def run_query_scans(query_id: str, query_path: str) -> list[tuple[str, str, int]]: +def run_query_scans(query_id: str, query_path: str) -> tuple[list[tuple[str, str, int]], bool]: positives = find_positive_tests(query_path) if not positives: print(f"[WARN] No positive tests found in {query_path}/test, skipping.", file=sys.stderr) - return [] + return [], False payloads_dir = os.path.join(query_path, "payloads") os.makedirs(payloads_dir, exist_ok=True) @@ -121,21 +120,22 @@ def run_query_scans(query_id: str, query_path: str) -> list[tuple[str, str, int] output_name = f"{label}.json" print(f"\n -> {label}: {os.path.relpath(scan_path, REPO_ROOT)}") rc = run_scan(query_id, scan_path, payload_path, output_path, output_name) - if rc != 0: + if rc not in KICS_RESULT_CODES: failed.append((scan_path, payload_path, rc)) - collect_and_write_expected_results(query_path) - return failed + written = collect_and_write_expected_results(query_path) + return failed, written -def collect_and_write_expected_results(query_path: str) -> None: +def collect_and_write_expected_results(query_path: str) -> bool: """ Read all positive*.json result files from results/, extract findings, - sort by (fileName, line), and write test/positive_expected_result.json. + sort by (fileName, line, issueType, searchKey, similarityID), and write + test/positive_expected_result.json. Returns True if the file was written. """ results_dir = os.path.join(query_path, "results") if not os.path.isdir(results_dir): - return + return False entries = [] for filename in sorted(os.listdir(results_dir)): @@ -144,7 +144,8 @@ def collect_and_write_expected_results(query_path: str) -> None: with open(os.path.join(results_dir, filename), encoding="utf-8") as f: data = json.load(f) - for query in data.get("queries", []): + all_findings = data.get("queries", []) + data.get("bill_of_materials", []) + for query in all_findings: query_name = query.get("query_name", "") severity = query.get("severity", "") for file_entry in query.get("files", []): @@ -165,6 +166,9 @@ def collect_and_write_expected_results(query_path: str) -> None: } entries.append({k: entry[k] for k in FIELD_ORDER}) + if not entries: + return False + entries.sort(key=lambda x: ( x["fileName"], x["line"], x["issueType"], x["searchKey"], x["similarityID"] )) @@ -175,6 +179,7 @@ def collect_and_write_expected_results(query_path: str) -> None: f.write("\n") print(f" -> Written {len(entries)} entries to {os.path.relpath(out_path, REPO_ROOT)}") + return True def iter_queries(): @@ -196,30 +201,35 @@ def main(): args = parse_args() if args.run_all: - all_failed = [] + all_failed = [] + written_count = 0 queries = list(iter_queries()) - print(f"Found {len(queries)} queries. Starting scans...\n") - time.sleep(5) # mudar para menos, isto é só para efeitos de debug - for query_id, query_path in queries: - print(f"\n=== {os.path.relpath(query_path, REPO_ROOT)} ({query_id}) ===") - failed = run_query_scans(query_id, query_path) + total = len(queries) + width = len(str(total)) + print(f"Found {total} queries. Starting scans...\n") + for idx, (query_id, query_path) in enumerate(queries, start=1): + print(f"\n[{idx:{width}d}/{total}] {os.path.relpath(query_path, REPO_ROOT)}") + failed, written = run_query_scans(query_id, query_path) all_failed.extend(failed) + if written: + written_count += 1 print("\n" + "=" * 60) + print(f"[SUMMARY] {written_count}/{total} positive_expected_result.json written") if all_failed: - print(f"[SUMMARY] {len(all_failed)} scan(s) failed:") + print(f" {len(all_failed)} scan(s) failed:") for scan_path, payload_path, rc in all_failed: print(f" - {os.path.relpath(scan_path, REPO_ROOT)} → exit {rc}") sys.exit(1) else: - print(f"[SUMMARY] All scans completed successfully.") + print(" All scans completed successfully.") sys.exit(0) else: if not args.queryPath: print("[ERROR] --queryPath is required when not using --run-all.", file=sys.stderr) sys.exit(1) query_path = os.path.normpath(os.path.join(REPO_ROOT, args.queryPath)) - failed = run_query_scans(args.queryID, query_path) + failed, _ = run_query_scans(args.queryID, query_path) sys.exit(1 if failed else 0) From 993f45bed9785a4476eccec45c874ba36323201b Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:50:25 +0100 Subject: [PATCH 08/10] added 1 to result codes --- .../generate_positive_expected_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py index 5bcba2fb5c8..8bed496ecff 100644 --- a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -10,7 +10,7 @@ "expectedValue", "actualValue", "issueType", "similarityID", "search_line", ] -KICS_RESULT_CODES = {0, 20, 30, 40, 50, 60} +KICS_RESULT_CODES = {0, 1, 20, 30, 40, 50, 60} SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) From b4936970bdf96af67874834129303e5f86226887 Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:33:47 +0100 Subject: [PATCH 09/10] fixing script to take into account scenarios where there is multiple tests with the same name but different extension --- .../generate_positive_expected_result.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py index 8bed496ecff..e16362543aa 100644 --- a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -1,6 +1,7 @@ import argparse import json import os +import re import subprocess import sys @@ -12,6 +13,12 @@ KICS_RESULT_CODES = {0, 1, 20, 30, 40, 50, 60} + +def _natural_sort_key(s: str): + """'positive2.tf' → ['positive', 2, '.tf'] so numeric parts sort numerically.""" + return [int(c) if c.isdigit() else c for c in re.split(r'(\d+)', s)] + + SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.normpath(os.path.join(SCRIPT_DIR, "../../..")) QUERIES_DIR = os.path.join(REPO_ROOT, "assets", "queries") @@ -67,9 +74,13 @@ def find_positive_tests(query_path: str) -> list[tuple[str, str]]: Return a sorted list of (label, scan_path) for each positive test in test/. Handles two layouts: - - File: test/positiveX. → label='positiveX', scan_path=the file + - File: test/positiveX. → label='positiveX_', scan_path=the file - Directory: test/positiveX/ → for each positiveX_Y. inside, - label='positiveX_Y', scan_path=the file + label='positiveX_Y_', scan_path=the file + + The extension is always included in the label so that files with the same + base name but different extensions (e.g. positive1.json / positive1.yaml) + produce distinct payloads and result files. """ test_dir = os.path.join(query_path, "test") if not os.path.isdir(test_dir): @@ -86,19 +97,21 @@ def find_positive_tests(query_path: str) -> list[tuple[str, str]]: file_path = os.path.join(full_path, file) if not os.path.isfile(file_path): continue - label = os.path.splitext(file)[0] # e.g. 'positive2_1' - after = label[len("positive"):] - if not after or not after[0].isdigit(): # skip positive_expected_result etc. + base_label = os.path.splitext(file)[0] # e.g. 'positive2_1' + after = base_label[len("positive"):] + if not after or not after[0].isdigit(): # skip positive_expected_result etc. continue - positives.append((label, file_path)) + ext = os.path.splitext(file)[1].lstrip(".") # e.g. 'json', 'yaml', 'tf' + positives.append((f"{base_label}_{ext}", file_path)) else: # File: positive. or positiveX. suffix = entry[len("positive"):].split(".")[0] if suffix and not suffix.isdigit(): continue # skip positive_expected_result.json etc. - positives.append((f"positive{suffix}", full_path)) + ext = os.path.splitext(entry)[1].lstrip(".") # e.g. 'json', 'yaml', 'tf' + positives.append((f"positive{suffix}_{ext}", full_path)) - positives.sort(key=lambda x: x[0]) + positives.sort(key=lambda x: _natural_sort_key(x[0])) return positives @@ -144,7 +157,6 @@ def collect_and_write_expected_results(query_path: str) -> bool: with open(os.path.join(results_dir, filename), encoding="utf-8") as f: data = json.load(f) - all_findings = data.get("queries", []) + data.get("bill_of_materials", []) for query in all_findings: query_name = query.get("query_name", "") severity = query.get("severity", "") @@ -170,7 +182,7 @@ def collect_and_write_expected_results(query_path: str) -> bool: return False entries.sort(key=lambda x: ( - x["fileName"], x["line"], x["issueType"], x["searchKey"], x["similarityID"] + _natural_sort_key(x["fileName"]), x["line"], x["issueType"], x["searchKey"], x["similarityID"] )) out_path = os.path.join(query_path, "test", "positive_expected_result.json") From a0c83c3a81e5f7361b879e0bdee349b8413a1bea Mon Sep 17 00:00:00 2001 From: Ricardo Jesus <219317970+cx-ricardo-jesus@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:37:01 +0100 Subject: [PATCH 10/10] redefining all_findings variable --- .../generate_positive_expected_result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py index e16362543aa..f5e9beb4849 100644 --- a/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py +++ b/.github/scripts/generate-positive-expected-results/generate_positive_expected_result.py @@ -157,6 +157,7 @@ def collect_and_write_expected_results(query_path: str) -> bool: with open(os.path.join(results_dir, filename), encoding="utf-8") as f: data = json.load(f) + all_findings = data.get("queries", []) + data.get("bill_of_materials", []) for query in all_findings: query_name = query.get("query_name", "") severity = query.get("severity", "")