From e71d51652efa2a14e6576ab5daf13422d6cca890 Mon Sep 17 00:00:00 2001 From: Christian Bruckmayer Date: Mon, 17 Mar 2025 16:54:06 +0000 Subject: [PATCH 1/5] Run CI on pull requests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8c644979..c943ac72 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ name: Tests -on: [push] +on: [push, pull_request] jobs: ruby-tests: From c4eaa5212e81d311ceee50011fcddb574a9c457e Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Thu, 27 Mar 2025 10:28:00 -0400 Subject: [PATCH 2/5] Reapply "Dump JSON directly to file instead of buffering" This reverts commit 30e16eed40773b9f81074cb08fab3dfa8f322316. --- ruby/lib/minitest/queue/build_status_reporter.rb | 4 ++-- ruby/lib/minitest/queue/runner.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ruby/lib/minitest/queue/build_status_reporter.rb b/ruby/lib/minitest/queue/build_status_reporter.rb index ea59269f..d9344512 100644 --- a/ruby/lib/minitest/queue/build_status_reporter.rb +++ b/ruby/lib/minitest/queue/build_status_reporter.rb @@ -107,11 +107,11 @@ def progress end def write_failure_file(file) - File.write(file, error_reports.map(&:to_h).to_json) + JSON.dump(error_reports.map(&:to_h), File.open(file, 'w')) end def write_flaky_tests_file(file) - File.write(file, flaky_reports.to_json) + JSON.dump(flaky_reports, File.open(file, 'w')) end private diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb index 31183a82..79c1a917 100644 --- a/ruby/lib/minitest/queue/runner.rb +++ b/ruby/lib/minitest/queue/runner.rb @@ -323,7 +323,7 @@ def display_warnings(build) warnings = build.pop_warnings.map do |type, attributes| attributes.merge(type: type) end.compact - File.write(queue_config.warnings_file, warnings.to_json) + JSON.dump(warnings, File.open(queue_config.warnings_file, 'w')) end def run_tests_in_fork(queue) From 80050a346e1f570868d26fa204ff485afb67900f Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Thu, 27 Mar 2025 11:06:51 -0400 Subject: [PATCH 3/5] Use block so Files are closed after writing This fixes test errors where the file had no content. --- ruby/lib/minitest/queue/build_status_reporter.rb | 8 ++++++-- ruby/lib/minitest/queue/runner.rb | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ruby/lib/minitest/queue/build_status_reporter.rb b/ruby/lib/minitest/queue/build_status_reporter.rb index d9344512..39d57383 100644 --- a/ruby/lib/minitest/queue/build_status_reporter.rb +++ b/ruby/lib/minitest/queue/build_status_reporter.rb @@ -107,11 +107,15 @@ def progress end def write_failure_file(file) - JSON.dump(error_reports.map(&:to_h), File.open(file, 'w')) + File.open(file, 'w') do |f| + JSON.dump(error_reports.map(&:to_h), f) + end end def write_flaky_tests_file(file) - JSON.dump(flaky_reports, File.open(file, 'w')) + File.open(file, 'w') do |f| + JSON.dump(flaky_reports, f) + end end private diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb index 79c1a917..f76c5888 100644 --- a/ruby/lib/minitest/queue/runner.rb +++ b/ruby/lib/minitest/queue/runner.rb @@ -323,7 +323,9 @@ def display_warnings(build) warnings = build.pop_warnings.map do |type, attributes| attributes.merge(type: type) end.compact - JSON.dump(warnings, File.open(queue_config.warnings_file, 'w')) + File.open(queue_config.warnings_file, 'w') do |f| + JSON.dump(warnings, f) + end end def run_tests_in_fork(queue) From b0356b34b5458481c8afaa6b4120be8a61490408 Mon Sep 17 00:00:00 2001 From: Zarif Mahfuz Date: Thu, 27 Mar 2025 10:28:00 -0600 Subject: [PATCH 4/5] Specify flaky tests in junit.xml format --- ruby/lib/ci/queue/configuration.rb | 9 ++++++++- ruby/test/ci/queue/configuration_test.rb | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ruby/lib/ci/queue/configuration.rb b/ruby/lib/ci/queue/configuration.rb index 1941b81e..75455437 100644 --- a/ruby/lib/ci/queue/configuration.rb +++ b/ruby/lib/ci/queue/configuration.rb @@ -25,7 +25,14 @@ def from_env(env) def load_flaky_tests(path) return [] unless path - ::File.readlines(path).map(&:chomp).to_set + if ::File.extname(path) == ".xml" + require 'rexml/document' + REXML::Document.new(::File.read(path)).elements.to_a("//testcase").map do |element| + "#{element.attributes['classname']}##{element.attributes['name']}" + end.to_set + else + ::File.readlines(path).map(&:chomp).to_set + end rescue SystemCallError [] end diff --git a/ruby/test/ci/queue/configuration_test.rb b/ruby/test/ci/queue/configuration_test.rb index f79c6aef..18fce529 100644 --- a/ruby/test/ci/queue/configuration_test.rb +++ b/ruby/test/ci/queue/configuration_test.rb @@ -90,6 +90,21 @@ def test_parses_file_correctly flaky_tests = Configuration.load_flaky_tests('/tmp/does-not-exist') assert_empty flaky_tests + + Tempfile.open(['flaky_test_file', '.junit.xml']) do |file| + file.write(<<~XML) + + + + + XML + file.close + + flaky_tests = Configuration.load_flaky_tests(file.path) + assert_equal 2, flaky_tests.size + assert_includes flaky_tests, "ATest#test_foo" + assert_includes flaky_tests, "ATest#test_bar" + end end def test_queue_init_timeout_unset From 07515fa50d9ea82af968f4bb506f8b356967a730 Mon Sep 17 00:00:00 2001 From: Christian Bruckmayer Date: Wed, 26 Mar 2025 18:37:54 +0000 Subject: [PATCH 5/5] Export JunitXML failure file --- .../minitest/queue/build_status_reporter.rb | 84 +++++++++++++++++++ ruby/lib/minitest/queue/error_report.rb | 4 + ruby/test/integration/minitest_redis_test.rb | 8 ++ 3 files changed, 96 insertions(+) diff --git a/ruby/lib/minitest/queue/build_status_reporter.rb b/ruby/lib/minitest/queue/build_status_reporter.rb index 39d57383..f050a695 100644 --- a/ruby/lib/minitest/queue/build_status_reporter.rb +++ b/ruby/lib/minitest/queue/build_status_reporter.rb @@ -4,6 +4,88 @@ module Queue class BuildStatusReporter < Minitest::Reporters::BaseReporter include ::CI::Queue::OutputHelpers + class JUnitReporter + def initialize(file, error_reports) + @file = file + @error_reports = error_reports + end + + def write + File.open(@file, 'w+') do |file| + format_document(generate_document(@error_reports), file) + end + end + + private + + def generate_document(error_reports) + suites = error_reports.group_by { |error_report| error_report.test_suite } + + doc = REXML::Document.new(nil, { + :prologue_quote => :quote, + :attribute_quote => :quote, + }) + doc << REXML::XMLDecl.new('1.1', 'utf-8') + + testsuites = doc.add_element('testsuites') + suites.each do |suite, error_reports| + add_tests_to(testsuites, suite, error_reports) + end + + doc + end + + def format_document(doc, io) + formatter = REXML::Formatters::Pretty.new + formatter.write(doc, io) + io << "\n" + end + + def add_tests_to(testsuites, suite, error_reports) + testsuite = testsuites.add_element( + 'testsuite', + 'name' => suite, + 'filepath' => Minitest::Queue.relative_path(error_reports.first.test_file), + 'tests' => error_reports.count, + ) + + error_reports.each do |error_report| + attributes = { + 'name' => error_report.test_name, + 'classname' => error_report.test_suite, + } + attributes['lineno'] = error_report.test_line + + testcase = testsuite.add_element('testcase', attributes) + add_xml_message_for(testcase, error_report) + rescue REXML::ParseException, RuntimeError => error + puts error + end + end + + def add_xml_message_for(testcase, error_report) + failure = testcase.add_element('failure', 'type' => error_report.error_class, 'message' => truncate_message(error_report.to_s)) + failure.add_text(REXML::CData.new(message_for(error_report))) + end + + def truncate_message(message) + message.lines.first.chomp.gsub(/\e\[[^m]+m/, '') + end + + def project_root_path_matcher + @project_root_path_matcher ||= %r{(?<=\s)#{Regexp.escape(Minitest::Queue.project_root)}/} + end + + def message_for(error_report) + suite = error_report.test_suite + name = error_report.test_name + error = error_report.to_s + + message_with_relative_paths = error.gsub(project_root_path_matcher, '') + "\nFailure:\n#{name}(#{suite}) [#{Minitest::Queue.relative_path(error_report.test_file)}]:\n#{message_with_relative_paths}\n" + end + end + def initialize(supervisor:, **options) @supervisor = supervisor @build = supervisor.build @@ -110,6 +192,8 @@ def write_failure_file(file) File.open(file, 'w') do |f| JSON.dump(error_reports.map(&:to_h), f) end + xml_file = File.join(File.dirname(file), "#{File.basename(file, File.extname(file))}.xml") + JUnitReporter.new(xml_file, error_reports).write end def write_flaky_tests_file(file) diff --git a/ruby/lib/minitest/queue/error_report.rb b/ruby/lib/minitest/queue/error_report.rb index 281c248b..06dd3693 100644 --- a/ruby/lib/minitest/queue/error_report.rb +++ b/ruby/lib/minitest/queue/error_report.rb @@ -54,6 +54,10 @@ def test_name @data[:test_name] end + def error_class + @data[:error_class] + end + def test_and_module_name @data[:test_and_module_name] end diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index 59ccb769..7b8c1172 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -816,6 +816,14 @@ def test_redis_reporter_failure_file .sort_by { |failure_report| failure_report[:test_line] } .first + xml_file = File.join(File.dirname(failure_file), "#{File.basename(failure_file, File.extname(failure_file))}.xml") + xml_content = File.read(xml_file) + xml = REXML::Document.new(xml_content) + testcase = xml.elements['testsuites/testsuite/testcase[@name="test_bar"]'] + assert_equal "ATest", testcase.attributes['classname'] + assert_equal "test_bar", testcase.attributes['name'] + assert_equal "test/dummy_test.rb", testcase.parent.attributes['filepath'] + assert_equal "ATest", testcase.parent.attributes['name'] ## output and test_file expected = {