From 32178f7fb72be6ce88d56d1306d03bd16e1e8888 Mon Sep 17 00:00:00 2001 From: Nick Charlton Date: Wed, 22 Jan 2025 19:54:41 +0000 Subject: [PATCH 1/2] Introduce acceptance testing This is the culmination of several months work, and at least one side quest, in laying down the foundations of a test suite which will run Administrate completely from scratch. For each scenario, we create a new Rails application in a clean environment, install Administrate and run the generators. We can then test against this fresh application. Whilst we seem to have had a good track record when it comes to past and future compatibility, the hope is that by having such an outside perspective focused test suite, this will provide much broader confidence, especially where different frontend configurations are concerned. We'll also be able to test speculatively, such as against the `main` branch of Rails which isn't something we could've done with confidence in the past with so many different components involved. To do this, we introduce the `jet_black` gem, which helps with running "black box" tests for command line tools, and `sack_race`, which helps run and manage processes inside the test suite. To run the tests themselves, we use Capybara like we do elsewhere, only pointed against the Rails project we created as part of the test. It's not expected that we'd re-use our existing Capybara tests here, but rather create new ones for each scenario. As these tests are always going to be slow, we don't run these as usual when running `rspec`, so they excluded. We'll run them in GitHub Actions as standalone jobs. Over time, it's expected that we'd create acceptance tests for all supported configurations, covering different frontend tools, to projects with complex model configurations and beyond. https://github.com/odlp/jet_black https://github.com/nickcharlton/sack_race --- .rspec | 1 + Gemfile | 2 + Gemfile.lock | 9 ++++ bin/bundle | 3 -- gemfiles/pundit21.gemfile | 2 + gemfiles/rails60.gemfile | 2 + gemfiles/rails61.gemfile | 2 + gemfiles/rails70.gemfile | 2 + gemfiles/rails80.gemfile | 2 + spec/black_box/default_rails_app_spec.rb | 33 ++++++++++++ spec/spec_helper.rb | 3 +- spec/support/acceptance_helpers.rb | 65 ++++++++++++++++++++++++ spec/support/rails_process.rb | 31 +++++++++++ 13 files changed, 153 insertions(+), 4 deletions(-) delete mode 100755 bin/bundle create mode 100644 spec/black_box/default_rails_app_spec.rb create mode 100644 spec/support/acceptance_helpers.rb create mode 100644 spec/support/rails_process.rb diff --git a/.rspec b/.rspec index 83e16f8044..893e1291c4 100644 --- a/.rspec +++ b/.rspec @@ -1,2 +1,3 @@ --color --require spec_helper +--exclude "**/black_box/*_spec.rb" diff --git a/Gemfile b/Gemfile index 9d7051b65d..cb992f5b9c 100644 --- a/Gemfile +++ b/Gemfile @@ -36,7 +36,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/Gemfile.lock b/Gemfile.lock index a864489103..33c16b6ebb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,9 @@ +GIT + remote: https://github.com/nickcharlton/sack_race.git + revision: 89b4e5fff452aa6e48498aa356da87f8d1b65ae9 + specs: + sack_race (1.0.0.rc1) + PATH remote: . specs: @@ -174,6 +180,7 @@ GEM pp (>= 0.6.0) rdoc (>= 4.0.0) reline (>= 0.4.2) + jet_black (0.7.1) jsbundling-rails (1.3.1) railties (>= 6.0.0) json (2.13.2) @@ -426,6 +433,7 @@ DEPENDENCIES globalid i18n-tasks (= 1.0.15) image_processing + jet_black jsbundling-rails (~> 1.3) kaminari-i18n launchy @@ -434,6 +442,7 @@ DEPENDENCIES pundit rack-timeout redcarpet + sack_race! selenium-webdriver sentry-rails sentry-ruby diff --git a/bin/bundle b/bin/bundle deleted file mode 100755 index 66e9889e8b..0000000000 --- a/bin/bundle +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env ruby -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) -load Gem.bin_path('bundler', 'bundle') diff --git a/gemfiles/pundit21.gemfile b/gemfiles/pundit21.gemfile index 6d45e2abb5..229c420054 100644 --- a/gemfiles/pundit21.gemfile +++ b/gemfiles/pundit21.gemfile @@ -36,7 +36,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/gemfiles/rails60.gemfile b/gemfiles/rails60.gemfile index a747a55662..8e915df4b3 100644 --- a/gemfiles/rails60.gemfile +++ b/gemfiles/rails60.gemfile @@ -40,7 +40,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/gemfiles/rails61.gemfile b/gemfiles/rails61.gemfile index 7bc1a5e6ac..2a17e25f92 100644 --- a/gemfiles/rails61.gemfile +++ b/gemfiles/rails61.gemfile @@ -39,7 +39,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/gemfiles/rails70.gemfile b/gemfiles/rails70.gemfile index 35ca11ca84..ccc4997c30 100644 --- a/gemfiles/rails70.gemfile +++ b/gemfiles/rails70.gemfile @@ -37,7 +37,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/gemfiles/rails80.gemfile b/gemfiles/rails80.gemfile index 86de0cc8e5..2083380e99 100644 --- a/gemfiles/rails80.gemfile +++ b/gemfiles/rails80.gemfile @@ -37,7 +37,9 @@ group :test do gem "capybara" gem "database_cleaner" gem "formulaic" + gem "jet_black" gem "launchy" + gem "sack_race", github: "nickcharlton/sack_race" gem "selenium-webdriver" gem "shoulda-matchers" gem "timecop" diff --git a/spec/black_box/default_rails_app_spec.rb b/spec/black_box/default_rails_app_spec.rb new file mode 100644 index 0000000000..113446162a --- /dev/null +++ b/spec/black_box/default_rails_app_spec.rb @@ -0,0 +1,33 @@ +require "rails_helper" + +RSpec.describe "with a Default Rails app" do + it "works" do + create_rails_application + setup_post_model + setup_administrate + + app.start_and_wait_for_ready + + visit("/admin") + + expect(page).to have_css("header h1#page-title", text: "Posts") + click_on "New post" + + fill_in "Body", with: "Some words!" + fill_in "Title", with: "A post with words" + click_button "Create Post" + + expect(page).to have_content("Post was successfully created.") + expect(page).to have_content("Show Post #1") + + app.stop + end + + def session + @session ||= JetBlack::Session.new(options: {clean_bundler_env: true}) + end + + def app + @app ||= RailsProcess.new(session) + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bf7687ba2b..c466293543 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,5 @@ -# http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +require "jet_black/rspec" + RSpec.configure do |config| config.expect_with :rspec do |expectations| expectations.syntax = :expect diff --git a/spec/support/acceptance_helpers.rb b/spec/support/acceptance_helpers.rb new file mode 100644 index 0000000000..b6e16de944 --- /dev/null +++ b/spec/support/acceptance_helpers.rb @@ -0,0 +1,65 @@ +module AcceptanceHelpers + def create_rails_application + session.create_file("Gemfile", <<~RUBY) + source "http://rubygems.org" + + gem "rails" + RUBY + + session.run("bundle install") + + rails_new_cmd = [ + "bundle exec rails new .", + "--skip-test", + "--skip-spring", + "--force" + ].join(" ") + + expect(session.run(rails_new_cmd)) + .to be_a_success.and have_stdout("force Gemfile") + end + + def gem_path + File.expand_path(Rails.root + "../../") + end + + def setup_administrate + session.append_to_file("Gemfile", "gem \"administrate\", path: \"#{gem_path}\"") + session.run("bundle install") + + generate_dashboards_run = session.run( + "bundle exec rails g administrate:install" + ) + + expect(generate_dashboards_run) + .to be_a_success.and have_stdout("create app/dashboards/post_dashboard.rb") + end + + def setup_post_model + new_post_cmd = session.run([ + "bundle exec rails g model post", + "title:string", + "body:text", + "published_at:datetime" + ].join(" ")) + + expect(new_post_cmd) + .to be_a_success.and have_stdout("create app/models/post.rb") + + migrate_run = session.run("bundle exec rails db:migrate") + + expect(migrate_run).to be_a_success.and have_stdout("CreatePosts: migrated") + end +end + +RSpec.configure do |config| + config.include Capybara::DSL, type: :black_box + config.include Capybara::RSpecMatchers, type: :black_box + config.include AcceptanceHelpers, type: :black_box + + config.before(:each, type: :black_box) do + Capybara.current_driver = :chrome + Capybara.run_server = false + Capybara.app_host = "http://localhost:3000" + end +end diff --git a/spec/support/rails_process.rb b/spec/support/rails_process.rb new file mode 100644 index 0000000000..32f72f86f6 --- /dev/null +++ b/spec/support/rails_process.rb @@ -0,0 +1,31 @@ +class RailsProcess + def initialize(session) + @session = session + @process = + Bundler.with_unbundled_env do + SackRace::Process.new( + "bundle exec rails server", + { + chdir: session.directory, + verbose: true + } + ) + end + end + + def start_and_wait_for_ready + Bundler.with_unbundled_env do + process.add_handler(:ready, "Use Ctrl-C to stop\n") + process.start + process.wait_for_handler(:ready) + end + end + + def stop + process.stop + end + + private + + attr_reader :process, :session +end From c5356a331589e790b66e2558d7400228b794b987 Mon Sep 17 00:00:00 2001 From: Nick Charlton Date: Tue, 7 Oct 2025 14:27:58 +0100 Subject: [PATCH 2/2] RailsProcess -> TestAppProcess --- spec/black_box/default_rails_app_spec.rb | 2 +- spec/support/{rails_process.rb => test_app_process.rb} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename spec/support/{rails_process.rb => test_app_process.rb} (96%) diff --git a/spec/black_box/default_rails_app_spec.rb b/spec/black_box/default_rails_app_spec.rb index 113446162a..c6f7326c42 100644 --- a/spec/black_box/default_rails_app_spec.rb +++ b/spec/black_box/default_rails_app_spec.rb @@ -28,6 +28,6 @@ def session end def app - @app ||= RailsProcess.new(session) + @app ||= TestAppProcess.new(session) end end diff --git a/spec/support/rails_process.rb b/spec/support/test_app_process.rb similarity index 96% rename from spec/support/rails_process.rb rename to spec/support/test_app_process.rb index 32f72f86f6..4bcbc3d08c 100644 --- a/spec/support/rails_process.rb +++ b/spec/support/test_app_process.rb @@ -1,4 +1,4 @@ -class RailsProcess +class TestAppProcess def initialize(session) @session = session @process =