diff --git a/test/collections_test.rb b/test/collections_test.rb
index 657c832f40b5..d75cabe903fc 100644
--- a/test/collections_test.rb
+++ b/test/collections_test.rb
@@ -3,6 +3,11 @@
 describe "collections" do
   collections.each do |collection|
     describe "#{collection} collection" do
+      it "has valid YAML frontmatter" do
+        error = yaml_syntax_error_for(collections_dir, collection)
+        assert_nil error, error
+      end
+
       unless ENV["AUTOCORRECT_RENAMED_REPOS"] == "1"
         it "has a valid name" do
           assert valid_collection?(collection), invalid_collection_message(collection)
diff --git a/test/test_helper.rb b/test/test_helper.rb
index b7af1b1b983c..cf205d617803 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -235,17 +235,36 @@ def valid_uri_scheme?(scheme)
   %w[http https].include?(scheme.downcase)
 end
 
-def metadata_for(dir, name)
+def frontmatter_for(dir, name)
   path = File.join(dir, name, "index.md")
   return unless File.file?(path)
 
   parts = File.read(path).split("---", 3)
   return unless parts.size >= 2
 
+  parts[1]
+end
+
+def metadata_for(dir, name)
+  frontmatter = frontmatter_for(dir, name)
+  return unless frontmatter
+
+  begin
+    YAML.safe_load(frontmatter)
+  rescue Psych::SyntaxError
+    nil
+  end
+end
+
+def yaml_syntax_error_for(dir, name)
+  frontmatter = frontmatter_for(dir, name)
+  return unless frontmatter
+
   begin
-    YAML.safe_load(parts[1])
+    YAML.safe_load(frontmatter)
+    nil
   rescue Psych::SyntaxError => error
-    flunk "invalid YAML: #{error.message}"
+    "invalid YAML in #{File.join(dir, name, 'index.md')}: #{error.message}"
   end
 end
 
diff --git a/test/topics_test.rb b/test/topics_test.rb
index fd7ea440e960..3609e2b262c6 100644
--- a/test/topics_test.rb
+++ b/test/topics_test.rb
@@ -9,6 +9,11 @@
 
   topics.each do |topic|
     describe "#{topic} topic" do
+      it "has valid YAML frontmatter" do
+        error = yaml_syntax_error_for(topics_dir, topic)
+        assert_nil error, error
+      end
+
       it "has a valid name" do
         assert valid_topic?(topic), invalid_topic_message(topic)
       end