Check we executed all tests

Author: ChrisBr

ruby/lib/ci/queue/redis/build_record.rb

@@ -72,6 +72,8 @@ def record_success(id, stats: nil, skip_flaky_record: false, acknowledge: true)
           error_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline|
             pipeline.hdel(key('error-reports'), id)
             pipeline.hget(key('requeues-count'), id)
+            pipeline.sadd(key('success-reports'), id)
+            pipeline.expire(key('success-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)
           end
           record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)

ruby/lib/ci/queue/redis/worker.rb

@@ -69,6 +69,20 @@ def poll
             pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
             pipeline.expire(key('processed'), config.redis_ttl)
           end
+
+          # check we executed all tests
+          # only the master should perform this check because otherwise we will DDoS Redis when all workers
+          # try to fetch the processed tests at the same time
+          if master? && exhausted?
+            executed_tests = (redis.smembers(key('success-reports')) + redis.hkeys(key('error-reports')))
+            missing_tests = @index.keys - executed_tests
+
+            if missing_tests.size > 0
+              puts "ci-queue did not process all tests!"
+              puts missing_tests
+              exit!(46)
+            end
+          end
         rescue *CONNECTION_ERRORS
         end