DEV: Restore the documentation format in system tests (#21471)

.github/workflows/tests.yml

@@ -180,7 +180,7 @@ jobs:
 
       - name: Plugin RSpec
         if: matrix.build_type == 'backend' && matrix.target == 'plugins'
-        run: bin/rake plugin:turbo_spec
+        run: bin/rake plugin:turbo_spec['*','--verbose --format documentation']
 
       - name: Plugin QUnit
         if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
@@ -197,11 +197,11 @@ jobs:
 
       - name: Core System Tests
         if: matrix.build_type == 'system' && matrix.target == 'core'
-        run: PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --verbose spec/system
+        run: PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --verbose --format documentation spec/system
 
       - name: Plugin System Tests
         if: matrix.build_type == 'system' && matrix.target == 'plugins'
-        run: LOAD_PLUGINS=1 PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --verbose plugins/*/spec/system
+        run: LOAD_PLUGINS=1 PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --verbose --format documentation plugins/*/spec/system
         timeout-minutes: 30
 
       - name: Upload failed system test screenshots

lib/tasks/documentation.rake

@@ -16,7 +16,7 @@ def generate_chat_documentation
   ]
   `yarn --silent jsdoc --readme plugins/chat/README.md -c #{config} #{files.join(" ")} -d #{destination}`
 
-  # unecessary files
+  # unnecessary files
   %w[
     documentation/chat/frontend/scripts/prism.min.js
     documentation/chat/frontend/scripts/prism.js

lib/tasks/plugin.rake

@@ -168,13 +168,13 @@ task "plugin:install_gems", :plugin do |t, args|
   puts "Done"
 end
 
-def spec(plugin, parallel: false)
+def spec(plugin, parallel: false, argv: nil)
   params = []
   params << "--profile" if !parallel
   params << "--fail-fast" if ENV["RSPEC_FAILFAST"]
   params << "--seed #{ENV["RSPEC_SEED"]}" if Integer(ENV["RSPEC_SEED"], exception: false)
+  params << argv if argv
 
-  ruby = `which ruby`.strip
   # reject system specs as they are slow and need dedicated setup
   files =
     Dir.glob("./plugins/#{plugin}/spec/**/*_spec.rb").reject { |f| f.include?("spec/system/") }.sort
@@ -187,15 +187,15 @@ def spec(plugin, parallel: false)
 end
 
 desc "run plugin specs"
-task "plugin:spec", :plugin do |t, args|
+task "plugin:spec", %i[plugin argv] do |_, args|
   args.with_defaults(plugin: "*")
-  spec(args[:plugin])
+  spec(args[:plugin], argv: args[:argv])
 end
 
 desc "run plugin specs in parallel"
-task "plugin:turbo_spec", :plugin do |t, args|
+task "plugin:turbo_spec", %i[plugin argv] do |_, args|
   args.with_defaults(plugin: "*")
-  spec(args[:plugin], parallel: true)
+  spec(args[:plugin], parallel: true, argv: args[:argv])
 end
 
 desc "run plugin qunit tests"

lib/turbo_tests.rb

@@ -15,6 +15,7 @@ require "parallel_tests/rspec/runner"
 require "./lib/turbo_tests/reporter"
 require "./lib/turbo_tests/runner"
 require "./lib/turbo_tests/json_rows_formatter"
+require "./lib/turbo_tests/documentation_formatter"
 
 module TurboTests
   FakeException = Struct.new(:backtrace, :message, :cause)
@@ -54,9 +55,18 @@ module TurboTests
   end
 
   FakeExample =
-    Struct.new(:execution_result, :location, :full_description, :metadata, :location_rerun_argument)
+    Struct.new(
+      :execution_result,
+      :location,
+      :description,
+      :full_description,
+      :metadata,
+      :location_rerun_argument,
+      :process_id,
+    )
+
   class FakeExample
-    def self.from_obj(obj)
+    def self.from_obj(obj, process_id)
       obj = obj.symbolize_keys
       metadata = obj[:metadata].symbolize_keys
 
@@ -71,9 +81,11 @@ module TurboTests
       new(
         FakeExecutionResult.from_obj(obj[:execution_result]),
         obj[:location],
+        obj[:description],
         obj[:full_description],
         metadata,
         obj[:location_rerun_argument],
+        process_id,
       )
     end
 

lib/turbo_tests/documentation_formatter.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+RSpec::Support.require_rspec_core "formatters/base_text_formatter"
+
+module TurboTests
+  # An RSpec formatter that prepends the process id to all messages
+  class DocumentationFormatter < ::RSpec::Core::Formatters::BaseTextFormatter
+    RSpec::Core::Formatters.register(self, :example_failed, :example_passed, :example_pending)
+
+    def example_passed(notification)
+      output.puts RSpec::Core::Formatters::ConsoleCodes.wrap(
+                    "[#{notification.example.process_id}] #{notification.example.full_description}",
+                    :success,
+                  )
+      output.flush
+    end
+
+    def example_pending(notification)
+      message = notification.example.execution_result.pending_message
+      output.puts RSpec::Core::Formatters::ConsoleCodes.wrap(
+                    "[#{notification.example.process_id}] #{notification.example.full_description}" \
+                      "(PENDING: #{message})",
+                    :pending,
+                  )
+      output.flush
+    end
+
+    def example_failed(notification)
+      output.puts RSpec::Core::Formatters::ConsoleCodes.wrap(
+                    "[#{notification.example.process_id}] #{notification.example.full_description}" \
+                      "(FAILED - #{next_failure_index})",
+                    :failure,
+                  )
+      output.flush
+    end
+
+    private
+
+    def next_failure_index
+      @next_failure_index ||= 0
+      @next_failure_index += 1
+    end
+  end
+end

lib/turbo_tests/json_rows_formatter.rb

@@ -49,6 +49,7 @@ module TurboTests
       {
         execution_result: execution_result_to_json(example.execution_result),
         location: example.location,
+        description: example.description,
         full_description: example.full_description,
         metadata: {
           shared_group_inclusion_backtrace:

lib/turbo_tests/reporter.rb

@@ -35,6 +35,8 @@ module TurboTests
           case name
           when "p", "progress"
             RSpec::Core::Formatters::ProgressFormatter
+          when "d", "documentation"
+            TurboTests::DocumentationFormatter
           else
             Kernel.const_get(name)
           end

lib/turbo_tests/runner.rb

@@ -215,13 +215,13 @@ module TurboTests
           message = @messages.pop
           case message[:type]
           when "example_passed"
-            example = FakeExample.from_obj(message[:example])
+            example = FakeExample.from_obj(message[:example], message[:process_id])
             @reporter.example_passed(example)
           when "example_pending"
-            example = FakeExample.from_obj(message[:example])
+            example = FakeExample.from_obj(message[:example], message[:process_id])
             @reporter.example_pending(example)
           when "example_failed"
-            example = FakeExample.from_obj(message[:example])
+            example = FakeExample.from_obj(message[:example], message[:process_id])
             @reporter.example_failed(example)
             @failure_count += 1
             if fail_fast_met
@@ -237,6 +237,7 @@ module TurboTests
             @error = true
           when "exit"
             exited += 1
+            @reporter.message("[#{message[:process_id]}] DONE (#{exited}/#{@num_processes + 1})")
             break if exited == @num_processes + 1
           else
             STDERR.puts("Unhandled message in main process: #{message}")